add checksum for sdb

include/util/taoserror.h

@@ -66,13 +66,14 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_INVALID_PTR                   TAOS_DEF_ERROR_CODE(0, 0x0103)
 #define TSDB_CODE_MEMORY_CORRUPTED              TAOS_DEF_ERROR_CODE(0, 0x0104)
 #define TSDB_CODE_FILE_CORRUPTED                TAOS_DEF_ERROR_CODE(0, 0x0106)
-#define TSDB_CODE_INVALID_MSG                   TAOS_DEF_ERROR_CODE(0, 0x0107)
-#define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0108)
-#define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0109)
-#define TSDB_CODE_REF_ID_REMOVED                TAOS_DEF_ERROR_CODE(0, 0x010A)
-#define TSDB_CODE_REF_INVALID_ID                TAOS_DEF_ERROR_CODE(0, 0x010B)
-#define TSDB_CODE_REF_ALREADY_EXIST             TAOS_DEF_ERROR_CODE(0, 0x010C)
-#define TSDB_CODE_REF_NOT_EXIST                 TAOS_DEF_ERROR_CODE(0, 0x010D)
+#define TSDB_CODE_CHECKSUM_ERROR                TAOS_DEF_ERROR_CODE(0, 0x0107)
+#define TSDB_CODE_INVALID_MSG                   TAOS_DEF_ERROR_CODE(0, 0x0108)
+#define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0109)
+#define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x010A)
+#define TSDB_CODE_REF_ID_REMOVED                TAOS_DEF_ERROR_CODE(0, 0x010B)
+#define TSDB_CODE_REF_INVALID_ID                TAOS_DEF_ERROR_CODE(0, 0x010C)
+#define TSDB_CODE_REF_ALREADY_EXIST             TAOS_DEF_ERROR_CODE(0, 0x010D)
+#define TSDB_CODE_REF_NOT_EXIST                 TAOS_DEF_ERROR_CODE(0, 0x010E)
 
 //client
 #define TSDB_CODE_TSC_INVALID_OPERATION         TAOS_DEF_ERROR_CODE(0, 0x0200)  //"Invalid Operation")

source/dnode/mnode/sdb/inc/sdbInt.h

@@ -41,7 +41,6 @@ typedef struct SSdbRaw {
   int8_t  sver;
   int8_t  status;
   int8_t  reserved;
-  int32_t cksum;
   int32_t dataLen;
   char    pData[];
 } SSdbRaw;

source/dnode/mnode/sdb/src/sdbFile.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "sdbInt.h"
 #include "tglobal.h"
+#include "tchecksum.h"
 
 static int32_t sdbCreateDir() {
   mDebug("start to create mnode at %s", tsMnodeDir);
@@ -56,10 +57,6 @@ static int32_t sdbRunDeployFp() {
   return 0;
 }
 
-static int32_t sdbWriteVersion(FileFd fd) { return 0; }
-
-static int32_t sdbReadVersion(FileFd fd) { return 0; }
-
 static int32_t sdbReadDataFile() {
   SSdbRaw *pRaw = malloc(SDB_MAX_SIZE);
   if (pRaw == NULL) {
@@ -71,6 +68,7 @@ static int32_t sdbReadDataFile() {
   snprintf(file, sizeof(file), "%ssdb.data", tsSdb.currDir);
   FileFd fd = taosOpenFileCreateWrite(file);
   if (fd <= 0) {
+    free(pRaw);
     terrno = TAOS_SYSTEM_ERROR(errno);
     mError("failed to open file:%s for read since %s", file, terrstr());
     return -1;
@@ -90,7 +88,27 @@ static int32_t sdbReadDataFile() {
     }
 
     if (ret < sizeof(SSdbRaw)) {
-      code = TSDB_CODE_SDB_APP_ERROR;
+      code = TSDB_CODE_FILE_CORRUPTED;
+      mError("failed to read file:%s since %s", file, tstrerror(code));
+      break;
+    }
+
+    ret = taosReadFile(fd, pRaw->pData, pRaw->dataLen + sizeof(int32_t));
+    if (ret < 0) {
+      code = TAOS_SYSTEM_ERROR(errno);
+      mError("failed to read file:%s since %s", file, tstrerror(code));
+      break;
+    }
+
+    if (ret < pRaw->dataLen + sizeof(int32_t)) {
+      code = TSDB_CODE_FILE_CORRUPTED;
+      mError("failed to read file:%s since %s", file, tstrerror(code));
+      break;
+    }
+
+    uint32_t cksum = *(int32_t *)(pRaw->pData + pRaw->dataLen);
+    if (!taosCheckChecksumWhole(pRaw, sizeof(SSdbRaw) + pRaw->dataLen + sizeof(int32_t)) != 0) {
+      code = TSDB_CODE_CHECKSUM_ERROR;
       mError("failed to read file:%s since %s", file, tstrerror(code));
       break;
     }
@@ -106,6 +124,7 @@ static int32_t sdbReadDataFile() {
 
 PARSE_SDB_DATA_ERROR:
   taosCloseFile(fd);
+  sdbFreeRaw(pRaw);
   terrno = code;
   return code;
 }
@@ -134,10 +153,19 @@ static int32_t sdbWriteDataFile() {
     SSdbRow *pRow = taosHashIterate(hash, NULL);
     while (pRow != NULL) {
       if (pRow->status != SDB_STATUS_READY) continue;
-      
+
       SSdbRaw *pRaw = (*encodeFp)(pRow->pObj);
       if (pRaw != NULL) {
-        taosWriteFile(fd, pRaw, sizeof(SSdbRaw) + pRaw->dataLen);
+        int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen;
+        if (taosWriteFile(fd, pRaw, writeLen) != writeLen) {
+          code = TAOS_SYSTEM_ERROR(terrno);
+          break;
+        }
+        int32_t cksum = taosCalcChecksum(0, pRaw, sizeof(SSdbRaw) + pRaw->dataLen);
+        if (taosWriteFile(fd, &cksum, sizeof(int32_t)) != sizeof(int32_t)) {
+          code = TAOS_SYSTEM_ERROR(terrno);
+          break;
+        }
       } else {
         taosHashCancelIterate(hash, pRow);
         code = TSDB_CODE_SDB_APP_ERROR;
@@ -149,10 +177,6 @@ static int32_t sdbWriteDataFile() {
     taosWUnLockLatch(pLock);
   }
 
-  if (code == 0) {
-    code = sdbWriteVersion(fd);
-  }
-
   taosCloseFile(fd);
 
   if (code == 0) {

source/util/src/terror.c

@@ -76,6 +76,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RANGE,                 "Out of range")
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PTR,                  "Invalid pointer")
 TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED,             "Memory corrupted")
 TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED,               "Data file corrupted")
+TAOS_DEFINE_ERROR(TSDB_CODE_CHECKSUM_ERROR,               "Checksum error")
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG,                  "Invalid config message")
 TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY,                "Ref out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL,                     "too many Ref Objs")