Merge pull request #4554 from taosdata/feature/wal

Feature/wal

cmake/define.inc

@@ -45,7 +45,7 @@ IF (TD_LINUX_64)
   ADD_DEFINITIONS(-D_M_X64)
   ADD_DEFINITIONS(-D_TD_LINUX_64)
   MESSAGE(STATUS "linux64 is defined")
-  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
   ADD_DEFINITIONS(-DUSE_LIBICONV)
 ENDIF ()
 
@@ -53,7 +53,7 @@ IF (TD_LINUX_32)
   ADD_DEFINITIONS(-D_TD_LINUX_32)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   MESSAGE(STATUS "linux32 is defined")
-  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -munaligned-access -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
 ENDIF ()
 
 IF (TD_ARM_64)
@@ -62,7 +62,7 @@ IF (TD_ARM_64)
   ADD_DEFINITIONS(-D_TD_ARM_)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   MESSAGE(STATUS "arm64 is defined")
-  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
 ENDIF ()
 
 IF (TD_ARM_32)
@@ -70,21 +70,21 @@ IF (TD_ARM_32)
   ADD_DEFINITIONS(-D_TD_ARM_)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   MESSAGE(STATUS "arm32 is defined")
-  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ")
+  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ")
 ENDIF ()
 
 IF (TD_MIPS_64)
   ADD_DEFINITIONS(-D_TD_MIPS_64_)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   MESSAGE(STATUS "mips64 is defined")
-  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
 ENDIF ()
 
 IF (TD_MIPS_32)
   ADD_DEFINITIONS(-D_TD_MIPS_32_)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   MESSAGE(STATUS "mips32 is defined")
-  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
 ENDIF ()
 
 IF (TD_APLHINE)
@@ -105,8 +105,8 @@ IF (TD_LINUX)
     MESSAGE(STATUS "set ningsi macro to true")
   ENDIF ()
   
-  SET(DEBUG_FLAGS "-O0 -DDEBUG")
-  SET(RELEASE_FLAGS "-O0 -Wno-unused-variable -Wunused-but-set-variable")
+  SET(DEBUG_FLAGS "-O0 -g3 -DDEBUG")
+  SET(RELEASE_FLAGS "-O3 -Wno-error")
   
   IF (${COVER} MATCHES "true")
     MESSAGE(STATUS "Test coverage mode, add extra flags")
@@ -125,9 +125,9 @@ IF (TD_DARWIN_64)
   ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   MESSAGE(STATUS "darwin64 is defined")
-  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-  SET(DEBUG_FLAGS "-O0 -DDEBUG")
-  SET(RELEASE_FLAGS "-O0")
+  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(DEBUG_FLAGS "-O0 -g3 -DDEBUG")
+  SET(RELEASE_FLAGS "-Og")
 ENDIF ()
 
 IF (TD_WINDOWS)
@@ -140,7 +140,7 @@ IF (TD_WINDOWS)
   IF (NOT TD_GODLL)
     SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd2220 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
     SET(DEBUG_FLAGS "/Zi /W3 /GL")
-    SET(RELEASE_FLAGS "/W0 /GL")
+    SET(RELEASE_FLAGS "/W0 /O3 /GL")
   ENDIF ()
 
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)

cmake/env.inc

@@ -41,8 +41,10 @@ SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${COMMON_FLAGS} ${RELEASE_FL
 # SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${COMMON_CXX_FLAGS} ${RELEASE_FLAGS}")
 
 IF (${CMAKE_BUILD_TYPE} MATCHES "Debug")
+  SET(CMAKE_BUILD_TYPE "Debug")
   MESSAGE(STATUS "Build Debug Version")
 ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release")
+  SET(CMAKE_BUILD_TYPE "Release")
   MESSAGE(STATUS "Build Release Version")
 ELSE ()
   IF (TD_WINDOWS)

src/client/src/tscLocal.c

@@ -571,7 +571,7 @@ static int32_t tscRebuildDDLForSubTable(SSqlObj *pSql, const char *tableName, ch
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
-  char fullName[TSDB_TABLE_FNAME_LEN] = {0};
+  char fullName[TSDB_TABLE_FNAME_LEN * 2] = {0};
   extractDBName(pTableMetaInfo->name, fullName);
   extractTableName(pMeta->sTableId, param->sTableName);
   snprintf(fullName + strlen(fullName), TSDB_TABLE_FNAME_LEN - strlen(fullName),  ".%s", param->sTableName);

src/common/src/tvariant.c

@@ -705,7 +705,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
           *((int32_t *)payload) = TSDB_DATA_FLOAT_NULL;
           return 0;
         } else {
-          double  value;
+          double  value = -1;
           int32_t ret;
           ret = convertToDouble(pVariant->pz, pVariant->nLen, &value);
           if ((errno == ERANGE && (float)value == -1) || (ret != 0)) {

src/cq/src/cqMain.c

@@ -97,7 +97,7 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
 
   pthread_mutex_init(&pContext->mutex, NULL);
 
-  cInfo("vgId:%d, CQ is opened", pContext->vgId);
+  cDebug("vgId:%d, CQ is opened", pContext->vgId);
 
   return pContext;
 }
@@ -131,7 +131,7 @@ void cqClose(void *handle) {
   taosTmrCleanUp(pContext->tmrCtrl);
   pContext->tmrCtrl = NULL;
 
-  cInfo("vgId:%d, CQ is closed", pContext->vgId);
+  cDebug("vgId:%d, CQ is closed", pContext->vgId);
   free(pContext);
 }
 
@@ -142,7 +142,7 @@ void cqStart(void *handle) {
   SCqContext *pContext = handle;
   if (pContext->dbConn || pContext->master) return;
 
-  cInfo("vgId:%d, start all CQs", pContext->vgId);
+  cDebug("vgId:%d, start all CQs", pContext->vgId);
   pthread_mutex_lock(&pContext->mutex);
 
   pContext->master = 1;
@@ -298,7 +298,7 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) {
     if (pObj->pStream) {
       tscSetStreamDestTable(pObj->pStream, pObj->dstTable);
       pContext->num++;
-      cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
+      cDebug("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr);
     } else {
       cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr);
     }

src/dnode/src/dnodeMain.c

@@ -147,6 +147,7 @@ int32_t dnodeInitSystem() {
 
   dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING);
 
+  dnodeReportStep("TDengine", "initialized successfully", 1);
   dInfo("TDengine is initialized successfully");
 
   return 0;

src/dnode/src/dnodeVnodes.c

@@ -127,7 +127,7 @@ int32_t dnodeInitVnodes() {
     pThread->vnodeList[pThread->vnodeNum++] = vnodeList[v];
   }
 
-  dDebug("start %d threads to open %d vnodes", threadNum, numOfVnodes);
+  dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes);
 
   for (int32_t t = 0; t < threadNum; ++t) {
     SOpenVnodeThread *pThread = &threads[t];

src/mnode/src/mnodeDnode.c

@@ -104,7 +104,7 @@ static int32_t mnodeDnodeActionInsert(SSdbRow *pRow) {
   dnodeUpdateEp(pDnode->dnodeId, pDnode->dnodeEp, pDnode->dnodeFqdn, &pDnode->dnodePort);
   mnodeUpdateDnodeEps();
 
-  mInfo("dnode:%d, fqdn:%s ep:%s port:%d, do insert action", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort);
+  mInfo("dnode:%d, fqdn:%s ep:%s port:%d is created", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp, pDnode->dnodePort);
   return TSDB_CODE_SUCCESS;
 }
 

src/mnode/src/mnodeMnode.c

@@ -72,7 +72,7 @@ static int32_t mnodeMnodeActionInsert(SSdbRow *pRow) {
   pDnode->isMgmt = true;
   mnodeDecDnodeRef(pDnode);
 
-  mInfo("mnode:%d, fqdn:%s ep:%s port:%u, do insert action", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp,
+  mInfo("mnode:%d, fqdn:%s ep:%s port:%u is created", pMnode->mnodeId, pDnode->dnodeFqdn, pDnode->dnodeEp,
         pDnode->dnodePort);
   return TSDB_CODE_SUCCESS;
 }
@@ -202,13 +202,13 @@ void mnodeCancelGetNextMnode(void *pIter) {
 void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) {
   bool    set = false;
   SMInfos mInfos = {0};
-  mInfo("vgId:1, update mnodes epSet, numOfMnodes:%d pMinfos:%p", mnodeGetMnodesNum(), pMinfos);
 
   if (pMinfos != NULL) {
+    mInfo("vgId:1, update mnodes epSet, numOfMinfos:%d", pMinfos->mnodeNum);
     set = true;
     mInfos = *pMinfos;
-  }
-  else {
+  } else {
+    mInfo("vgId:1, update mnodes epSet, numOfMnodes:%d", mnodeGetMnodesNum());
     int32_t index = 0;
     void *  pIter = NULL;
     while (1) {

src/mnode/src/mnodeSdb.c

@@ -183,18 +183,23 @@ static int32_t sdbInitWal() {
     return -1;
   }
 
-  sdbInfo("vgId:1, open wal for restore");
+  sdbInfo("vgId:1, open sdb wal for restore");
   int32_t code = walRestore(tsSdbMgmt.wal, NULL, sdbProcessWrite);
   if (code != TSDB_CODE_SUCCESS) {
     sdbError("vgId:1, failed to open wal for restore since %s", tstrerror(code));
     return -1;
   }
+
+  sdbInfo("vgId:1, sdb wal load success");
   return 0;
 }
 
 static void sdbRestoreTables() {
   int32_t totalRows = 0;
   int32_t numOfTables = 0;
+
+  sdbInfo("vgId:1, sdb start to check for integrity");
+
   for (int32_t tableId = 0; tableId < SDB_TABLE_MAX; ++tableId) {
     SSdbTable *pTable = sdbGetTableFromId(tableId);
     if (pTable == NULL) continue;
@@ -204,7 +209,7 @@ static void sdbRestoreTables() {
 
     totalRows += pTable->numOfRows;
     numOfTables++;
-    sdbDebug("vgId:1, sdb:%s is restored, rows:%" PRId64, pTable->name, pTable->numOfRows);
+    sdbInfo("vgId:1, sdb:%s is checked, rows:%" PRId64, pTable->name, pTable->numOfRows);
   }
 
   sdbInfo("vgId:1, sdb is restored, mver:%" PRIu64 " rows:%d tables:%d", tsSdbMgmt.version, totalRows, numOfTables);

src/mnode/src/mnodeTable.c

@@ -49,12 +49,14 @@
 #define CREATE_CTABLE_RETRY_TIMES 10
 #define CREATE_CTABLE_RETRY_SEC   14
 
-int64_t        tsCTableRid = -1;
-static void *  tsChildTableSdb;
-int64_t        tsSTableRid = -1;
-static void *  tsSuperTableSdb;
-static int32_t tsChildTableUpdateSize;
-static int32_t tsSuperTableUpdateSize;
+int64_t          tsCTableRid = -1;
+static void *    tsChildTableSdb;
+int64_t          tsSTableRid = -1;
+static void *    tsSuperTableSdb;
+static SHashObj *tsSTableUidHash;
+static int32_t   tsChildTableUpdateSize;
+static int32_t   tsSuperTableUpdateSize;
+
 static void *  mnodeGetChildTable(char *tableId);
 static void *  mnodeGetSuperTable(char *tableId);
 static void *  mnodeGetSuperTableByUid(uint64_t uid);
@@ -289,6 +291,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) {
 }
 
 static int32_t mnodeChildTableActionRestored() {
+#if 0
   void *pIter = NULL;
   SCTableObj *pTable = NULL;
 
@@ -345,6 +348,7 @@ static int32_t mnodeChildTableActionRestored() {
   }
 
   mnodeCancelGetNextChildTable(pIter);
+#endif
   return 0;
 }
 
@@ -447,6 +451,7 @@ static int32_t mnodeSuperTableActionInsert(SSdbRow *pRow) {
   }
   mnodeDecDbRef(pDb);
 
+  taosHashPut(tsSTableUidHash, &pStable->uid, sizeof(int64_t), &pStable, sizeof(int64_t));
   return TSDB_CODE_SUCCESS;
 }
 
@@ -459,6 +464,7 @@ static int32_t mnodeSuperTableActionDelete(SSdbRow *pRow) {
   }
   mnodeDecDbRef(pDb);
 
+  taosHashRemove(tsSTableUidHash, &pStable->uid, sizeof(int64_t));
   return TSDB_CODE_SUCCESS;
 }
 
@@ -570,6 +576,7 @@ static int32_t mnodeInitSuperTables() {
     .fpRestored   = mnodeSuperTableActionRestored
   };
 
+  tsSTableUidHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
   tsSTableRid = sdbOpenTable(&desc);
   tsSuperTableSdb = sdbGetTableByRid(tsSTableRid);
   if (tsSuperTableSdb == NULL) {
@@ -584,6 +591,9 @@ static int32_t mnodeInitSuperTables() {
 static void mnodeCleanupSuperTables() {
   sdbCloseTable(tsSTableRid);
   tsSuperTableSdb = NULL;
+
+  taosHashCleanup(tsSTableUidHash);
+  tsSTableUidHash = NULL;
 }
 
 int32_t mnodeInitTables() {
@@ -633,20 +643,12 @@ static void *mnodeGetSuperTable(char *tableId) {
 }
 
 static void *mnodeGetSuperTableByUid(uint64_t uid) {
-  SSTableObj *pStable = NULL;
-  void *pIter = NULL;
+  SSTableObj **ppStable = taosHashGet(tsSTableUidHash, &uid, sizeof(int64_t));
+  if (ppStable == NULL || *ppStable == NULL) return NULL;
 
-  while (1) {
-    pIter = mnodeGetNextSuperTable(pIter, &pStable);
-    if (pStable == NULL) break;
-    if (pStable->uid == uid) {
-      mnodeCancelGetNextSuperTable(pIter);
-      return pStable;
-    }
-    mnodeDecTableRef(pStable);
-  }
-
-  return NULL;
+  SSTableObj *pStable = *ppStable;
+  mnodeIncTableRef(pStable);
+  return pStable;
 }
 
 void *mnodeGetTable(char *tableId) {