Merge branch 'develop' into updatetestcase

src/common/src/tglobal.c

@@ -40,7 +40,7 @@ uint16_t tsSyncPort = 6040;
 int32_t  tsStatusInterval = 1;  // second
 int32_t  tsNumOfMnodes = 3;
 int32_t  tsEnableVnodeBak = 1;
-int32_t  tsEnableTelemetryReporting = 1;
+int32_t  tsEnableTelemetryReporting = 0;
 
 // common
 int32_t tsRpcTimer = 1000;

src/dnode/src/dnodeMgmt.c

@@ -489,8 +489,7 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
   SDMDnodeCfg *pCfg = &pStatusRsp->dnodeCfg;
   pCfg->numOfVnodes  = htonl(pCfg->numOfVnodes);
   pCfg->moduleStatus = htonl(pCfg->moduleStatus);
-  pCfg->dnodeId      = htonl(pCfg->dnodeId);
-  pCfg->clusterId    = htonl(pCfg->clusterId);
+  pCfg->dnodeId = htonl(pCfg->dnodeId);
 
   for (int32_t i = 0; i < pMnodes->nodeNum; ++i) {
     SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i];
@@ -696,12 +695,12 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
   //strcpy(pStatus->dnodeName, tsDnodeName);
   pStatus->version          = htonl(tsVersion);
   pStatus->dnodeId          = htonl(tsDnodeCfg.dnodeId);
-  pStatus->clusterId        = htonl(tsDnodeCfg.clusterId);
-  strcpy(pStatus->dnodeEp, tsLocalEp);
   pStatus->lastReboot       = htonl(tsRebootTime);
   pStatus->numOfCores       = htons((uint16_t) tsNumOfCores);
   pStatus->diskAvailable    = tsAvailDataDirGB;
   pStatus->alternativeRole  = (uint8_t) tsAlternativeRole;
+  tstrncpy(pStatus->clusterId, tsDnodeCfg.clusterId, TSDB_CLUSTER_ID_LEN);
+  tstrncpy(pStatus->dnodeEp, tsLocalEp, TSDB_EP_LEN);
 
   // fill cluster cfg parameters
   pStatus->clusterCfg.numOfMnodes        = htonl(tsNumOfMnodes);
@@ -768,11 +767,11 @@ static bool dnodeReadDnodeCfg() {
   tsDnodeCfg.dnodeId = dnodeId->valueint;
 
   cJSON* clusterId = cJSON_GetObjectItem(root, "clusterId");
-  if (!clusterId || clusterId->type != cJSON_Number) {
+  if (!clusterId || clusterId->type != cJSON_String) {
     dError("failed to read dnodeCfg.json, clusterId not found");
     goto PARSE_CFG_OVER;
   }
-  tsDnodeCfg.clusterId = clusterId->valueint;
+  tstrncpy(tsDnodeCfg.clusterId, clusterId->valuestring, TSDB_CLUSTER_ID_LEN);
 
   ret = true;
 
@@ -793,13 +792,13 @@ static void dnodeSaveDnodeCfg() {
   if (!fp) return;
 
   int32_t len = 0;
-  int32_t maxLen = 100;
+  int32_t maxLen = 200;
   char *  content = calloc(1, maxLen + 1);
 
   len += snprintf(content + len, maxLen - len, "{\n");
   len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", tsDnodeCfg.dnodeId);
-  len += snprintf(content + len, maxLen - len, "  \"clusterId\": %d\n", tsDnodeCfg.clusterId);
-  len += snprintf(content + len, maxLen - len, "}\n"); 
+  len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%s\"\n", tsDnodeCfg.clusterId);
+  len += snprintf(content + len, maxLen - len, "}\n");
 
   fwrite(content, 1, len, fp);
   fflush(fp);
@@ -811,9 +810,9 @@ static void dnodeSaveDnodeCfg() {
 
 void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg) {
   if (tsDnodeCfg.dnodeId == 0) {
-    dInfo("dnodeId is set to %d, clusterId is set to %d", pCfg->dnodeId, pCfg->clusterId);  
+    dInfo("dnodeId is set to %d, clusterId is set to %s", pCfg->dnodeId, pCfg->clusterId);  
     tsDnodeCfg.dnodeId = pCfg->dnodeId;
-    tsDnodeCfg.clusterId = pCfg->clusterId;
+    tstrncpy(tsDnodeCfg.clusterId, pCfg->clusterId, TSDB_CLUSTER_ID_LEN);
     dnodeSaveDnodeCfg();
   }
 }

src/dnode/src/dnodeTelemetry.c

@@ -174,7 +174,7 @@ static void addVersionInfo(SBufferWriter* bw) {
 }
 
 static void addRuntimeInfo(SBufferWriter* bw) {
-  addIntField(bw, "clusterId", mnodeGetClusterId());
+  addStringField(bw, "clusterId", mnodeGetClusterId());
   // addIntField(&bw, "numOfDnode", 1);
   // addIntField(&bw, "numOfVnode", 1);
   // addIntField(&bw, "numOfStable", 1);
@@ -235,7 +235,9 @@ static void* telemetryThread(void* param) {
     }
 
     int startAt = taosGetTimestampSec();
-    struct timespec timeout = {.tv_sec = timeToWait, .tv_nsec = 0};
+    struct timespec timeout = {.tv_sec = 0, .tv_nsec = 0};
+    clock_gettime(CLOCK_REALTIME, &timeout);
+    timeout.tv_sec += timeToWait;
     if (sem_timedwait(&tsExitSem, &timeout) == 0) {
       break;
     }
@@ -279,4 +281,4 @@ void dnodeCleanupTelemetry() {
     pthread_join(tsTelemetryThread, NULL);
     sem_destroy(&tsExitSem);
   }
-}
\ No newline at end of file
+}

src/inc/taosdef.h

@@ -254,6 +254,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_TIMEZONE_LEN         96
 #define TSDB_LABEL_LEN            8 
 
+#define TSDB_CLUSTER_ID_LEN       40
 #define TSDB_FQDN_LEN             128
 #define TSDB_EP_LEN               (TSDB_FQDN_LEN+6)
 #define TSDB_IPv4ADDR_LEN      	  16

src/inc/taosmsg.h

@@ -548,10 +548,10 @@ typedef struct {
 
 typedef struct {
   int32_t  dnodeId;
-  int32_t  clusterId;
   uint32_t moduleStatus;
   uint32_t numOfVnodes;
-  uint32_t reserved;
+  char     clusterId[TSDB_CLUSTER_ID_LEN];
+  char     reserved[16];
 } SDMDnodeCfg;
 
 typedef struct {
@@ -589,7 +589,7 @@ typedef struct {
   uint16_t    openVnodes;
   uint16_t    numOfCores;
   float       diskAvailable;  // GB
-  int32_t     clusterId;
+  char        clusterId[TSDB_CLUSTER_ID_LEN];
   uint8_t     alternativeRole;
   uint8_t     reserve2[15];
   SClusterCfg clusterCfg;

src/mnode/inc/mnodeCluster.h

@@ -20,16 +20,10 @@
 extern "C" {
 #endif
 
-struct SClusterObj;
-
 int32_t mnodeInitCluster();
 void    mnodeCleanupCluster();
-int32_t mnodeGetClusterId();
 void    mnodeUpdateClusterId();
-void *  mnodeGetCluster(int32_t clusterId);
-void *  mnodeGetNextCluster(void *pIter, struct SClusterObj **pCluster);
-void    mnodeIncClusterRef(struct SClusterObj *pCluster);
-void    mnodeDecClusterRef(struct SClusterObj *pCluster);
+const char* mnodeGetClusterId();
 
 #ifdef __cplusplus
 }

src/mnode/inc/mnodeDef.h

@@ -37,9 +37,9 @@ struct define notes:
 */
 
 typedef struct SClusterObj {
-  int32_t clusterId;
+  char    uid[TSDB_CLUSTER_ID_LEN];
   int64_t createdTime;
-  int8_t  reserved[36];
+  int8_t  reserved[12];
   int8_t  updateEnd[4];
   int32_t refCount;
 } SClusterObj;
@@ -90,7 +90,7 @@ typedef struct STableObj {
 
 typedef struct SSuperTableObj {
   STableObj  info; 
-  int8_t     reserved0[1]; // for fill struct STableObj to 4byte align
+  int8_t     reserved0[9]; // for fill struct STableObj to 4byte align
   int16_t    nextColId;
   int32_t    sversion;
   uint64_t   uid;
@@ -103,12 +103,11 @@ typedef struct SSuperTableObj {
   int32_t    numOfTables;
   SSchema *  schema;
   void *     vgHash;
-  int8_t     reserved2[6];
 } SSuperTableObj;
 
 typedef struct {
   STableObj  info;  
-  int8_t     reserved0[1]; // for fill struct STableObj to 4byte align
+  int8_t     reserved0[9]; // for fill struct STableObj to 4byte align
   int16_t    nextColId;    //used by normal table
   int32_t    sversion;     //used by normal table  
   uint64_t   uid;
@@ -144,7 +143,7 @@ typedef struct SVgObj {
   int8_t         status;
   int8_t         reserved0[4];
   SVnodeGid      vnodeGid[TSDB_MAX_REPLICA];
-  int8_t         reserved1[4];
+  int8_t         reserved1[12];
   int8_t         updateEnd[4];
   int32_t        refCount;
   int32_t        numOfTables;

src/mnode/src/mnodeCluster.c

@@ -26,7 +26,7 @@
 
 static void *  tsClusterSdb = NULL;
 static int32_t tsClusterUpdateSize;
-static int32_t tsClusterId;
+static char    tsClusterId[TSDB_CLUSTER_ID_LEN];
 static int32_t mnodeCreateCluster();
 
 static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
@@ -90,7 +90,7 @@ int32_t mnodeInitCluster() {
     .hashSessions = TSDB_DEFAULT_CLUSTER_HASH_SIZE,
     .maxRowSize   = tsClusterUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
-    .keyType      = SDB_KEY_INT,
+    .keyType      = SDB_KEY_STRING,
     .insertFp     = mnodeClusterActionInsert,
     .deleteFp     = mnodeClusterActionDelete,
     .updateFp     = mnodeClusterActionUpdate,
@@ -118,10 +118,6 @@ void mnodeCleanupCluster() {
   tsClusterSdb = NULL;
 }
 
-void *mnodeGetCluster(int32_t clusterId) {
-  return sdbGetRow(tsClusterSdb, &clusterId);
-}
-
 void *mnodeGetNextCluster(void *pIter, SClusterObj **pCluster) {
   return sdbFetchRow(tsClusterSdb, pIter, (void **)pCluster); 
 }
@@ -141,8 +137,14 @@ static int32_t mnodeCreateCluster() {
   SClusterObj *pCluster = malloc(sizeof(SClusterObj));
   memset(pCluster, 0, sizeof(SClusterObj));
   pCluster->createdTime = taosGetTimestampMs();
-  pCluster->clusterId = labs((pCluster->createdTime >> 32) & (pCluster->createdTime)) | (*(int32_t*)tsFirst);
-  
+  bool getuid = taosGetSystemUid(pCluster->uid);
+  if (!getuid) {
+    strcpy(pCluster->uid, "tdengine2.0");
+    mError("failed to get uid from system, set to default val %s", pCluster->uid);
+  } else {
+    mDebug("uid is %s", pCluster->uid);
+  }
+
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsClusterSdb,
@@ -152,7 +154,7 @@ static int32_t mnodeCreateCluster() {
   return sdbInsertRow(&oper);
 }
 
-int32_t mnodeGetClusterId() {
+const char* mnodeGetClusterId() {
   return tsClusterId;
 }
 
@@ -160,8 +162,8 @@ void mnodeUpdateClusterId() {
   SClusterObj *pCluster = NULL;
   void *pIter = mnodeGetNextCluster(NULL, &pCluster);
   if (pCluster != NULL) {
-    tsClusterId = pCluster->clusterId;
-    mInfo("cluster id is %d", tsClusterId);
+    tstrncpy(tsClusterId, pCluster->uid, TSDB_CLUSTER_ID_LEN);
+    mInfo("cluster id is set to %s", tsClusterId);
   }
 
   mnodeDecClusterRef(pCluster);
@@ -172,8 +174,8 @@ static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *
   int32_t cols = 0;
   SSchema *pSchema = pMeta->schema;
 
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  pShow->bytes[cols] = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "clusterId");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
@@ -212,7 +214,7 @@ static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows,
     cols = 0;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int32_t *) pWrite = pCluster->clusterId;
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pCluster->uid, TSDB_CLUSTER_ID_LEN);
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

src/mnode/src/mnodeDnode.c

@@ -353,7 +353,6 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
   pStatus->moduleStatus = htonl(pStatus->moduleStatus);
   pStatus->lastReboot   = htonl(pStatus->lastReboot);
   pStatus->numOfCores   = htons(pStatus->numOfCores);
-  pStatus->clusterId    = htonl(pStatus->clusterId);
   
   uint32_t version = htonl(pStatus->version);
   if (version != tsVersion) {
@@ -383,10 +382,10 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
   pDnode->moduleStatus     = pStatus->moduleStatus;
 
   if (pStatus->dnodeId == 0) {
-    mDebug("dnode:%d %s, first access, set clusterId %d", pDnode->dnodeId, pDnode->dnodeEp, mnodeGetClusterId());
+    mDebug("dnode:%d %s, first access, set clusterId %s", pDnode->dnodeId, pDnode->dnodeEp, mnodeGetClusterId());
   } else {
-    if (pStatus->clusterId != mnodeGetClusterId()) {
-      mError("dnode:%d, input clusterId %d not match with exist %d", pDnode->dnodeId, pStatus->clusterId,
+    if (strncmp(pStatus->clusterId, mnodeGetClusterId(), TSDB_CLUSTER_ID_LEN - 1) != 0) {
+      mError("dnode:%d, input clusterId %s not match with exist %s", pDnode->dnodeId, pStatus->clusterId,
              mnodeGetClusterId());
       return TSDB_CODE_MND_INVALID_CLUSTER_ID;
     } else {
@@ -405,7 +404,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) {
   pRsp->dnodeCfg.dnodeId = htonl(pDnode->dnodeId);
   pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt);
   pRsp->dnodeCfg.numOfVnodes = htonl(openVnodes);
-  pRsp->dnodeCfg.clusterId = htonl(mnodeGetClusterId());
+  tstrncpy(pRsp->dnodeCfg.clusterId, mnodeGetClusterId(), TSDB_CLUSTER_ID_LEN);
   SDMVgroupAccess *pAccess = (SDMVgroupAccess *)((char *)pRsp + sizeof(SDMStatusRsp));
 
   for (int32_t j = 0; j < openVnodes; ++j) {

src/mnode/src/mnodeSdb.c

@@ -988,7 +988,7 @@ int32_t sdbAllocWriteQueue() {
 }
 
 void sdbFreeWritequeue() {
-  taosCloseQset(tsSdbWriteQueue);
+  taosCloseQueue(tsSdbWriteQueue);
   taosFreeQall(tsSdbWriteQall);
   taosCloseQset(tsSdbWriteQset);
   tsSdbWriteQall = NULL;

src/mnode/src/mnodeVgroup.c

@@ -450,6 +450,7 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi
   }
 
   if (pDb->numOfVgroups < 1) {
+    pthread_mutex_unlock(&pDb->mutex);
     mDebug("app:%p:%p, db:%s, failed create new vgroup since:%s, numOfVgroups:%d maxVgroupsPerDb:%d ",
            pMsg->rpcMsg.ahandle, pMsg, pDb->name, tstrerror(code), pDb->numOfVgroups, maxVgroupsPerDb);
     return code;

src/os/inc/osSysinfo.h

@@ -31,6 +31,7 @@ bool taosGetSysMemory(float *memoryUsedMB);
 void taosPrintOsInfo();
 int  taosSystem(const char * cmd) ;
 void taosKillSystem();
+bool taosGetSystemUid(char *uid);
 
 // TAOS_OS_FUNC_SYSINFO_CORE
 void taosSetCoreDump();

src/os/src/detail/osSysinfo.c

@@ -639,4 +639,19 @@ void taosSetCoreDump() {
 
 }
 
+bool taosGetSystemUid(char *uid) {
+  int fd;
+  int len = 0;
+
+  fd = open("/proc/sys/kernel/random/uuid", 0);
+  if (fd < 0) {
+    return false;
+  } else {
+    len = read(fd, uid, TSDB_CLUSTER_ID_LEN);
+    close(fd);
+  }
+
+  return len > 0;
+}
+
 #endif
\ No newline at end of file

src/rpc/src/rpcMain.c

@@ -747,7 +747,8 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) {
   if (pConn) {
     if (pConn->linkUid != pHead->linkUid) {
       terrno = TSDB_CODE_RPC_MISMATCHED_LINK_ID;
-      tError("%s %p %p, linkUid:0x%x is not matched with received:0x%x", pRpc->label, pConn, (void*)pHead->ahandle, pConn->linkUid, pHead->linkUid);
+      tDebug("%s %p %p, linkUid:0x%x is not matched with received:0x%x", pRpc->label, pConn, (void *)pHead->ahandle,
+             pConn->linkUid, pHead->linkUid);
       pConn = NULL;
     }
   }

src/util/src/tqueue.c

@@ -241,7 +241,6 @@ void taosCloseQset(taos_qset param) {
   if (param == NULL) return;
   STaosQset *qset = (STaosQset *)param;
 
-#if 0
   // remove all the queues from qset
   pthread_mutex_lock(&qset->mutex);
   while (qset->head) {
@@ -252,7 +251,6 @@ void taosCloseQset(taos_qset param) {
     queue->next = NULL;
   }
   pthread_mutex_unlock(&qset->mutex);
-#endif  
 
   pthread_mutex_destroy(&qset->mutex);
   tsem_destroy(&qset->sem);

tests/pytest/util/dnodes.py

@@ -191,6 +191,7 @@ class TDDnode:
         self.cfg("charset", "UTF-8")
         self.cfg("asyncLog", "0")
         self.cfg("anyIp", "0")
+        self.cfg("tsEnableTelemetryReporting", "0")
         self.cfg("dDebugFlag", "135")
         self.cfg("mDebugFlag", "135")
         self.cfg("sdbDebugFlag", "135")

tests/script/sh/deploy.sh

@@ -147,5 +147,6 @@ echo "asyncLog               0"                  >> $TAOS_CFG
 echo "numOfMnodes            1"                  >> $TAOS_CFG
 echo "locale                 en_US.UTF-8"        >> $TAOS_CFG
 echo "fsync                  0"                  >> $TAOS_CFG
+echo "tsEnableTelemetryReporting 0"              >> $TAOS_CFG
 echo " "                                         >> $TAOS_CFG