Merge branch 'main' of https://github.com/taosdata/TDengine into test/fix/comp

docs/en/14-reference/03-connector/07-python.mdx

@@ -373,7 +373,7 @@ conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (locat
 <TabItem value="websocket" label="WebSocket connection">
 
 ```python
-conn = taosws.connect(url="ws://localhost:6041")
+conn = taosws.connect("taosws://localhost:6041")
 # Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement.
 conn.execute("DROP DATABASE IF EXISTS test")
 conn.execute("CREATE DATABASE test")

docs/en/28-releases/01-tdengine.md

@@ -12,6 +12,11 @@ import Release from "/components/ReleaseV3";
 
 ## 3.1.0.0
 
+:::note IMPORTANT
+- Once you upgrade to TDengine 3.1.0.0, you cannot roll back to any previous version of TDengine. Upgrading to 3.1.0.0 will alter your data such that it cannot be read by previous versions.
+- You must remove all streams before upgrading to TDengine 3.1.0.0. If you upgrade a deployment that contains streams, the upgrade will fail and your deployment will become nonoperational.
+:::
+
 <Release type="tdengine" version="3.1.0.0" />
 
 ## 3.0.7.1

docs/zh/08-connector/30-python.mdx

@@ -375,7 +375,7 @@ conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (locat
 <TabItem value="websocket" label="WebSocket 连接">
 
 ```python
-conn = taosws.connect(url="ws://localhost:6041")
+conn = taosws.connect("taosws://localhost:6041")
 # Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement.
 conn.execute("DROP DATABASE IF EXISTS test")
 conn.execute("CREATE DATABASE test")

packaging/tools/install.sh

@@ -613,6 +613,11 @@ function install_examples() {
   fi
 }
 
+function install_web() {
+  if [ -d "${script_dir}/share" ]; then
+    ${csudo}cp -rf ${script_dir}/share/* ${install_main_dir}/share > /dev/null 2>&1 ||:
+  fi
+}
 
 function clean_service_on_sysvinit() {
   if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then
@@ -888,6 +893,7 @@ function updateProduct() {
   fi
 
   install_examples
+  install_web
   if [ -z $1 ]; then
     install_bin
     install_service
@@ -898,29 +904,29 @@ function updateProduct() {
     openresty_work=false
 
     echo
-    echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}"
+    echo -e "${GREEN_DARK}To configure ${productName2} ${NC}\t: edit ${cfg_install_dir}/${configFile2}"
     [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
-      echo -e "${GREEN_DARK}To configure ${clientName2}Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml"
+      echo -e "${GREEN_DARK}To configure ${clientName2}Adapter ${NC}\t: edit ${configDir}/${clientName2}adapter.toml"
     if ((${service_mod} == 0)); then
-      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}: ${csudo}systemctl start ${serverName2}${NC}"
+      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}\t: ${csudo}systemctl start ${serverName2}${NC}"
       [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
-        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}"
+        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}systemctl start ${clientName2}adapter ${NC}"
     elif ((${service_mod} == 1)); then
-      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}: ${csudo}service ${serverName2} start${NC}"
+      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}\t: ${csudo}service ${serverName2} start${NC}"
       [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
-        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}"
+        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}service ${clientName2}adapter start${NC}"
     else
-      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}: ./${serverName2}${NC}"
+      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}\t: ./${serverName2}${NC}"
       [ -f ${installDir}/bin/${clientName2}adapter ] && \
-        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${clientName2}adapter ${NC}"
+        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${clientName2}adapter ${NC}"
     fi
     
-    echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}: sudo systemctl enable ${clientName2}keeper ${NC}"
+    echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}\t: sudo systemctl enable ${clientName2}keeper ${NC}"
 
     if [ ${openresty_work} = 'true' ]; then
-      echo -e "${GREEN_DARK}To access ${productName2}    ${NC}: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}"
+      echo -e "${GREEN_DARK}To access ${productName2}    ${NC}\t: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}"
     else
-      echo -e "${GREEN_DARK}To access ${productName2}    ${NC}: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell${NC}"
+      echo -e "${GREEN_DARK}To access ${productName2}    ${NC}\t: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell${NC}"
     fi
 
     if ((${prompt_force} == 1)); then
@@ -968,7 +974,7 @@ function installProduct() {
       install_connector
   fi
   install_examples  
-
+  install_web
   if [ -z $1 ]; then # install service and client
     # For installing new
     install_bin
@@ -982,24 +988,24 @@ function installProduct() {
 
     # Ask if to start the service
     echo
-    echo -e "${GREEN_DARK}To configure ${productName2} ${NC}: edit ${cfg_install_dir}/${configFile2}"
+    echo -e "${GREEN_DARK}To configure ${productName2} ${NC}\t: edit ${cfg_install_dir}/${configFile2}"
     [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
-      echo -e "${GREEN_DARK}To configure ${clientName2}Adapter ${NC}: edit ${configDir}/${clientName2}adapter.toml"
+      echo -e "${GREEN_DARK}To configure ${clientName2}Adapter ${NC}\t: edit ${configDir}/${clientName2}adapter.toml"
     if ((${service_mod} == 0)); then
-      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}: ${csudo}systemctl start ${serverName2}${NC}"
+      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}\t: ${csudo}systemctl start ${serverName2}${NC}"
       [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
-        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${csudo}systemctl start ${clientName2}adapter ${NC}"
+        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}systemctl start ${clientName2}adapter ${NC}"
     elif ((${service_mod} == 1)); then
-      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}: ${csudo}service ${serverName2} start${NC}"
+      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}\t: ${csudo}service ${serverName2} start${NC}"
       [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \
-        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${csudo}service ${clientName2}adapter start${NC}"
+        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}service ${clientName2}adapter start${NC}"
     else
-      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}: ${serverName2}${NC}"
+      echo -e "${GREEN_DARK}To start ${productName2}     ${NC}\t: ${serverName2}${NC}"
       [ -f ${installDir}/bin/${clientName2}adapter ] && \
-        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}: ${clientName2}adapter ${NC}"
+        echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${clientName2}adapter ${NC}"
     fi
 
-    echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}: sudo systemctl enable ${clientName2}keeper ${NC}"
+    echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}\t: sudo systemctl enable ${clientName2}keeper ${NC}"
 
     if [ ! -z "$firstEp" ]; then
       tmpFqdn=${firstEp%%:*}
@@ -1010,14 +1016,14 @@ function installProduct() {
         tmpPort=""
       fi
       if [[ "$tmpPort" != "" ]]; then
-        echo -e "${GREEN_DARK}To access ${productName2}    ${NC}: ${clientName2} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
+        echo -e "${GREEN_DARK}To access ${productName2}    ${NC}\t: ${clientName2} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}"
       else
-        echo -e "${GREEN_DARK}To access ${productName2}    ${NC}: ${clientName2} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
+        echo -e "${GREEN_DARK}To access ${productName2}    ${NC}\t: ${clientName2} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}"
       fi
       echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}"
       echo
     elif [ ! -z "$serverFqdn" ]; then
-      echo -e "${GREEN_DARK}To access ${productName2}    ${NC}: ${clientName2} -h $serverFqdn${GREEN_DARK} to login into ${productName2} server${NC}"
+      echo -e "${GREEN_DARK}To access ${productName2}    ${NC}\t: ${clientName2} -h $serverFqdn${GREEN_DARK} to login into ${productName2} server${NC}"
       echo
     fi
 

packaging/tools/makepkg.sh

@@ -319,6 +319,11 @@ if [[ $dbName == "taos" ]]; then
     mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json
   fi
 
+  if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then    
+    mkdir -p ${install_dir}/share/        
+    cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||:    
+  fi
+
 fi
 
 # Copy driver

source/dnode/mnode/impl/src/mndIndex.c

@@ -515,7 +515,6 @@ int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, i
     if (pDb == NULL) return 0;
   }
   SSmaAndTagIter *pIter = pShow->pIter;
-  int             invalid = -1;
   while (numOfRows < rows) {
     pIter->pIdxIter = sdbFetch(pSdb, SDB_IDX, pIter->pIdxIter, (void **)&pIdx);
     if (pIter->pIdxIter == NULL) break;
@@ -552,7 +551,7 @@ int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, i
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
 
-    colDataSetVal(pColInfo, numOfRows, (const char *)&invalid, false);
+    colDataSetVal(pColInfo, numOfRows, NULL, true);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataSetVal(pColInfo, numOfRows, (const char *)&pIdx->createdTime, false);

source/dnode/mnode/impl/src/mndTopic.c

@@ -799,6 +799,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
         mndTransDrop(pTrans);
         return -1;
       }
+      sdbRelease(pSdb, pVgroup);
     }
   }
 

source/dnode/vnode/src/tsdb/tsdbCommit2.c

@@ -49,7 +49,7 @@ typedef struct {
   } ctx[1];
 
   // reader
-  SSttFileReader *sttReader;
+  TSttFileReaderArray sttReaderArray[1];
 
   // iter
   TTsdbIterArray dataIterArray[1];
@@ -226,7 +226,7 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) {
   int32_t code = 0;
   int32_t lino = 0;
 
-  ASSERT(committer->sttReader == NULL);
+  ASSERT(TARRAY2_SIZE(committer->sttReaderArray) == 0);
 
   if (committer->ctx->fset == NULL                        //
       || committer->sttTrigger > 1                        //
@@ -241,30 +241,31 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) {
 
   ASSERT(lvl->level == 0);
 
-  if (TARRAY2_SIZE(lvl->fobjArr) == 0) {
-    return 0;
-  }
+  STFileObj *fobj = NULL;
+  TARRAY2_FOREACH(lvl->fobjArr, fobj) {
+    SSttFileReader *sttReader;
 
-  ASSERT(TARRAY2_SIZE(lvl->fobjArr) == 1);
+    SSttFileReaderConfig config = {
+        .tsdb = committer->tsdb,
+        .szPage = committer->szPage,
+        .file = fobj->f[0],
+    };
 
-  STFileObj *fobj = TARRAY2_FIRST(lvl->fobjArr);
+    code = tsdbSttFileReaderOpen(fobj->fname, &config, &sttReader);
+    TSDB_CHECK_CODE(code, lino, _exit);
 
-  SSttFileReaderConfig config = {
-      .tsdb = committer->tsdb,
-      .szPage = committer->szPage,
-      .file = fobj->f[0],
-  };
-  code = tsdbSttFileReaderOpen(fobj->fname, &config, &committer->sttReader);
-  TSDB_CHECK_CODE(code, lino, _exit);
+    code = TARRAY2_APPEND(committer->sttReaderArray, sttReader);
+    TSDB_CHECK_CODE(code, lino, _exit);
 
-  STFileOp op = {
-      .optype = TSDB_FOP_REMOVE,
-      .fid = fobj->f->fid,
-      .of = fobj->f[0],
-  };
+    STFileOp op = {
+        .optype = TSDB_FOP_REMOVE,
+        .fid = fobj->f->fid,
+        .of = fobj->f[0],
+    };
 
-  code = TARRAY2_APPEND(committer->fopArray, op);
-  TSDB_CHECK_CODE(code, lino, _exit);
+    code = TARRAY2_APPEND(committer->fopArray, op);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
 
 _exit:
   if (code) {
@@ -273,7 +274,10 @@ _exit:
   return code;
 }
 
-static int32_t tsdbCommitCloseReader(SCommitter2 *committer) { return tsdbSttFileReaderClose(&committer->sttReader); }
+static int32_t tsdbCommitCloseReader(SCommitter2 *committer) {
+  TARRAY2_CLEAR(committer->sttReaderArray, tsdbSttFileReaderClose);
+  return 0;
+}
 
 static int32_t tsdbCommitOpenIter(SCommitter2 *committer) {
   int32_t code = 0;
@@ -310,10 +314,11 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) {
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // STT
-  if (committer->sttReader) {
+  SSttFileReader *sttReader;
+  TARRAY2_FOREACH(committer->sttReaderArray, sttReader) {
     // data iter
     config.type = TSDB_ITER_TYPE_STT;
-    config.sttReader = committer->sttReader;
+    config.sttReader = sttReader;
 
     code = tsdbIterOpen(&config, &iter);
     TSDB_CHECK_CODE(code, lino, _exit);
@@ -323,7 +328,7 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) {
 
     // tomb iter
     config.type = TSDB_ITER_TYPE_STT_TOMB;
-    config.sttReader = committer->sttReader;
+    config.sttReader = sttReader;
 
     code = tsdbIterOpen(&config, &iter);
     TSDB_CHECK_CODE(code, lino, _exit);

source/dnode/vnode/src/tsdb/tsdbFS2.c

@@ -780,19 +780,20 @@ static int32_t tsdbFSRunBgTask(void *arg) {
   return 0;
 }
 
-static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT type, int32_t (*run)(void *), void (*free)(void *),
-                                        void *arg, int64_t *taskid) {
+static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT   type, int32_t (*run)(void *),
+                                        void (*destroy)(void *), void *arg, int64_t *taskid) {
   if (fs->stop) {
+    if (destroy) {
+      destroy(arg);
+    }
     return 0;  // TODO: use a better error code
   }
 
-  // check if same task is on
-  // if (fs->bgTaskRunning && fs->bgTaskRunning->type == type) {
-  //   return 0;
-  // }
-
   for (STFSBgTask *task = fs->bgTaskQueue->next; task != fs->bgTaskQueue; task = task->next) {
     if (task->type == type) {
+      if (destroy) {
+        destroy(arg);
+      }
       return 0;
     }
   }
@@ -804,7 +805,7 @@ static int32_t tsdbFSScheduleBgTaskImpl(STFileSystem *fs, EFSBgTaskT type, int32
 
   task->type = type;
   task->run = run;
-  task->free = free;
+  task->free = destroy;
   task->arg = arg;
   task->scheduleTime = taosGetTimestampMs();
   task->taskid = ++fs->taskid;

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -342,18 +342,18 @@ static int32_t tsdbSnapCmprTombData(STsdbSnapReader* reader, uint8_t** data) {
   int32_t code = 0;
   int32_t lino = 0;
 
-  int64_t size = sizeof(SSnapDataHdr);
+  int64_t size = 0;
   for (int32_t i = 0; i < ARRAY_SIZE(reader->tombBlock->dataArr); i++) {
     size += TARRAY2_DATA_LEN(reader->tombBlock->dataArr + i);
   }
 
-  data[0] = taosMemoryMalloc(size);
+  data[0] = taosMemoryMalloc(size + sizeof(SSnapDataHdr));
   if (data[0] == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
-  SSnapDataHdr* hdr = (SSnapDataHdr*)data[0];
+  SSnapDataHdr* hdr = (SSnapDataHdr*)(data[0]);
   hdr->type = SNAP_DATA_DEL;
   hdr->size = size;
 
@@ -938,7 +938,7 @@ static int32_t tsdbSnapWriteDecmprTombBlock(SSnapDataHdr* hdr, STombBlock* tombB
   int32_t code = 0;
   int32_t lino = 0;
 
-  int64_t size = hdr->size - sizeof(*hdr);
+  int64_t size = hdr->size;
   ASSERT(size % TOMB_RECORD_ELEM_NUM == 0);
   size = size / TOMB_RECORD_ELEM_NUM;
   ASSERT(size % sizeof(int64_t) == 0);

source/libs/executor/src/tsort.c

@@ -995,7 +995,12 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   SArray*          aExtSrc = taosArrayInit(nSrc, POINTER_BYTES);
 
   size_t maxBufSize = pHandle->numOfPages * pHandle->pageSize;
-  createPageBuf(pHandle);
+
+  int32_t code = createPageBuf(pHandle);
+  if (code != TSDB_CODE_SUCCESS) {
+    taosArrayDestroy(aExtSrc);
+    return code;
+  }
 
   SSortSource* pSrc = taosArrayGetP(pHandle->pOrderedSource, 0);
   int32_t      szSort = 0;
@@ -1070,7 +1075,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   taosArrayDestroy(aExtSrc);
 
   pHandle->type = SORT_SINGLESOURCE_SORT;
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {

source/libs/stream/src/streamRecover.c

@@ -502,11 +502,13 @@ int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask) {
 
   taosWLockLatch(&pMeta->lock);
   streamMetaSaveTask(pMeta, pTask);
+  streamMetaCommit(pMeta);
   taosWUnLockLatch(&pMeta->lock);
 
   // history data scan in the stream time window finished, now let's enable the pause
   streamTaskEnablePause(pTask);
 
+  // for source tasks, let's continue execute.
   if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
     streamSchedExec(pTask);
   }

tests/parallel_test/cases.task

@@ -161,7 +161,7 @@
 ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py
 ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py
 ,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py
-#,,n,system-test,python3 ./test.py -f 0-others/splitVGroup.py -N 5
+,,n,system-test,python3 ./test.py -f 0-others/splitVGroup.py -N 5
 ,,n,system-test,python3 ./test.py -f 0-others/timeRangeWise.py -N 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3

tests/system-test/0-others/show_tag_index.py

@@ -64,7 +64,7 @@ class TDTestCase:
         tdSql.checkData(0, 0, 'idx1')
         tdSql.checkData(0, 1, 'db')
         tdSql.checkData(0, 2, 'stb')
-        tdSql.checkData(0, 3, -1)
+        tdSql.checkData(0, 3, None)
         tdSql.checkData(0, 5, 't1')
         tdSql.checkData(0, 6, 'tag_index')