Merge branch '3.0' of https://github.com/taosdata/TDengine into enh/tsdb_optimize

cmake/taosadapter_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taosadapter
 ExternalProject_Add(taosadapter
         GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
-        GIT_TAG 565ca21
+        GIT_TAG 283b50d
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

packaging/docker/dockerbuild.sh

@@ -123,6 +123,16 @@ else
     echo "Unknown cpuType: ${cpuType}"
     exit 1
 fi
+# check the tdengine cloud base image existed or not
+if [ "$cloudBuild" == "y" ]; then
+  CloudBase=$(docker images | grep tdengine/tdengine-cloud-base ||:)
+  if [[ "$CloudBase" == "" ]]; then
+    echo "Rebuild tdengine cloud base image..."
+    docker build --rm -f "${communityDir}/packaging/docker/DockerfileCloud.base" -t tdengine/tdengine-cloud-base "." --build-arg cpuType=${cpuTypeAlias}
+  else
+    echo "Already found tdengine cloud base image"
+  fi
+fi
 
 docker build --rm -f "${Dockerfile}"  --network=host -t tdengine/tdengine-${dockername}:${version} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} --build-arg cpuType=${cpuTypeAlias}
 if [ "$cloudBuild" != "y" ]; then

packaging/tools/makeclient.sh

@@ -175,7 +175,7 @@ if [ "$pagMode" == "lite" ]; then
 fi
 chmod a+x ${install_dir}/install_client.sh
 
-if [[ $productName == "TDengine" ]]; then
+if [[ $productName == "TDengine" ]] && [ "$verMode" != "cloud" ]; then
   # Copy example code
   mkdir -p ${install_dir}/examples
   examples_dir="${top_dir}/examples"
@@ -191,7 +191,7 @@ if [[ $productName == "TDengine" ]]; then
     mkdir -p ${install_dir}/examples/taosbenchmark-json && cp ${examples_dir}/../tools/taos-tools/example/* ${install_dir}/examples/taosbenchmark-json
   fi
 
-  if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then
+  if [ "$verMode" == "cluster" ]; then
       # Copy connector
       connector_dir="${code_dir}/connector"
       mkdir -p ${install_dir}/connector

packaging/tools/makepkg.sh

@@ -69,25 +69,30 @@ if [ "$pagMode" == "lite" ]; then
   bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark "
   taostools_bin_files=""
 else
-
-  wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \
+  if [ "$verMode" == "cloud" ]; then
+    taostools_bin_files=" ${build_dir}/bin/taosdump \
+      ${build_dir}/bin/taosBenchmark"
+  else
+    wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${build_dir}/bin/TDinsight.sh \
       && echo "TDinsight.sh downloaded!" \
       || echo "failed to download TDinsight.sh"
-  # download TDinsight caches
-  orig_pwd=$(pwd)
-  tdinsight_caches=""
-  cd ${build_dir}/bin/ && \
-    chmod +x TDinsight.sh
-  ./TDinsight.sh --download-only ||:
-#  tdinsight_caches=$(./TDinsight.sh --download-only | xargs -I printf "${build_dir}/bin/{} ")
-  cd $orig_pwd
-  echo "TDinsight caches: $tdinsight_caches"
-
-  taostools_bin_files=" ${build_dir}/bin/taosdump \
+    # download TDinsight caches
+    orig_pwd=$(pwd)
+    tdinsight_caches=""
+    cd ${build_dir}/bin/ && \
+      chmod +x TDinsight.sh
+    ./TDinsight.sh --download-only ||:
+    #  tdinsight_caches=$(./TDinsight.sh --download-only | xargs -I printf "${build_dir}/bin/{} ")
+    cd $orig_pwd
+    echo "TDinsight caches: $tdinsight_caches"
+
+    taostools_bin_files=" ${build_dir}/bin/taosdump \
       ${build_dir}/bin/taosBenchmark \
       ${build_dir}/bin/TDinsight.sh \
       ${build_dir}/bin/tdengine-datasource.zip \
       ${build_dir}/bin/tdengine-datasource.zip.md5sum"
+  fi
+
   [ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx"
   explorer_bin_files=$(find ${build_dir}/bin/ -name '*-explorer')
 
@@ -334,7 +339,7 @@ mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo
 [ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || :
 
 # Copy connector
-if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then
+if [ "$verMode" == "cluster" ]; then
     connector_dir="${code_dir}/connector"
     mkdir -p ${install_dir}/connector
     if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -211,10 +211,18 @@ static void tsdbCloseRocksCache(STsdb *pTsdb) {
   taosMemoryFree(pTsdb->rCache.pTSchema);
 }
 
-static void rocksMayWrite(STsdb *pTsdb, bool force, bool read) {
-  rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch;
+static void rocksMayWrite(STsdb *pTsdb, bool force, bool read, bool lock) {
+  rocksdb_writebatch_t *wb = NULL;
   if (read) {
+    if (lock) {
+      taosThreadMutexLock(&pTsdb->lruMutex);
+    }
     wb = pTsdb->rCache.rwritebatch;
+  } else {
+    if (lock) {
+      taosThreadMutexLock(&pTsdb->rCache.rMutex);
+    }
+    wb = pTsdb->rCache.writebatch;
   }
   int count = rocksdb_writebatch_count(wb);
   if ((force && count > 0) || count >= 1024) {
@@ -228,14 +236,19 @@ static void rocksMayWrite(STsdb *pTsdb, bool force, bool read) {
 
     rocksdb_writebatch_clear(wb);
   }
+  if (read) {
+    if (lock) taosThreadMutexUnlock(&pTsdb->lruMutex);
+  } else {
+    if (lock) taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
+  }
 }
 
 int32_t tsdbCacheCommit(STsdb *pTsdb) {
   int32_t code = 0;
   char   *err = NULL;
 
-  rocksMayWrite(pTsdb, true, false);
-  rocksMayWrite(pTsdb, true, true);
+  rocksMayWrite(pTsdb, true, false, true);
+  rocksMayWrite(pTsdb, true, true, true);
   rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err);
   if (NULL != err) {
     tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err);
@@ -461,7 +474,7 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
   taosMemoryFree(values_list);
   taosMemoryFree(values_list_sizes);
 
-  rocksMayWrite(pTsdb, false, false);
+  rocksMayWrite(pTsdb, false, false, false);
   taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
 
 _exit:
@@ -560,7 +573,7 @@ int32_t tsdbCacheGetSlow(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheR
       }
 
       if (wb) {
-        rocksMayWrite(pTsdb, false, true);
+        rocksMayWrite(pTsdb, false, true, false);
       }
 
       taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
@@ -628,7 +641,7 @@ static SLastCol *tsdbCacheLoadCol(STsdb *pTsdb, SCacheRowsReader *pr, int16_t sl
     }
 
     if (wb) {
-      rocksMayWrite(pTsdb, false, true);
+      rocksMayWrite(pTsdb, false, true, false);
     }
 
     taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
@@ -714,7 +727,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
   }
 
   if (wb) {
-    rocksMayWrite(pTsdb, false, true);
+    rocksMayWrite(pTsdb, false, true, false);
   }
 
   taosArrayDestroy(pTmpColArray);
@@ -944,7 +957,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
   size_t *values_list_sizes = taosMemoryCalloc(num_keys * 2, sizeof(size_t));
   char  **errs = taosMemoryCalloc(num_keys * 2, sizeof(char *));
   taosThreadMutexLock(&pTsdb->rCache.rMutex);
-  rocksMayWrite(pTsdb, true, false);
+  rocksMayWrite(pTsdb, true, false, false);
   rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys * 2, (const char *const *)keys_list,
                     keys_list_sizes, values_list, values_list_sizes, errs);
   for (int i = 0; i < num_keys; ++i) {
@@ -983,7 +996,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
   taosMemoryFree(values_list);
   taosMemoryFree(values_list_sizes);
 
-  rocksMayWrite(pTsdb, true, false);
+  rocksMayWrite(pTsdb, true, false, false);
   taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
 
 _exit:

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -63,9 +63,6 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
       SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i);
       SColVal*  pVal = &pColVal->colVal;
 
-      if (COL_VAL_IS_NONE(&pColVal->colVal)) {
-        continue;
-      }
       allNullRow = false;
       if (IS_VAR_DATA_TYPE(pColVal->colVal.type)) {
         if (!COL_VAL_IS_VALUE(&pColVal->colVal)) {

source/libs/catalog/CMakeLists.txt

@@ -11,6 +11,6 @@ target_link_libraries(
    PRIVATE os util transport qcom nodes
 )
 
-if(${BUILD_TEST})
-    ADD_SUBDIRECTORY(test)
-endif(${BUILD_TEST})
+# if(${BUILD_TEST})
+#     ADD_SUBDIRECTORY(test)
+# endif(${BUILD_TEST})

source/libs/planner/src/planOptimizer.c

@@ -2233,7 +2233,7 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) {
       SNode* pPar = nodesListGetNode(pAggFunc->pParameterList, 0);
       if (QUERY_NODE_COLUMN == nodeType(pPar)) {
         SColumnNode* pCol = (SColumnNode*)pPar;
-        if (pCol->colType == COLUMN_TYPE_TAG) {
+        if (pCol->colType != COLUMN_TYPE_COLUMN) {
           return false;      
         }
       }

tests/parallel_test/cases.task

@@ -513,7 +513,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py
-,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
+#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDnodeRestart.py