Merge branch '3.0' of https://github.com/taosdata/TDengine into refact/new_data_format

include/libs/tfs/tfs.h

@@ -69,6 +69,14 @@ void tfsUpdateSize(STfs *pTfs);
  */
 SDiskSize tfsGetSize(STfs *pTfs);
 
+/**
+ * @brief Get level of multi-tier storage.
+ *
+ * @param pTfs
+ * @return int32_t
+ */
+int32_t tfsGetLevel(STfs *pTfs);
+
 /**
  * @brief Allocate an existing available tier level from fs.
  *

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -58,7 +58,7 @@ static int32_t tsdbGnrtCurrent(STsdb *pTsdb, STsdbFS *pFS, char *fname) {
   taosCalcChecksumAppend(0, pData, size);
 
   // create and write
-  pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE);
+  pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
   if (pFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;

source/dnode/vnode/src/tsdb/tsdbRetention.c

@@ -16,9 +16,19 @@
 #include "tsdb.h"
 
 static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) {
+  STsdbKeepCfg *keepCfg = &pTsdb->keepCfg;
+
+  if ((keepCfg->keep0 == keepCfg->keep1) && (keepCfg->keep1 == keepCfg->keep2)) {
+    return false;
+  }
+
+  if (tfsGetLevel(pTsdb->pVnode->pTfs) <= 1) {
+    return false;
+  }
+
   for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) {
     SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet);
-    int32_t    expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now);
+    int32_t    expLevel = tsdbFidLevel(pSet->fid, keepCfg, now);
     SDiskID    did;
 
     if (expLevel == pSet->diskId.level) continue;
@@ -53,7 +63,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
   if (code) goto _err;
 
   for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); iSet++) {
-    SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet);
+    SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet);
     int32_t    expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now);
     SDiskID    did;
 
@@ -65,6 +75,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
       taosArrayRemove(fs.aDFileSet, iSet);
       iSet--;
     } else {
+      if (expLevel == 0) continue;
       if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) {
         code = terrno;
         goto _exit;

source/libs/function/src/builtinsimpl.c

@@ -4780,7 +4780,7 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) {
       colDataAppendNULL(pOutput, i);
       // handle selectivity
       if (pCtx->subsidiaries.num > 0) {
-        appendSelectivityValue(pCtx, i, i);
+        appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1);
       }
       continue;
     }
@@ -4797,11 +4797,11 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) {
     } else {
       pInfo->durationStart = 0;
     }
-    colDataAppend(pOutput, i, (char*)&output, false);
+    colDataAppend(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false);
 
     // handle selectivity
     if (pCtx->subsidiaries.num > 0) {
-      appendSelectivityValue(pCtx, i, i);
+      appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1);
     }
   }
 

source/libs/tfs/src/tfs.c

@@ -113,6 +113,8 @@ SDiskSize tfsGetSize(STfs *pTfs) {
   return size;
 }
 
+int32_t tfsGetLevel(STfs *pTfs) { return pTfs->nlevel; }
+
 int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) {
   pDiskId->level = expLevel;
   pDiskId->id = -1;

source/os/src/osSemaphore.c

@@ -400,6 +400,9 @@ int tsem_init(tsem_t *psem, int flags, unsigned int count) {
 }
 
 int tsem_destroy(tsem_t *psem) {
+  if (psem == NULL || *psem == NULL) return -1;
+  dispatch_release(*psem);
+  *psem = NULL;
   return 0;
 }
 
@@ -421,13 +424,7 @@ int tsem_timewait(tsem_t *psem, int64_t nanosecs) {
   return 0;
 }
 
-bool taosCheckPthreadValid(TdThread thread) {
-  int32_t ret = taosThreadKill(thread, 0);
-  if (ret == ESRCH) return false;
-  if (ret == EINVAL) return false;
-  // alive
-  return true;
-}
+bool taosCheckPthreadValid(TdThread thread) { return thread != 0; }
 
 int64_t taosGetSelfPthreadId() {
   TdThread thread = taosThreadSelf();

source/os/src/osSysinfo.c

@@ -344,30 +344,27 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) {
   *numOfCores = si.dwNumberOfProcessors;
   return 0;
 #elif defined(_TD_DARWIN_64)
-  char   *line = NULL;
-  size_t  size = 0;
+  char    buf[16];
   int32_t done = 0;
   int32_t code = -1;
 
-  TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM);
-  if (pFile == NULL) return false;
-
-  while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) {
-    line[size - 1] = '\0';
-    if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) {
-      const char *v = strchr(line, ':') + 2;
-      tstrncpy(cpuModel, v, maxLen);
-      code = 0;
-      done |= 1;
-    } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) {
-      const char *v = strchr(line, ':') + 2;
-      *numOfCores = atof(v);
-      done |= 2;
-    }
+  TdCmdPtr pCmd = taosOpenCmd("sysctl -n machdep.cpu.brand_string");
+  if (pCmd == NULL) return code;
+  if (taosGetsCmd(pCmd, maxLen, cpuModel) > 0) {
+    code = 0;
+    done |= 1;
   }
+  taosCloseCmd(&pCmd);
 
-  if (line != NULL) taosMemoryFree(line);
-  taosCloseFile(&pFile);
+  pCmd = taosOpenCmd("sysctl -n machdep.cpu.core_count");
+  if (pCmd == NULL) return code;
+  memset(buf, 0, sizeof(buf));
+  if (taosGetsCmd(pCmd, maxLen, cpuModel) > 0) {
+    code = 0;
+    done |= 2;
+    *numOfCores = atof(buf);
+  }
+  taosCloseCmd(&pCmd);
 
   return code;
 #else

tests/system-test/0-others/taosShellNetChk.py

@@ -231,7 +231,7 @@ class TDTestCase:
             if platform.system().lower() == 'windows':
                 os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9')
             else:
-                os.system('pkill taos')
+                os.system('pkill -9 taos')
 
     def stop(self):
         tdSql.close()

tests/system-test/1-insert/mutil_stage.py

 from datetime import datetime
+from platform import platform
 import time
 
 from typing import List, Any, Tuple
@@ -83,6 +84,8 @@ class TDTestCase:
 
     def del_old_datadir(self, filename):
         cmd = f"sed -i '/^dataDir/d' {filename}"
+        if platform.system().lower() == 'darwin':
+            cmd = f"sed -i '' '/^dataDir/d' {filename}"
         if os.system(cmd) != 0:
             tdLog.exit(cmd)