Merge branch '2.6' into fix/TS-2670-2.6

docs/en/25-application/01-telegraf.md

@@ -72,7 +72,7 @@ sudo systemctl start telegraf
 
 Log in to the Grafana interface using a web browser at `IP:3000`, with the system's initial username and password being `admin/admin`.
 Click on the gear icon on the left and select `Plugins`, you should find the TDengine data source plugin icon.
-Click on the plus icon on the left and select `Import` to get the data from `https://github.com/taosdata/grafanaplugin/blob/master/examples/telegraf/grafana/dashboards/telegraf-dashboard-v0.1.0.json`, download the dashboard JSON file and import it. You will then see the dashboard in the following screen.
+Click on the plus icon on the left and select `Import` to get the data from `https://github.com/taosdata/grafanaplugin/blob/master/examples/telegraf/grafana/dashboards/telegraf-dashboard-v2.json`, download the dashboard JSON file and import it. You will then see the dashboard in the following screen.
 
 ![TDengine Database IT-DevOps-Solutions-telegraf-dashboard](./IT-DevOps-Solutions-telegraf-dashboard.webp)
 

docs/zh/25-application/01-telegraf.md

@@ -72,7 +72,7 @@ sudo systemctl start telegraf
 
 使用 Web 浏览器访问 `IP:3000` 登录 Grafana 界面，系统初始用户名密码为 admin/admin。
 点击左侧齿轮图标并选择 `Plugins`，应该可以找到 TDengine data source 插件图标。
-点击左侧加号图标并选择 `Import`，从 `https://github.com/taosdata/grafanaplugin/blob/master/examples/telegraf/grafana/dashboards/telegraf-dashboard-v0.1.0.json` 下载 dashboard JSON 文件后导入。之后可以看到如下界面的仪表盘：
+点击左侧加号图标并选择 `Import`，从 `https://github.com/taosdata/grafanaplugin/blob/master/examples/telegraf/grafana/dashboards/telegraf-dashboard-v2.json` 下载 dashboard JSON 文件后导入。之后可以看到如下界面的仪表盘：
 
 ![TDengine Database IT-DevOps-Solutions-telegraf-dashboard](./IT-DevOps-Solutions-telegraf-dashboard.webp)
 

src/client/src/tscServer.c

@@ -1252,12 +1252,29 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   pQueryMsg->tsBuf.tsOffset = htonl((int32_t)(pMsg - pCmd->payload));
 
   if (pQueryInfo->tsBuf != NULL) {
+    bool qType = tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0);
+    if (qType) {
+      dumpFileBlockByGroupIndex(pQueryInfo->tsBuf, pTableMetaInfo->vgroupIndex, pMsg, &pQueryMsg->tsBuf.tsLen, &pQueryMsg->tsBuf.tsNumOfBlocks);
+      if (code != TSDB_CODE_SUCCESS) {
+        goto _end;
+      }
+    } else {
       // note: here used the idx instead of actual vnode id.
+      int32_t vgId = 0;
+      if (pTableMetaInfo->pVgroupTables != NULL) {
         int32_t           vnodeIndex = pTableMetaInfo->vgroupIndex;
-    code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsBuf.tsLen, &pQueryMsg->tsBuf.tsNumOfBlocks);
+        SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, vnodeIndex);
+        vgId = pTableInfo->vgInfo.vgId;
+      } else {
+        vgId = query.vgId;
+      }
+
+      code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vgId, pMsg, &pQueryMsg->tsBuf.tsLen,
+                                    &pQueryMsg->tsBuf.tsNumOfBlocks);
       if (code != TSDB_CODE_SUCCESS) {
         goto _end;
       }
+    }
 
     pMsg += pQueryMsg->tsBuf.tsLen;
 
@@ -1317,8 +1334,6 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   memcpy(pMsg, pSql->sqlstr, sqlLen);
   pMsg += sqlLen;
 
-
-
   pQueryMsg->extend = 1;
   
   STLV *tlv = (STLV *)pMsg;

taos-tools @ 0111c666

-Subproject commit 61cbfd2cf72d973bb35e91fa8c9df679da026499
+Subproject commit 0111c666ddb9fc05889592412ba70c2c3764febf

src/query/inc/qTsbuf.h

@@ -110,7 +110,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf);
 
 STSBuf* tsBufClone(STSBuf* pTSBuf);
 
-STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id);
+SArray* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id);
 
 void    tsBufFlush(STSBuf* pTSBuf);
 void    tsBufResetPos(STSBuf* pTSBuf);
@@ -136,6 +136,8 @@ void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id);
 
 int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t id, void* buf, int32_t* len, int32_t* numOfBlocks);
 
+int32_t dumpFileBlockByGroupIndex(STSBuf* pTSBuf, int32_t groupIndex, void* pBuf, int32_t* len, int32_t* numOfBlocks);
+
 STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag);
 
 bool tsBufIsValidElem(STSElem* pElem);

src/query/src/qTsbuf.c

@@ -593,22 +593,22 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex
   }
   
   STSCursor* pCur = &pTSBuf->cur;
-  if (pCur->vgroupIndex == groupIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) ||
-      (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) {
-    int32_t i = 0;
-    bool    decomp = false;
-    int32_t step = abs(blockIndex - pCur->blockIndex);
-    
-    while ((++i) <= step) {
-      if (readDataFromDisk(pTSBuf, pCur->order, decomp) == NULL) {
-        return;
-      }
-    }
-  } else {
+//  if (pCur->vgroupIndex == groupIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) ||
+//      (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) {
+//    int32_t i = 0;
+//    bool    decomp = false;
+//    int32_t step = abs(blockIndex - pCur->blockIndex);
+//
+//    while ((++i) <= step) {
+//      if (readDataFromDisk(pTSBuf, pCur->order, decomp) == NULL) {
+//        return;
+//      }
+//    }
+//  } else {
     if (tsBufFindBlock(pTSBuf, pBlockInfo, blockIndex) == -1) {
       assert(false);
     }
-  }
+//  }
   
   STSBlock* pBlock = &pTSBuf->block;
   
@@ -651,13 +651,16 @@ static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockIn
   return 0;
 }
 
-STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) {
-  int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id);
-  if (j == -1) {
-    return NULL;
+SArray* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) {
+  SArray* pList = taosArrayInit(4, sizeof(STSGroupBlockInfo));
+
+  for(int32_t i = 0; i < pTSBuf->numOfGroups; ++i) {
+    if (pTSBuf->pData[i].info.id == id) {
+      taosArrayPush(pList, &pTSBuf->pData[i].info);
+    }
   }
 
-  return &pTSBuf->pData[j].info;
+  return pList;
 }
 
 int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) {
@@ -1099,23 +1102,54 @@ void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) {
   }
 }
 
-int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, int32_t* len, int32_t* numOfBlocks) {
+int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupId, void* buf, int32_t* len, int32_t* numOfBlocks) {
+  SArray* pList = tsBufGetGroupBlockInfo(pTSBuf, groupId);
+
+  *len = 0;
+  *numOfBlocks = 0;
+
+  char* p = buf;
+  for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
+    STSGroupBlockInfo* pBlockInfo = taosArrayGet(pList, i);
+
+    if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
+      int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
+      //    qError("%p: fseek failed: %s", pSql, tstrerror(code));
+      return code;
+    }
+
+    size_t s = fread(p, 1, pBlockInfo->compLen, pTSBuf->f);
+    if (s != pBlockInfo->compLen) {
+      int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
+      //    tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
+      return code;
+    }
+
+    *len += pBlockInfo->compLen;
+    *numOfBlocks += pBlockInfo->numOfBlocks;
+    p += pBlockInfo->compLen;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t dumpFileBlockByGroupIndex(STSBuf* pTSBuf, int32_t groupIndex, void* pBuf, int32_t* len, int32_t* numOfBlocks) {
   assert(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups);
-  STSGroupBlockInfo *pBlockInfo = &pTSBuf->pData[groupIndex].info;
+  STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info;
 
   *len = 0;
   *numOfBlocks = 0;
 
   if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
     int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
-//    qError("%p: fseek failed: %s", pSql, tstrerror(code));
+    //    qError("%p: fseek failed: %s", pSql, tstrerror(code));
     return code;
   }
 
-  size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f);
+  size_t s = fread(pBuf, 1, pBlockInfo->compLen, pTSBuf->f);
   if (s != pBlockInfo->compLen) {
     int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
-//    tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
+    //    tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
     return code;
   }
 

src/query/tests/CMakeLists.txt

@@ -30,11 +30,11 @@ IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR))
     ENDIF()
 ENDIF()
 
-SET_SOURCE_FILES_PROPERTIES(./astTest.cpp PROPERTIES COMPILE_FLAGS -w)
-SET_SOURCE_FILES_PROPERTIES(./histogramTest.cpp PROPERTIES COMPILE_FLAGS -w)
-SET_SOURCE_FILES_PROPERTIES(./percentileTest.cpp PROPERTIES COMPILE_FLAGS -w)
-SET_SOURCE_FILES_PROPERTIES(./apercentileTest.cpp PROPERTIES COMPILE_FLAGS -w)
-SET_SOURCE_FILES_PROPERTIES(./resultBufferTest.cpp PROPERTIES COMPILE_FLAGS -w)
+#SET_SOURCE_FILES_PROPERTIES(./astTest.cpp PROPERTIES COMPILE_FLAGS -w)
+#SET_SOURCE_FILES_PROPERTIES(./histogramTest.cpp PROPERTIES COMPILE_FLAGS -w)
+#SET_SOURCE_FILES_PROPERTIES(./percentileTest.cpp PROPERTIES COMPILE_FLAGS -w)
+#SET_SOURCE_FILES_PROPERTIES(./apercentileTest.cpp PROPERTIES COMPILE_FLAGS -w)
+#SET_SOURCE_FILES_PROPERTIES(./resultBufferTest.cpp PROPERTIES COMPILE_FLAGS -w)
 SET_SOURCE_FILES_PROPERTIES(./tsBufTest.cpp PROPERTIES COMPILE_FLAGS -w)
-SET_SOURCE_FILES_PROPERTIES(./unitTest.cpp PROPERTIES COMPILE_FLAGS -w)
-SET_SOURCE_FILES_PROPERTIES(./rangeMergeTest.cpp PROPERTIES COMPILE_FLAGS -w)
+#SET_SOURCE_FILES_PROPERTIES(./unitTest.cpp PROPERTIES COMPILE_FLAGS -w)
+#SET_SOURCE_FILES_PROPERTIES(./rangeMergeTest.cpp PROPERTIES COMPILE_FLAGS -w)

src/query/tests/apercentileTest.cpp

@@ -291,13 +291,13 @@ void tdigestTest() {
         for (int32_t j = 0; j < typeTimes; ++j) {
           printf("DMode:%d,Type:%d,randP:%d -", dataMode[i], dataTypes[j], randPers[p]);
           for (int32_t m = 0; m < numTimes; ++m) {          
-            printf(" %d:%f", totalNum[m], useTime[0][i][j][m][p]);
+            printf(" %ld:%f", totalNum[m], useTime[0][i][j][m][p]);
           }
           printf("\n");
 
           printf("HMode:%d,Type:%d,randP:%d -", dataMode[i], dataTypes[j], randPers[p]);
           for (int32_t m = 0; m < numTimes; ++m) {          
-            printf(" %d:%f", totalNum[m], useTime[1][i][j][m][p]);
+            printf(" %ld:%f", totalNum[m], useTime[1][i][j][m][p]);
           }
           printf("\n");          
         }
@@ -307,13 +307,13 @@ void tdigestTest() {
         for (int32_t j = 0; j < typeTimes; ++j) {
           printf("DMode:%d,Type:%d,randL:%d -", dataMode[i], dataTypes[j], randLimits[p]);
           for (int64_t m = 0; m < numTimes; ++m) {
-            printf(" %d:%f", totalNum[m], useTime[0][i][j][m][p]);
+            printf(" %ld:%f", totalNum[m], useTime[0][i][j][m][p]);
           }
           printf("\n");
 
           printf("HMode:%d,Type:%d,randL:%d -", dataMode[i], dataTypes[j], randLimits[p]);
           for (int64_t m = 0; m < numTimes; ++m) {
-            printf(" %d:%f", totalNum[m], useTime[1][i][j][m][p]);
+            printf(" %ld:%f", totalNum[m], useTime[1][i][j][m][p]);
           }
           printf("\n");          
         }
@@ -322,13 +322,13 @@ void tdigestTest() {
       for (int32_t j = 0; j < typeTimes; ++j) {
         printf("DMode:%d,Type:%d -", dataMode[i], dataTypes[j]);
         for (int64_t m = 0; m < numTimes; ++m) {
-          printf(" %d:%f", totalNum[m], useTime[0][i][j][m][0]);
+          printf(" %ld:%f", totalNum[m], useTime[0][i][j][m][0]);
         }
         printf("\n");
 
         printf("HMode:%d,Type:%d -", dataMode[i], dataTypes[j]);
         for (int64_t m = 0; m < numTimes; ++m) {
-          printf(" %d:%f", totalNum[m], useTime[1][i][j][m][0]);
+          printf(" %ld:%f", totalNum[m], useTime[1][i][j][m][0]);
         }
         printf("\n");        
       }

src/query/tests/histogramTest.cpp

@@ -35,7 +35,7 @@ void doHistogramAddTest() {
       (int64_t)systemTime.tv_sec * 1000L + (uint64_t)systemTime.tv_usec / 1000;
   printf("total elapsed time: %ld\n", et - st);
 
-  printf("elements: %d, slot:%d \n", pHisto->numOfElems, pHisto->numOfEntries);
+  printf("elements: %ld, slot:%d \n", pHisto->numOfElems, pHisto->numOfEntries);
   tHistogramPrint(pHisto);
 
   printf("%ld\n", tHistogramSum(pHisto, 1.5));

src/query/tests/tsBufTest.cpp

@@ -497,19 +497,103 @@ void mergeIdenticalVnodeBufferTest() {
   tsBufDestroy(pTSBuf1);
   tsBufDestroy(pTSBuf2);
 }
+
+void mergeMultiBlockFromOneGroupTest() {
+  STSBuf* pTSBuf1 = tsBufCreate(true, TSDB_ORDER_ASC);
+  STSBuf* pTSBuf2 = tsBufCreate(true, TSDB_ORDER_ASC);
+
+  tVariant t = {0};
+  t.nType = TSDB_DATA_TYPE_BIGINT;
+
+  int32_t step = 30;
+  int32_t num = 1000;
+  int32_t numOfTags = 2;
+
+  // vnodeId:0
+  int64_t start = 10000;
+  for (int32_t i = 0; i < numOfTags; ++i) {
+    int64_t* list = createTsList(num, start, step);
+    t.i64 = i;
+
+    tsBufAppend(pTSBuf1, 12, &t, (const char*)list, num * sizeof(int64_t));
+    free(list);
+
+    start += step * num;
+  }
+
+  tsBufFlush(pTSBuf1);
+
+  for (int32_t i = numOfTags; i < numOfTags * 2; ++i) {
+    int64_t* list = createTsList(num, start, step);
+
+    t.i64 = i;
+    tsBufAppend(pTSBuf1, 77, &t, (const char*)list, num * sizeof(int64_t));
+    free(list);
+
+    start += step * num;
+  }
+
+  tsBufFlush(pTSBuf1);
+
+  start = 10000;
+  for (int32_t i = 911; i < 912; ++i) {
+    int64_t* list = createTsList(num, start, step);
+    t.i64 = i;
+
+    tsBufAppend(pTSBuf1, 12, &t, (const char*)list, num * sizeof(int64_t));
+    free(list);
+
+    start += step * num;
+  }
+  tsBufFlush(pTSBuf1);
+
+  char* p = (char*) malloc(1024768);
+  int32_t len = 0;
+  int32_t numOfBlocks = 0;
+
+  dumpFileBlockByGroupId(pTSBuf1, 12, p, &len, &numOfBlocks);
+
+  STSBuf* pNew = tsBufCreateFromCompBlocks(p, numOfBlocks, len, 1, 12);
+  printf("%p\n", pNew);
+
+  tsBufDisplay(pNew);
+
+//  tsBufMerge(pTSBuf1, pTSBuf2);
+//  EXPECT_EQ(pTSBuf1->numOfGroups, 2);
+//  EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num);
+//
+//  tsBufResetPos(pTSBuf1);
+//
+//  int32_t count = 0;
+//  while (tsBufNextPos(pTSBuf1)) {
+//    STSElem elem = tsBufGetElem(pTSBuf1);
+//
+//    if (count++ < numOfTags * num) {
+//      EXPECT_EQ(elem.id, 12);
+//    } else {
+//      EXPECT_EQ(elem.id, 77);
+//    }
+//
+//    printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.id, elem.tag->i64, elem.ts);
+//  }
+//
+//  tsBufDestroy(pTSBuf1);
+//  tsBufDestroy(pTSBuf2);
+}
 }  // namespace
 
 
 //TODO add binary tag value test case
 TEST(testCase, tsBufTest) {
-  simpleTest();
-  largeTSTest();
-  multiTagsTest();
-  multiVnodeTagsTest();
-  loadDataTest();
-  invalidFileTest();
+//  simpleTest();
+//  largeTSTest();
+//  multiTagsTest();
+//  multiVnodeTagsTest();
+//  loadDataTest();
+//  invalidFileTest();
 //    randomIncTsTest();
-  TSTraverse();
-  mergeDiffVnodeBufferTest();
-  mergeIdenticalVnodeBufferTest();
+//  TSTraverse();
+//  mergeDiffVnodeBufferTest();
+//  mergeIdenticalVnodeBufferTest();
+  mergeMultiBlockFromOneGroupTest();
 }