add ut

src/query/src/tdigest.c

@@ -48,7 +48,7 @@ void tdigestAutoFill(TDigest* t, int32_t compression) {
 }
 
 TDigest *tdigestNewFrom(void* pBuf, int32_t compression) {
-    memset(pBuf, 0, sizeof(TDigest) + sizeof(SCentroid)*(compression + 1));
+    memset(pBuf, 0, TDIGEST_SIZE(compression));
     TDigest* t = (TDigest*)pBuf;
     tdigestAutoFill(t, compression);
 
@@ -129,20 +129,26 @@ void tdigestCompress(TDigest *t) {
         SCentroid *b = &t->centroids[j];
 
         if (a->mean <= b->mean) {
-            mergeCentroid(&args, a);
+            mergeCentroid(&args, a);            
+            assert(args.idx < t->size);
             i++;
         } else {
             mergeCentroid(&args, b);
+            assert(args.idx < t->size);
             j++;
         }
     }
 
-    while (i < num_unmerged)
+    while (i < num_unmerged) {
         mergeCentroid(&args, &unmerged_centroids[i++]);
+        assert(args.idx < t->size);
+    }
     free((void*)unmerged_centroids);
 
-    while (j < t->num_centroids)
+    while (j < t->num_centroids) {
         mergeCentroid(&args, &t->centroids[j++]);
+        assert(args.idx < t->size);
+    }
 
     if (t->total_weight > 0) {
         t->min = MIN(t->min, args.min);
@@ -166,7 +172,7 @@ void tdigestAdd(TDigest* t, double x, int64_t w) {
     t->buffered_pts[i].weight = w;
     t->num_buffered_pts++;
 
-    if (t->num_buffered_pts > t->threshold)
+    if (t->num_buffered_pts >= t->threshold)
         tdigestCompress(t);
 }
 
@@ -294,4 +300,4 @@ void tdigestMerge(TDigest *t1, TDigest *t2) {
     for (int32_t i = 0; i < t2->num_centroids; i++) {
         tdigestAdd(t1, t2->centroids[i].mean, t2->centroids[i].weight);
     }
-}
\ No newline at end of file
+}

src/query/tests/CMakeLists.txt

@@ -17,13 +17,14 @@ IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR))
     INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
     AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-    ADD_EXECUTABLE(queryTest ${SOURCE_LIST})
-    TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread)
+    ADD_EXECUTABLE(queryUnitTest ${SOURCE_LIST})
+    TARGET_LINK_LIBRARIES(queryUnitTest taos query gtest pthread)
 ENDIF()
 
 SET_SOURCE_FILES_PROPERTIES(./astTest.cpp PROPERTIES COMPILE_FLAGS -w)
 SET_SOURCE_FILES_PROPERTIES(./histogramTest.cpp PROPERTIES COMPILE_FLAGS -w)
 SET_SOURCE_FILES_PROPERTIES(./percentileTest.cpp PROPERTIES COMPILE_FLAGS -w)
+SET_SOURCE_FILES_PROPERTIES(./apercentileTest.cpp PROPERTIES COMPILE_FLAGS -w)
 SET_SOURCE_FILES_PROPERTIES(./resultBufferTest.cpp PROPERTIES COMPILE_FLAGS -w)
 SET_SOURCE_FILES_PROPERTIES(./tsBufTest.cpp PROPERTIES COMPILE_FLAGS -w)
 SET_SOURCE_FILES_PROPERTIES(./unitTest.cpp PROPERTIES COMPILE_FLAGS -w)

src/query/tests/apercentileTest.cpp

+#include <gtest/gtest.h>
+#include <iostream>
+
+#include "qResultbuf.h"
+#include "taos.h"
+#include "taosdef.h"
+
+#include "assert.h"
+
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+
+extern "C" {
+#include "tdigest.h"
+}
+
+
+namespace {
+
+enum {
+  TEST_DATA_TYPE_INT  = 0,
+  TEST_DATA_TYPE_BIGINT,
+  TEST_DATA_TYPE_FLOAT,
+  TEST_DATA_TYPE_DOUBLE
+};
+
+enum {
+  TEST_DATA_MODE_SEQ  = 0,
+  TEST_DATA_MODE_DSEQ,
+  TEST_DATA_MODE_RAND_PER,
+  TEST_DATA_MODE_RAND_LIMIT,
+};
+
+
+void tdigest_init(TDigest **pTDigest) {
+  void *tmp = calloc(1, (int16_t)(TDIGEST_SIZE(COMPRESSION)));
+  *pTDigest = tdigestNewFrom(tmp, COMPRESSION);
+}
+
+static FORCE_INLINE int64_t testGetTimestampUs() {
+  struct timeval systemTime;
+  gettimeofday(&systemTime, NULL);
+  return (int64_t)systemTime.tv_sec * 1000000L + (int64_t)systemTime.tv_usec;
+}
+
+
+void setTestData(void *data, int64_t idx, int32_t type, int64_t value) {
+  switch (type) {
+    case TEST_DATA_TYPE_INT:
+      *((int32_t*)data + idx) = (int32_t)value;
+      break;
+    case TEST_DATA_TYPE_BIGINT:
+      *((int64_t*)data + idx) = (int64_t)value;
+      break;
+    case TEST_DATA_TYPE_FLOAT:
+      *((float*)data + idx) = (float)value;
+      break;
+    case TEST_DATA_TYPE_DOUBLE:
+      *((double*)data + idx) = (double)value;
+      break;
+    default:
+      assert(0);
+  }
+}
+
+
+void addTestData(void *data, int64_t idx, int32_t type, TDigest* pTDigest) {
+  switch (type) {
+    case TEST_DATA_TYPE_INT:
+      tdigestAdd(pTDigest, (double)*((int32_t*)data + idx), 1);
+      break;
+    case TEST_DATA_TYPE_BIGINT:
+      tdigestAdd(pTDigest, (double)*((int64_t*)data + idx), 1);
+      break;
+    case TEST_DATA_TYPE_FLOAT:
+      tdigestAdd(pTDigest, (double)*((float*)data + idx), 1);
+      break;
+    case TEST_DATA_TYPE_DOUBLE:
+      tdigestAdd(pTDigest, (double)*((double*)data + idx), 1);
+      break;
+    default:
+      assert(0);
+  }
+}
+
+
+
+void initTestData(void **data, int32_t type, int64_t num, int32_t mode, int32_t randPar) {
+  int32_t tsize[] = {4, 8, 4, 8};
+  
+  *data = malloc(num * tsize[type]);
+
+  switch (mode) {
+    case TEST_DATA_MODE_SEQ:
+      for (int64_t i = 0; i < num; ++i) {
+        setTestData(*data, i, type, i);
+      }
+      break;
+    case TEST_DATA_MODE_DSEQ:
+      for (int64_t i = 0; i < num; ++i) {
+        setTestData(*data, i, type, num - i);
+      }
+      break;
+    case TEST_DATA_MODE_RAND_PER: {
+        srand(time(NULL));
+        int64_t randMax = num * randPar / 100;
+
+        if (randMax == 0) {
+          for (int64_t i = 0; i < num; ++i) {
+            setTestData(*data, i, type, rand());
+          }
+        } else {
+          for (int64_t i = 0; i < num; ++i) {
+            setTestData(*data, i, type, rand() % randMax);
+          }
+        }
+      }
+      break;
+    case TEST_DATA_MODE_RAND_LIMIT:
+      srand(time(NULL));
+      for (int64_t i = 0; i < num; ++i) {
+        setTestData(*data, i, type, rand() % randPar);
+      }
+      break;
+
+    default:
+      assert(0);
+  }
+}
+
+
+void tdigestTest() {
+  printf("running %s\n", __FUNCTION__);
+
+  TDigest *pTDigest = NULL;
+  void *data = NULL;
+  
+  int64_t totalNum[] = {100,10000,10000000};
+  int32_t numTimes = sizeof(totalNum)/sizeof(totalNum[0]);
+  int64_t biggestNum = totalNum[numTimes - 1];
+  int32_t unitNum[] = {1,10,100,1000,5000,10000,100000};
+  int32_t unitTimes = sizeof(unitNum)/sizeof(unitNum[0]);
+  int32_t dataMode[] = {TEST_DATA_MODE_SEQ, TEST_DATA_MODE_DSEQ, TEST_DATA_MODE_RAND_PER, TEST_DATA_MODE_RAND_LIMIT};
+  int32_t modeTimes = sizeof(dataMode)/sizeof(dataMode[0]);
+  int32_t dataTypes[] = {TEST_DATA_TYPE_INT, TEST_DATA_TYPE_BIGINT, TEST_DATA_TYPE_FLOAT, TEST_DATA_TYPE_DOUBLE};
+  int32_t typeTimes = sizeof(dataTypes)/sizeof(dataTypes[0]);
+  int32_t randPers[] = {0, 1, 10, 50, 90};
+  int32_t randPTimes = sizeof(randPers)/sizeof(randPers[0]);
+  int32_t randLimits[] = {10, 50, 100, 1000, 10000};
+  int32_t randLTimes = sizeof(randLimits)/sizeof(randLimits[0]);
+
+  double useTime[10][10][10][10] = {0.0};
+  
+  for (int32_t i = 0; i < modeTimes; ++i) {
+    if (dataMode[i] == TEST_DATA_MODE_RAND_PER) {
+      for (int32_t p = 0; p < randPTimes; ++p) {
+        for (int32_t j = 0; j < typeTimes; ++j) {
+          initTestData(&data, dataTypes[j], biggestNum, dataMode[i], randPers[p]);
+          for (int32_t m = 0; m < numTimes; ++m) {          
+            int64_t startu = testGetTimestampUs();
+            tdigest_init(&pTDigest);
+            for (int64_t n = 0; n < totalNum[m]; ++n) {
+              addTestData(data, n, dataTypes[j], pTDigest);
+            }
+            double res = tdigestQuantile(pTDigest, 50/100);
+            free(pTDigest);
+            useTime[i][j][m][p] = ((double)(testGetTimestampUs() - startu))/1000;
+            printf("Mode:%d,Type:%d,Num:%"PRId64",randP:%d,Used:%fms\n", dataMode[i], dataTypes[j], totalNum[m], randPers[p], useTime[i][j][m][p]);
+          }
+          free(data);
+        }
+      }
+    } else if (dataMode[i] == TEST_DATA_MODE_RAND_LIMIT) {
+      for (int32_t p = 0; p < randLTimes; ++p) {
+        for (int32_t j = 0; j < typeTimes; ++j) {
+          initTestData(&data, dataTypes[j], biggestNum, dataMode[i], randLimits[p]);
+          for (int64_t m = 0; m < numTimes; ++m) {
+            int64_t startu = testGetTimestampUs();          
+            tdigest_init(&pTDigest);
+            for (int64_t n = 0; n < totalNum[m]; ++n) {
+              addTestData(data, m, dataTypes[j], pTDigest);
+            }
+            double res = tdigestQuantile(pTDigest, 50/100);
+            free(pTDigest);                  
+            useTime[i][j][m][p] = ((double)(testGetTimestampUs() - startu))/1000;
+            printf("Mode:%d,Type:%d,Num:%"PRId64",randL:%d,Used:%fms\n", dataMode[i], dataTypes[j], totalNum[m], randLimits[p], useTime[i][j][m][p]);
+          }          
+          free(data);
+        }
+      }
+    } else {
+      for (int32_t j = 0; j < typeTimes; ++j) {
+        initTestData(&data, dataTypes[j], biggestNum, dataMode[i], 0);
+        for (int64_t m = 0; m < numTimes; ++m) {
+          int64_t startu = testGetTimestampUs();        
+          tdigest_init(&pTDigest);        
+          for (int64_t n = 0; n < totalNum[m]; ++n) {
+            addTestData(data, m, dataTypes[j], pTDigest);
+          }
+          double res = tdigestQuantile(pTDigest, 50/100);
+          free(pTDigest);
+          useTime[i][j][m][0] = ((double)(testGetTimestampUs() - startu))/1000;
+          printf("Mode:%d,Type:%d,Num:%"PRId64",Used:%fms\n", dataMode[i], dataTypes[j], totalNum[m], useTime[i][j][m][0]);
+        }        
+        free(data);
+      }
+    }
+  }
+
+
+  printf("\n\n");
+
+  
+  for (int32_t i = 0; i < modeTimes; ++i) {
+    if (dataMode[i] == TEST_DATA_MODE_RAND_PER) {
+      for (int32_t p = 0; p < randPTimes; ++p) {
+        for (int32_t j = 0; j < typeTimes; ++j) {
+          printf("Mode:%d,Type:%d,randP:%d -", dataMode[i], dataTypes[j], randPers[p]);
+          for (int32_t m = 0; m < numTimes; ++m) {          
+            printf(" %d:%f", totalNum[m], useTime[i][j][m][p]);
+          }
+          printf("\n");
+        }
+      }
+    } else if (dataMode[i] == TEST_DATA_MODE_RAND_LIMIT) {
+      for (int32_t p = 0; p < randLTimes; ++p) {
+        for (int32_t j = 0; j < typeTimes; ++j) {
+          printf("Mode:%d,Type:%d,randL:%d -", dataMode[i], dataTypes[j], randLimits[p]);
+          for (int64_t m = 0; m < numTimes; ++m) {
+            printf(" %d:%f", totalNum[m], useTime[i][j][m][p]);
+          }
+          printf("\n");
+        }
+      }
+    } else {
+      for (int32_t j = 0; j < typeTimes; ++j) {
+        printf("Mode:%d,Type:%d -", dataMode[i], dataTypes[j]);
+        for (int64_t m = 0; m < numTimes; ++m) {
+          printf(" %d:%f", totalNum[m], useTime[i][j][m][0]);
+        }
+        printf("\n");
+      }
+    }
+  }
+}
+
+
+}  // namespace
+
+TEST(testCase, apercentileTest) {
+  tdigestTest();
+}