From 61c75fe0d7b92ba84c4d0b6f552aa7a8aec24f3a Mon Sep 17 00:00:00 2001 From: wpan Date: Wed, 15 Sep 2021 17:17:06 +0800 Subject: [PATCH] add ut --- src/query/src/tdigest.c | 18 +- src/query/tests/CMakeLists.txt | 5 +- src/query/tests/apercentileTest.cpp | 252 ++++++++++++++++++++++++++++ 3 files changed, 267 insertions(+), 8 deletions(-) create mode 100644 src/query/tests/apercentileTest.cpp diff --git a/src/query/src/tdigest.c b/src/query/src/tdigest.c index 62ebbc87dc..660a42e3d8 100644 --- a/src/query/src/tdigest.c +++ b/src/query/src/tdigest.c @@ -48,7 +48,7 @@ void tdigestAutoFill(TDigest* t, int32_t compression) { } TDigest *tdigestNewFrom(void* pBuf, int32_t compression) { - memset(pBuf, 0, sizeof(TDigest) + sizeof(SCentroid)*(compression + 1)); + memset(pBuf, 0, TDIGEST_SIZE(compression)); TDigest* t = (TDigest*)pBuf; tdigestAutoFill(t, compression); @@ -129,20 +129,26 @@ void tdigestCompress(TDigest *t) { SCentroid *b = &t->centroids[j]; if (a->mean <= b->mean) { - mergeCentroid(&args, a); + mergeCentroid(&args, a); + assert(args.idx < t->size); i++; } else { mergeCentroid(&args, b); + assert(args.idx < t->size); j++; } } - while (i < num_unmerged) + while (i < num_unmerged) { mergeCentroid(&args, &unmerged_centroids[i++]); + assert(args.idx < t->size); + } free((void*)unmerged_centroids); - while (j < t->num_centroids) + while (j < t->num_centroids) { mergeCentroid(&args, &t->centroids[j++]); + assert(args.idx < t->size); + } if (t->total_weight > 0) { t->min = MIN(t->min, args.min); @@ -166,7 +172,7 @@ void tdigestAdd(TDigest* t, double x, int64_t w) { t->buffered_pts[i].weight = w; t->num_buffered_pts++; - if (t->num_buffered_pts > t->threshold) + if (t->num_buffered_pts >= t->threshold) tdigestCompress(t); } @@ -294,4 +300,4 @@ void tdigestMerge(TDigest *t1, TDigest *t2) { for (int32_t i = 0; i < t2->num_centroids; i++) { tdigestAdd(t1, t2->centroids[i].mean, t2->centroids[i].weight); } -} \ No newline at end of file +} diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt index 349d511f15..be3d2fc30b 100644 --- a/src/query/tests/CMakeLists.txt +++ b/src/query/tests/CMakeLists.txt @@ -17,13 +17,14 @@ IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - ADD_EXECUTABLE(queryTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread) + ADD_EXECUTABLE(queryUnitTest ${SOURCE_LIST}) + TARGET_LINK_LIBRARIES(queryUnitTest taos query gtest pthread) ENDIF() SET_SOURCE_FILES_PROPERTIES(./astTest.cpp PROPERTIES COMPILE_FLAGS -w) SET_SOURCE_FILES_PROPERTIES(./histogramTest.cpp PROPERTIES COMPILE_FLAGS -w) SET_SOURCE_FILES_PROPERTIES(./percentileTest.cpp PROPERTIES COMPILE_FLAGS -w) +SET_SOURCE_FILES_PROPERTIES(./apercentileTest.cpp PROPERTIES COMPILE_FLAGS -w) SET_SOURCE_FILES_PROPERTIES(./resultBufferTest.cpp PROPERTIES COMPILE_FLAGS -w) SET_SOURCE_FILES_PROPERTIES(./tsBufTest.cpp PROPERTIES COMPILE_FLAGS -w) SET_SOURCE_FILES_PROPERTIES(./unitTest.cpp PROPERTIES COMPILE_FLAGS -w) diff --git a/src/query/tests/apercentileTest.cpp b/src/query/tests/apercentileTest.cpp new file mode 100644 index 0000000000..3fc5f00e9b --- /dev/null +++ b/src/query/tests/apercentileTest.cpp @@ -0,0 +1,252 @@ +#include +#include + +#include "qResultbuf.h" +#include "taos.h" +#include "taosdef.h" + +#include "assert.h" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" + +extern "C" { +#include "tdigest.h" +} + + +namespace { + +enum { + TEST_DATA_TYPE_INT = 0, + TEST_DATA_TYPE_BIGINT, + TEST_DATA_TYPE_FLOAT, + TEST_DATA_TYPE_DOUBLE +}; + +enum { + TEST_DATA_MODE_SEQ = 0, + TEST_DATA_MODE_DSEQ, + TEST_DATA_MODE_RAND_PER, + TEST_DATA_MODE_RAND_LIMIT, +}; + + +void tdigest_init(TDigest **pTDigest) { + void *tmp = calloc(1, (int16_t)(TDIGEST_SIZE(COMPRESSION))); + *pTDigest = tdigestNewFrom(tmp, COMPRESSION); +} + +static FORCE_INLINE int64_t testGetTimestampUs() { + struct timeval systemTime; + gettimeofday(&systemTime, NULL); + return (int64_t)systemTime.tv_sec * 1000000L + (int64_t)systemTime.tv_usec; +} + + +void setTestData(void *data, int64_t idx, int32_t type, int64_t value) { + switch (type) { + case TEST_DATA_TYPE_INT: + *((int32_t*)data + idx) = (int32_t)value; + break; + case TEST_DATA_TYPE_BIGINT: + *((int64_t*)data + idx) = (int64_t)value; + break; + case TEST_DATA_TYPE_FLOAT: + *((float*)data + idx) = (float)value; + break; + case TEST_DATA_TYPE_DOUBLE: + *((double*)data + idx) = (double)value; + break; + default: + assert(0); + } +} + + +void addTestData(void *data, int64_t idx, int32_t type, TDigest* pTDigest) { + switch (type) { + case TEST_DATA_TYPE_INT: + tdigestAdd(pTDigest, (double)*((int32_t*)data + idx), 1); + break; + case TEST_DATA_TYPE_BIGINT: + tdigestAdd(pTDigest, (double)*((int64_t*)data + idx), 1); + break; + case TEST_DATA_TYPE_FLOAT: + tdigestAdd(pTDigest, (double)*((float*)data + idx), 1); + break; + case TEST_DATA_TYPE_DOUBLE: + tdigestAdd(pTDigest, (double)*((double*)data + idx), 1); + break; + default: + assert(0); + } +} + + + +void initTestData(void **data, int32_t type, int64_t num, int32_t mode, int32_t randPar) { + int32_t tsize[] = {4, 8, 4, 8}; + + *data = malloc(num * tsize[type]); + + switch (mode) { + case TEST_DATA_MODE_SEQ: + for (int64_t i = 0; i < num; ++i) { + setTestData(*data, i, type, i); + } + break; + case TEST_DATA_MODE_DSEQ: + for (int64_t i = 0; i < num; ++i) { + setTestData(*data, i, type, num - i); + } + break; + case TEST_DATA_MODE_RAND_PER: { + srand(time(NULL)); + int64_t randMax = num * randPar / 100; + + if (randMax == 0) { + for (int64_t i = 0; i < num; ++i) { + setTestData(*data, i, type, rand()); + } + } else { + for (int64_t i = 0; i < num; ++i) { + setTestData(*data, i, type, rand() % randMax); + } + } + } + break; + case TEST_DATA_MODE_RAND_LIMIT: + srand(time(NULL)); + for (int64_t i = 0; i < num; ++i) { + setTestData(*data, i, type, rand() % randPar); + } + break; + + default: + assert(0); + } +} + + +void tdigestTest() { + printf("running %s\n", __FUNCTION__); + + TDigest *pTDigest = NULL; + void *data = NULL; + + int64_t totalNum[] = {100,10000,10000000}; + int32_t numTimes = sizeof(totalNum)/sizeof(totalNum[0]); + int64_t biggestNum = totalNum[numTimes - 1]; + int32_t unitNum[] = {1,10,100,1000,5000,10000,100000}; + int32_t unitTimes = sizeof(unitNum)/sizeof(unitNum[0]); + int32_t dataMode[] = {TEST_DATA_MODE_SEQ, TEST_DATA_MODE_DSEQ, TEST_DATA_MODE_RAND_PER, TEST_DATA_MODE_RAND_LIMIT}; + int32_t modeTimes = sizeof(dataMode)/sizeof(dataMode[0]); + int32_t dataTypes[] = {TEST_DATA_TYPE_INT, TEST_DATA_TYPE_BIGINT, TEST_DATA_TYPE_FLOAT, TEST_DATA_TYPE_DOUBLE}; + int32_t typeTimes = sizeof(dataTypes)/sizeof(dataTypes[0]); + int32_t randPers[] = {0, 1, 10, 50, 90}; + int32_t randPTimes = sizeof(randPers)/sizeof(randPers[0]); + int32_t randLimits[] = {10, 50, 100, 1000, 10000}; + int32_t randLTimes = sizeof(randLimits)/sizeof(randLimits[0]); + + double useTime[10][10][10][10] = {0.0}; + + for (int32_t i = 0; i < modeTimes; ++i) { + if (dataMode[i] == TEST_DATA_MODE_RAND_PER) { + for (int32_t p = 0; p < randPTimes; ++p) { + for (int32_t j = 0; j < typeTimes; ++j) { + initTestData(&data, dataTypes[j], biggestNum, dataMode[i], randPers[p]); + for (int32_t m = 0; m < numTimes; ++m) { + int64_t startu = testGetTimestampUs(); + tdigest_init(&pTDigest); + for (int64_t n = 0; n < totalNum[m]; ++n) { + addTestData(data, n, dataTypes[j], pTDigest); + } + double res = tdigestQuantile(pTDigest, 50/100); + free(pTDigest); + useTime[i][j][m][p] = ((double)(testGetTimestampUs() - startu))/1000; + printf("Mode:%d,Type:%d,Num:%"PRId64",randP:%d,Used:%fms\n", dataMode[i], dataTypes[j], totalNum[m], randPers[p], useTime[i][j][m][p]); + } + free(data); + } + } + } else if (dataMode[i] == TEST_DATA_MODE_RAND_LIMIT) { + for (int32_t p = 0; p < randLTimes; ++p) { + for (int32_t j = 0; j < typeTimes; ++j) { + initTestData(&data, dataTypes[j], biggestNum, dataMode[i], randLimits[p]); + for (int64_t m = 0; m < numTimes; ++m) { + int64_t startu = testGetTimestampUs(); + tdigest_init(&pTDigest); + for (int64_t n = 0; n < totalNum[m]; ++n) { + addTestData(data, m, dataTypes[j], pTDigest); + } + double res = tdigestQuantile(pTDigest, 50/100); + free(pTDigest); + useTime[i][j][m][p] = ((double)(testGetTimestampUs() - startu))/1000; + printf("Mode:%d,Type:%d,Num:%"PRId64",randL:%d,Used:%fms\n", dataMode[i], dataTypes[j], totalNum[m], randLimits[p], useTime[i][j][m][p]); + } + free(data); + } + } + } else { + for (int32_t j = 0; j < typeTimes; ++j) { + initTestData(&data, dataTypes[j], biggestNum, dataMode[i], 0); + for (int64_t m = 0; m < numTimes; ++m) { + int64_t startu = testGetTimestampUs(); + tdigest_init(&pTDigest); + for (int64_t n = 0; n < totalNum[m]; ++n) { + addTestData(data, m, dataTypes[j], pTDigest); + } + double res = tdigestQuantile(pTDigest, 50/100); + free(pTDigest); + useTime[i][j][m][0] = ((double)(testGetTimestampUs() - startu))/1000; + printf("Mode:%d,Type:%d,Num:%"PRId64",Used:%fms\n", dataMode[i], dataTypes[j], totalNum[m], useTime[i][j][m][0]); + } + free(data); + } + } + } + + + printf("\n\n"); + + + for (int32_t i = 0; i < modeTimes; ++i) { + if (dataMode[i] == TEST_DATA_MODE_RAND_PER) { + for (int32_t p = 0; p < randPTimes; ++p) { + for (int32_t j = 0; j < typeTimes; ++j) { + printf("Mode:%d,Type:%d,randP:%d -", dataMode[i], dataTypes[j], randPers[p]); + for (int32_t m = 0; m < numTimes; ++m) { + printf(" %d:%f", totalNum[m], useTime[i][j][m][p]); + } + printf("\n"); + } + } + } else if (dataMode[i] == TEST_DATA_MODE_RAND_LIMIT) { + for (int32_t p = 0; p < randLTimes; ++p) { + for (int32_t j = 0; j < typeTimes; ++j) { + printf("Mode:%d,Type:%d,randL:%d -", dataMode[i], dataTypes[j], randLimits[p]); + for (int64_t m = 0; m < numTimes; ++m) { + printf(" %d:%f", totalNum[m], useTime[i][j][m][p]); + } + printf("\n"); + } + } + } else { + for (int32_t j = 0; j < typeTimes; ++j) { + printf("Mode:%d,Type:%d -", dataMode[i], dataTypes[j]); + for (int64_t m = 0; m < numTimes; ++m) { + printf(" %d:%f", totalNum[m], useTime[i][j][m][0]); + } + printf("\n"); + } + } + } +} + + +} // namespace + +TEST(testCase, apercentileTest) { + tdigestTest(); +} -- GitLab