From 9ff807755f613ce85f38e76fe19d27a87b8739f5 Mon Sep 17 00:00:00 2001 From: tickduan <417921451@qq.com> Date: Tue, 29 Jun 2021 16:44:09 +0800 Subject: [PATCH] test is passed by self with lossy compression --- src/kit/taospack/taospack.c | 142 ++++++++++++++++++++--------------- src/tsdb/src/tsdbMain.c | 6 +- src/util/inc/tscompression.h | 5 +- src/util/src/tcompression.c | 18 ++--- 4 files changed, 95 insertions(+), 76 deletions(-) diff --git a/src/kit/taospack/taospack.c b/src/kit/taospack/taospack.c index 0fe2c82955..2c89f3dbc7 100644 --- a/src/kit/taospack/taospack.c +++ b/src/kit/taospack/taospack.c @@ -144,7 +144,12 @@ bool testFile(const char* inFile, char algorithm){ printf(" file %s have count=%d \n", inFile, cnt); cost_start(); - int ret_len = tsCompressFloatLossy(input, input_len, cnt, output, output_len, algorithm, buff, buff_len); + int ret_len = 0; + if(algorithm == 2) + ret_len = tsCompressFloat(input, input_len, cnt, output, output_len, algorithm, buff, buff_len); + else + ret_len = tsCompressFloatLossy(input, input_len, cnt, output, output_len, algorithm, buff, buff_len); + if(ret_len == -1) { printf(" compress error.\n"); return 0; @@ -160,14 +165,21 @@ bool testFile(const char* inFile, char algorithm){ // float* ft2 = (float*)malloc(input_len); cost_start(); - int code = tsDecompressFloatLossy(output, ret_len, cnt, (char*)ft2, input_len, algorithm, buff, buff_len); + int code = 0; + + if(algorithm == 2) + code = tsDecompressFloat(output, ret_len, cnt, (char*)ft2, input_len, algorithm, buff, buff_len); + else + code = tsDecompressFloatLossy(output, ret_len, cnt, (char*)ft2, input_len, algorithm, buff, buff_len); + + double use_ms2 = cost_end("Decompress"); printf(" Decompress return length=%d \n", code); // compare same float same_rate = check_same(floats, ft2, cnt); - printf("\n ------------------ count:%d TD <%s> ---------------- \n", cnt, algorithm == ONE_STAGE_COMP?"ONE":"TWO"); + printf("\n ------------------ count:%d TD <%s> ---------------- \n", cnt, algorithm == 2?"TD":"SZ"); printf(" Compress Rate ......... [%.0f%%] \n", rate); double speed1 = (cnt*sizeof(float)*1000/1024/1024)/use_ms1; printf(" Compress Time ......... [%.4fms] speed=%.1f MB/s\n", use_ms1, speed1); @@ -184,61 +196,6 @@ bool testFile(const char* inFile, char algorithm){ return true; } - -int memTest(); -int memTestDouble(); -void test_threadsafe(int thread_count); -void test_threadsafe_double(int thread_count); - -// -// main -// -int main(int argc, char *argv[]) { - printf("welcome to use taospack tools v1.1. sizeof(STColumn) = %lu\n", sizeof(STColumn)); - - tsCompressInit(); - // - //tsCompressExit(); - //return 1; - - if(argc == 3){ - char algo = 0; - // t - if(strcmp(argv[1], "-tone") == 0 || strcmp(argv[1], "-t") == 0 ) { - algo = ONE_STAGE_COMP; - } - if(strcmp(argv[1], "-ttwo") == 0) { - algo = TWO_STAGE_COMP; - } - - if(strcmp(argv[1], "-sf") == 0) { - test_threadsafe(atoi(argv[2])); - return 0; - } - - if(strcmp(argv[1], "-sd") == 0) { - test_threadsafe_double(atoi(argv[2])); - return 0; - } - if(algo == 0){ - printf(" no param -tone -ttwo \n"); - return 0; - } - - bool ret = testFile(argv[2], algo); - printf(" test file %s. \n", ret ? "ok" : "err"); - return 1; - - } else if( argc == 2) { - if(strcmp(argv[1], "-mem") == 0) { - memTest(); - } - } - - //memTest(); - return 0; -} - // // txt to binary file // @@ -490,7 +447,11 @@ int memTest() { void* memTestThread(void* lparam) { //memTest(); printf(" enter thread ....\n"); - memTest(); + for(int i=0; i< 1000000; i++) + { + memTest(); + printf(" start i=%d .... \n", i); + } return NULL; } @@ -519,7 +480,11 @@ void test_threadsafe(int thread_count){ void* memTestThreadDouble(void* lparam) { //memTest(); printf(" enter thread ....\n"); - memTestDouble(); + for(int i=0; i< 50000; i++) + { + memTest(); + printf(" double start i=%d .... \n", i); + } return NULL; } @@ -542,4 +507,59 @@ void test_threadsafe_double(int thread_count){ printf("\n ---- double test thread safe end. not same count=%d-----\n", notsame_cnt); -} \ No newline at end of file +} + + + + + + +// +// ----------------- main ---------------------- +// +int main(int argc, char *argv[]) { + printf("welcome to use taospack tools v1.1. sizeof(STColumn) = %lu\n", sizeof(STColumn)); + + tsLossyInit(); + // + //tsCompressExit(); + //return 1; + + if(argc == 3){ + char algo = 0; + // t + if(strcmp(argv[1], "-tone") == 0 || strcmp(argv[1], "-t") == 0 ) { + algo = ONE_STAGE_COMP; + } + if(strcmp(argv[1], "-ttwo") == 0) { + algo = TWO_STAGE_COMP; + } + + if(strcmp(argv[1], "-sf") == 0) { + test_threadsafe(atoi(argv[2])); + return 0; + } + + if(strcmp(argv[1], "-sd") == 0) { + test_threadsafe_double(atoi(argv[2])); + return 0; + } + if(algo == 0){ + printf(" no param -tone -ttwo \n"); + return 0; + } + + bool ret = testFile(argv[2], algo); + printf(" test file %s. \n", ret ? "ok" : "err"); + return 1; + + } else if( argc == 2) { + if(strcmp(argv[1], "-mem") == 0) { + memTest(); + } + } + + //memTest(); + return 0; +} + diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 32748a5a37..8b2c81d567 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -68,7 +68,8 @@ STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { terrno = TSDB_CODE_SUCCESS; // Compress Init - tsCompressInit(); + //if(pCfg->compressLossy) + tsLossyInit(); // Check and set default configurations if (tsdbCheckAndSetDefaultCfg(&config) < 0) { @@ -143,9 +144,6 @@ int tsdbCloseRepo(STsdbRepo *repo, int toCommit) { tsdbFreeRepo(pRepo); tsdbDebug("vgId:%d repository is closed", vgId); - // compress exit - tsCompressExit(); - if (terrno != TSDB_CODE_SUCCESS) { return -1; } else { diff --git a/src/util/inc/tscompression.h b/src/util/inc/tscompression.h index 688b181ae9..63da5c9cc5 100644 --- a/src/util/inc/tscompression.h +++ b/src/util/inc/tscompression.h @@ -54,8 +54,9 @@ int tsCompressDoubleLossyImp(const char * input, const int nelements, const char int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, const char * output); // init -bool tsCompressInit(); -void tsCompressExit(); +bool tsLossyInit(); + + static FORCE_INLINE int tsCompressTinyint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, char *const buffer, int bufferSize) { diff --git a/src/util/src/tcompression.c b/src/util/src/tcompression.c index 07b534695f..300a06e9ac 100644 --- a/src/util/src/tcompression.c +++ b/src/util/src/tcompression.c @@ -891,18 +891,18 @@ int tsDecompressFloatImp(const char *const input, const int nelements, char *con // // ----------- global init and exit resource ------ // -int SZ_Init(const char *configFilePath); - - -bool tsCompressInit() { - SZ_Init("./sz.config"); +int SZ_Init(const char *configFilePath); //declare deps/sz/include/sz.h + +bool gLossyInited = false; +bool tsLossyInit() { + // init compress init + if(!gLossyInited){ + gLossyInited = true; + SZ_Init("./sz.config"); + } return true; } -void tsCompressExit(){ - -} - // // ---------- float double lossy ----------- // -- GitLab