diff --git a/deps/SZ/sz/include/conf.h b/deps/SZ/sz/include/conf.h index 3f33760162f0a5a23ca82d46e5256cb571120bc4..b3a1341f4c6e2fbfdc3124d89bd2bdb6d881df4e 100644 --- a/deps/SZ/sz/include/conf.h +++ b/deps/SZ/sz/include/conf.h @@ -16,6 +16,11 @@ extern "C" { #include +// +// set default value +// +void setDefaulParams(sz_exedata* exedata, sz_params* params); + //conf.c void updateQuantizationInfo(int quant_intervals); int SZ_ReadConf(const char* sz_cfgFile); diff --git a/deps/SZ/sz/include/pub.h b/deps/SZ/sz/include/pub.h index c30921c5f5f940f1d664b5e13b3a966313bb07e9..8e84a9c243fee02ef91371a6979f07c9a543d462 100644 --- a/deps/SZ/sz/include/pub.h +++ b/deps/SZ/sz/include/pub.h @@ -33,7 +33,8 @@ typedef struct sz_params int szMode; //* 0 (best speed) or 1 (better compression with Zstd/Gzip) or 3 temporal-dimension based compression int gzipMode; //* four options: Z_NO_COMPRESSION, or Z_BEST_SPEED, Z_BEST_COMPRESSION, Z_DEFAULT_COMPRESSION int errorBoundMode; //4bits (0.5byte), //SZ_ABS, REL, ABS_AND_REL, or ABS_OR_REL, PSNR, or PW_REL, PSNR - double absErrBound; //absolute error bound + double absErrBound; //absolute error bound for float + double absErrBoundDouble; // for double double relBoundRatio; //value range based relative error bound ratio double psnr; //PSNR double normErr; diff --git a/deps/SZ/sz/include/sz.h b/deps/SZ/sz/include/sz.h index c8ef9be7f67212849aa35be15fa9318b0ee5c277..d3ffdbf3668b7c282d8f0cf8c22d40d351a3b5a3 100644 --- a/deps/SZ/sz/include/sz.h +++ b/deps/SZ/sz/include/sz.h @@ -161,11 +161,9 @@ extern int dataEndianType; //*endian type of the data read from disk extern int sysEndianType; //*sysEndianType is actually set automatically. extern sz_params *confparams_cpr; -extern sz_params *confparams_dec; extern sz_exedata *exe_params; - void SZ_Finalize(); int SZ_Init(const char *configFilePath); int SZ_Init_Params(sz_params *params); diff --git a/deps/SZ/sz/include/td_sz.h b/deps/SZ/sz/include/td_sz.h index 24e6847ec6eba7d6b5abd89d2f27b85a0297b452..d533365a6795924aece0871e592e2fd7c1c3895d 100644 --- a/deps/SZ/sz/include/td_sz.h +++ b/deps/SZ/sz/include/td_sz.h @@ -3,12 +3,19 @@ #define _TD_SZ_H #include "defines.h" + #ifdef __cplusplus extern "C" { #endif void cost_start(); double cost_end(const char* tag); + +// +// Init success return 1 else 0 +// +void tdszInit(double fPrecision, double dPrecision, unsigned int maxIntervals, unsigned int intervals, const char* compressor); + // // compress interface to tdengine return value is count of output with bytes // @@ -19,6 +26,11 @@ int tdszCompress(int type, const char * input, const int nelements, const char * // int tdszDecompress(int type, const char * input, int compressedSize, const int nelements, const char * output); +// +// tdszExit +// +void tdszExit(); + #ifdef __cplusplus } #endif diff --git a/deps/SZ/sz/src/conf.c b/deps/SZ/sz/src/conf.c index ee3bae88d56bc256738cde1504135ac442eb6fe7..732cf58d6d3246740b845025d2005a13620cd046 100644 --- a/deps/SZ/sz/src/conf.c +++ b/deps/SZ/sz/src/conf.c @@ -12,25 +12,67 @@ #include "sz.h" #include "iniparser.h" #include "Huffman.h" -#include "pastri.h" -/*-------------------------------------------------------------------------*/ -/** - @brief It reads the configuration given in the configuration file. - @return integer 1 if successfull. +// +// set default value +// +void setDefaulParams(sz_exedata* exedata, sz_params* params) +{ + // sz_params + if(params) + { + // first important + params->errorBoundMode = SZ_ABS; + params->absErrBound = 1E-8; + params->absErrBoundDouble = 1E-16; + params->max_quant_intervals = 800; + params->quantization_intervals = 500; + params->losslessCompressor = ZSTD_COMPRESSOR; //other option: GZIP_COMPRESSOR; - This function reads the configuration given in the SZ configuration - file and sets other required parameters. + // second important + params->sol_ID = SZ; + params->maxRangeRadius = params->max_quant_intervals/2; + params->predThreshold = 0.99; + params->sampleDistance = 100; + params->szMode = SZ_BEST_COMPRESSION; + if(params->losslessCompressor==ZSTD_COMPRESSOR) + params->gzipMode = 3; //fast mode + else + params->gzipMode = 1; //high speed mode + + // other + params->psnr = 90; + params->relBoundRatio = 1E-8; + params->accelerate_pw_rel_compression = 1; + params->pw_relBoundRatio = 1E-3; + params->segment_size = 36; + params->pwr_type = SZ_PWR_MIN_TYPE; + params->snapshotCmprStep = 5; + params->withRegression = SZ_WITH_LINEAR_REGRESSION; + params->randomAccess = 0; //0: no random access , 1: support random access + params->protectValueRange = 0; + params->plus_bits = 3; + } + + // sz_exedata + if(exedata) + { + exedata->optQuantMode = 1; + exedata->SZ_SIZE_TYPE = 4; + if(params) + { + exedata->intvCapacity = params->maxRangeRadius*2; + exedata->intvRadius = params->maxRangeRadius; + } + else + { + exedata->intvCapacity = 500; + exedata->intvRadius = 200; + } + } + +} - **/ - -/*struct node_t *pool; -node *qqq; -node *qq; -int n_nodes = 0, qend; -unsigned long **code; -unsigned char *cout; -int n_inode;*/ unsigned int roundUpToPowerOf2(unsigned int base) { @@ -74,8 +116,10 @@ double computeABSErrBoundFromNORM_ERR(double normErr, size_t nbEle) int SZ_ReadConf(const char* sz_cfgFile) { // Check access to SZ configuration file and load dictionary //record the setting in confparams_cpr - confparams_cpr = (sz_params*)malloc(sizeof(sz_params)); - exe_params = (sz_exedata*)malloc(sizeof(sz_exedata)); + if(confparams_cpr == NULL) + confparams_cpr = (sz_params*)malloc(sizeof(sz_params)); + if(exe_params == NULL) + exe_params = (sz_exedata*)malloc(sizeof(sz_exedata)); int x = 1; char sol_name[256]; @@ -92,49 +136,12 @@ int SZ_ReadConf(const char* sz_cfgFile) { else //=0 sysEndianType = BIG_ENDIAN_SYSTEM; - confparams_cpr->plus_bits = 3; // default option if(sz_cfgFile == NULL || access(sz_cfgFile, F_OK) != 0) { dataEndianType = LITTLE_ENDIAN_DATA; - confparams_cpr->sol_ID = SZ; - confparams_cpr->max_quant_intervals = 800; - confparams_cpr->maxRangeRadius = confparams_cpr->max_quant_intervals/2; - - exe_params->intvCapacity = confparams_cpr->maxRangeRadius*2; - exe_params->intvRadius = confparams_cpr->maxRangeRadius; - - confparams_cpr->quantization_intervals = 500; - exe_params->optQuantMode = 1; - confparams_cpr->predThreshold = 0.99; - confparams_cpr->sampleDistance = 100; - - confparams_cpr->szMode = SZ_BEST_COMPRESSION; - confparams_cpr->losslessCompressor = ZSTD_COMPRESSOR; //other option: GZIP_COMPRESSOR; - if(confparams_cpr->losslessCompressor==ZSTD_COMPRESSOR) - confparams_cpr->gzipMode = 3; //fast mode - else - confparams_cpr->gzipMode = 1; //high speed mode - - confparams_cpr->errorBoundMode = SZ_ABS; - confparams_cpr->psnr = 90; - confparams_cpr->absErrBound = 1E-8; - confparams_cpr->relBoundRatio = 1E-8; - confparams_cpr->accelerate_pw_rel_compression = 1; - - confparams_cpr->pw_relBoundRatio = 1E-3; - confparams_cpr->segment_size = 36; - - confparams_cpr->pwr_type = SZ_PWR_MIN_TYPE; - - confparams_cpr->snapshotCmprStep = 5; - - confparams_cpr->withRegression = SZ_WITH_LINEAR_REGRESSION; - - confparams_cpr->randomAccess = 0; //0: no random access , 1: support random access - - confparams_cpr->protectValueRange = 0; + setDefaulParams(exe_params, confparams_cpr); return SZ_SUCCESS; } diff --git a/deps/SZ/sz/src/dataCompression.c b/deps/SZ/sz/src/dataCompression.c index 6b6316ae5c5c1394fb11e2f4bac1d1d4b67c9b0d..237a3315c1b56435154cee3efd9f01946b6919e3 100644 --- a/deps/SZ/sz/src/dataCompression.c +++ b/deps/SZ/sz/src/dataCompression.c @@ -95,7 +95,7 @@ double getRealPrecision_double(double valueRangeSize, int errBoundMode, double a int state = SZ_SUCCESS; double precision = 0; if(errBoundMode==SZ_ABS||errBoundMode==ABS_OR_PW_REL||errBoundMode==ABS_AND_PW_REL) - precision = absErrBound*0.00000001; + precision = absErrBound; else if(errBoundMode==REL||errBoundMode==REL_OR_PW_REL||errBoundMode==REL_AND_PW_REL) precision = relBoundRatio*valueRangeSize; else if(errBoundMode==ABS_AND_REL) diff --git a/deps/SZ/sz/src/sz.c b/deps/SZ/sz/src/sz.c index 222775df628fb925053076dd31b3e0222135e48f..442bf8ad55653732874e8fedfd177b957cca81bc 100644 --- a/deps/SZ/sz/src/sz.c +++ b/deps/SZ/sz/src/sz.c @@ -31,13 +31,8 @@ int sysEndianType = LITTLE_ENDIAN_SYSTEM ; //*sysEndianType is actually set aut //the confparams should be separate between compression and decopmression, in case of mutual-affection when calling compression/decompression alternatively sz_params *confparams_cpr = NULL; //used for compression -sz_params *confparams_dec = NULL; //used for decompression - sz_exedata *exe_params = NULL; - - - int SZ_Init(const char *configFilePath) { // check CPU EndianType @@ -152,14 +147,8 @@ size_t SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size return outSize; } - void SZ_Finalize() { - if(confparams_dec!=NULL) - { - free(confparams_dec); - confparams_dec = NULL; - } if(confparams_cpr!=NULL) { free(confparams_cpr); diff --git a/deps/SZ/sz/src/sz_double.c b/deps/SZ/sz/src/sz_double.c index 9bba10cf64142c36e5b631f414f80fc35e3a4ca7..1413c9ce306920cb4afd1875a94ad0cd24ec38c0 100644 --- a/deps/SZ/sz/src/sz_double.c +++ b/deps/SZ/sz/src/sz_double.c @@ -325,18 +325,18 @@ int SZ_compress_args_double(double *oriData, size_t r1, unsigned char* newByteDa if(params->errorBoundMode==PSNR) { params->errorBoundMode = SZ_ABS; - realPrecision = params->absErrBound = computeABSErrBoundFromPSNR(params->psnr, (double)params->predThreshold, valueRangeSize); + realPrecision = params->absErrBoundDouble = computeABSErrBoundFromPSNR(params->psnr, (double)params->predThreshold, valueRangeSize); } else if(params->errorBoundMode==NORM) //norm error = sqrt(sum((xi-xi_)^2)) { params->errorBoundMode = SZ_ABS; - realPrecision = params->absErrBound = computeABSErrBoundFromNORM_ERR(params->normErr, dataLength); + realPrecision = params->absErrBoundDouble = computeABSErrBoundFromNORM_ERR(params->normErr, dataLength); //printf("realPrecision=%lf\n", realPrecision); } else { - realPrecision = getRealPrecision_double(valueRangeSize, params->errorBoundMode, params->absErrBound, params->relBoundRatio, &status); - params->absErrBound = realPrecision; + realPrecision = getRealPrecision_double(valueRangeSize, params->errorBoundMode, params->absErrBoundDouble, params->relBoundRatio, &status); + params->absErrBoundDouble = realPrecision; } if(valueRangeSize <= realPrecision) { diff --git a/deps/SZ/sz/src/td_sz.c b/deps/SZ/sz/src/td_sz.c index fa6e83486c19237a3026a4d7004056d1757f25f8..c04c4aed6f43d9f4b2aa62465cb792c85a1c03c6 100644 --- a/deps/SZ/sz/src/td_sz.c +++ b/deps/SZ/sz/src/td_sz.c @@ -5,7 +5,34 @@ #include "td_sz.h" #include "sz.h" +#include "conf.h" +// +// Init success return 1 else 0 +// +void tdszInit(double fPrecision, double dPrecision, unsigned int maxIntervals, unsigned int intervals, const char* compressor) +{ + // need malloc + if(confparams_cpr == NULL) + confparams_cpr = (sz_params*)malloc(sizeof(sz_params)); + if(exe_params == NULL) + exe_params = (sz_exedata*)malloc(sizeof(sz_exedata)); + + // set default + setDefaulParams(exe_params, confparams_cpr); + + // overwrite with args + confparams_cpr->absErrBound = fPrecision; + confparams_cpr->absErrBoundDouble = dPrecision; + confparams_cpr->max_quant_intervals = maxIntervals; + confparams_cpr->quantization_intervals = intervals; + if(strcmp(compressor, "GZIP_COMPRESSOR")==0) + confparams_cpr->losslessCompressor = GZIP_COMPRESSOR; + else if(strcmp(compressor, "ZSTD_COMPRESSOR")==0) + confparams_cpr->losslessCompressor = ZSTD_COMPRESSOR; + + return ; +} // @@ -28,3 +55,20 @@ int tdszDecompress(int type, const char * input, int compressedSize, const int n size_t outSize = SZ_decompress(type, (void*)input, compressedSize, (size_t)nelements, (unsigned char*)output); return (int)outSize; } + +// +// tdszExit +// +void tdszExit() +{ + if(confparams_cpr!=NULL) + { + free(confparams_cpr); + confparams_cpr = NULL; + } + if(exe_params!=NULL) + { + free(exe_params); + exe_params = NULL; + } +} \ No newline at end of file diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index e35e805efefb7c017dd5c0c828023ecbe6e0c460..53b47eccbe7148b38c8732054efb88eee86ae436 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -205,6 +205,14 @@ extern int32_t wDebugFlag; extern int32_t cqDebugFlag; extern int32_t debugFlag; +// lossy +extern char lossyColumns[]; +extern double fPrecision; +extern double dPrecision; +extern uint32_t maxIntervals; +extern uint32_t intervals; +extern char Compressor[]; + typedef struct { char dir[TSDB_FILENAME_LEN]; int level; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index ec98915227e34223211341971f226fda64dc0504..f3a09b213d2e28e6d6115ca79e043901c96e0531 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -244,6 +244,18 @@ int32_t tsdbDebugFlag = 131; int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; +// +// lossy compress 6 +// +char lossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty can close lossy compress. +// below option can take effect when tsLossyColumns not empty +double fPrecision = 1E-8; // float column precision +double dPrecision = 1E-16; // double column precision +uint32_t maxIntervals = 800; // max intervals +uint32_t intervals = 500; // intervals +char Compressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR + + int32_t (*monStartSystemFp)() = NULL; void (*monStopSystemFp)() = NULL; void (*monExecuteSQLFp)(char *sql) = NULL; @@ -1517,6 +1529,68 @@ static void doInitGlobalConfig(void) { cfg.ptrLength = tListLen(tsTempDir); cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + + // lossy compress + cfg.option = "lossyColumns"; + cfg.ptr = lossyColumns; + cfg.valType = TAOS_CFG_VTYPE_STRING; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; + cfg.minValue = 0; + cfg.maxValue = 0; + cfg.ptrLength = tListLen(lossyColumns); + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "Compressor"; + cfg.ptr = Compressor; + cfg.valType = TAOS_CFG_VTYPE_STRING; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; + cfg.minValue = 0; + cfg.maxValue = 0; + cfg.ptrLength = tListLen(Compressor); + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "fPrecision"; + cfg.ptr = &fPrecision; + cfg.valType = TAOS_CFG_VTYPE_DOUBLE; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; + cfg.minValue = -MAXFLOAT; + cfg.maxValue = MAXFLOAT; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "dPrecision"; + cfg.ptr = &dPrecision; + cfg.valType = TAOS_CFG_VTYPE_DOUBLE; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; + cfg.minValue = -MAXFLOAT; + cfg.maxValue = MAXFLOAT; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "maxIntervals"; + cfg.ptr = &maxIntervals; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; + cfg.minValue = 0; + cfg.maxValue = 65536; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + cfg.option = "intervals"; + cfg.ptr = &intervals; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; + cfg.minValue = 0; + cfg.maxValue = 65536; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + } void taosInitGlobalCfg() { diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 61b11579bf8da901697df3202c872caf7d6c7371..eae332d2b4b3ea9503358dcf828f7a258ddeedc1 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -41,6 +41,7 @@ #include "dnodeTelemetry.h" #include "module.h" #include "mnode.h" +#include "tscompression.h" #if !defined(_MODULE) || !defined(_TD_LINUX) int32_t moduleStart() { return 0; } @@ -234,6 +235,10 @@ static void dnodeCheckDataDirOpenned(char *dir) { } static int32_t dnodeInitStorage() { + // compress module init + tsCompressInit(); + + // storage module init if (tsDiskCfgNum == 1 && dnodeCreateDir(tsDataDir) < 0) { dError("failed to create dir: %s, reason: %s", tsDataDir, strerror(errno)); return -1; @@ -309,7 +314,13 @@ static int32_t dnodeInitStorage() { return 0; } -static void dnodeCleanupStorage() { tfsDestroy(); } +static void dnodeCleanupStorage() { + // storage destroy + tfsDestroy(); + + // compress destroy + tsCompressExit(); +} bool dnodeIsFirstDeploy() { return strcmp(tsFirst, tsLocalEp) == 0; diff --git a/src/kit/taospack/taospack.c b/src/kit/taospack/taospack.c index 34f79961d1d2ae4d8caa9ca88c04a91a84539559..850417522ec0e6726cfec69f7251b590646808cb 100644 --- a/src/kit/taospack/taospack.c +++ b/src/kit/taospack/taospack.c @@ -142,7 +142,6 @@ double check_same_double(double* ft1, double* ft2, int count){ // // test compress and decompress // -extern bool gOpenLossy; bool DoDouble(double* doubles, int cnt, int algorithm) { // compress @@ -677,7 +676,8 @@ void test_same_double(int algo){ } - +extern bool lossyFloat; +extern bool lossyDouble; // // ----------------- main ---------------------- @@ -685,8 +685,9 @@ void test_same_double(int algo){ int main(int argc, char *argv[]) { printf("welcome to use taospack tools v1.3\n"); - gOpenLossy = false; - tsLossyInit(); + tsCompressInit(); + lossyFloat = lossyDouble = true; + // //tsCompressExit(); //return 1; @@ -697,11 +698,11 @@ int main(int argc, char *argv[]) { // t if(strcmp(argv[1], "-tone") == 0 || strcmp(argv[1], "-t") == 0 ) { algo = ONE_STAGE_COMP; - gOpenLossy = true; + lossyFloat = lossyDouble = true; } if(strcmp(argv[1], "-tw") == 0) { algo = TWO_STAGE_COMP; - gOpenLossy = false; + lossyFloat = lossyDouble = false; } if(strcmp(argv[1], "-sf") == 0) { @@ -737,7 +738,7 @@ int main(int argc, char *argv[]) { unitTestFloat(); } - //memTest(); + tsCompressExit(); return 0; } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index bfbe2c10daad5f64142272ecaf3356eb52d5bc6a..1308f9b85a2dacc523d5e5d45b3bccf2d206e6de 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -16,6 +16,7 @@ // no test file errors here #include "tsdbint.h" #include "tscompression.h" +#include "tsdbLog.h" #define IS_VALID_PRECISION(precision) \ (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO)) @@ -67,10 +68,6 @@ STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { terrno = TSDB_CODE_SUCCESS; - // Compress Init - //if(pCfg->compressLossy) - tsLossyInit(); - // Check and set default configurations if (tsdbCheckAndSetDefaultCfg(&config) < 0) { tsdbError("vgId:%d failed to open TSDB repository since %s", config.tsdbId, tstrerror(terrno)); diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index fdb2595fd8d3b1659800ca3a5b88c32b6fe95e93..f8695cb016e09096220e07ce9f5c3ae084e21993 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -20,7 +20,7 @@ extern "C" { #endif -#define TSDB_CFG_MAX_NUM 110 +#define TSDB_CFG_MAX_NUM 116 // 110 + 6 with lossy option #define TSDB_CFG_PRINT_LEN 23 #define TSDB_CFG_OPTION_LEN 24 #define TSDB_CFG_VALUE_LEN 41 @@ -32,6 +32,8 @@ extern "C" { #define TSDB_CFG_CTYPE_B_OPTION 16U // can be configured by taos_options function #define TSDB_CFG_CTYPE_B_NOT_PRINT 32U // such as password +#define MAXFLOAT 0x1.fffffep+127f + enum { TAOS_CFG_CSTATUS_NONE, // not configured TAOS_CFG_CSTATUS_DEFAULT, // use system default value @@ -50,6 +52,7 @@ enum { TAOS_CFG_VTYPE_IPSTR, TAOS_CFG_VTYPE_DIRECTORY, TAOS_CFG_VTYPE_DATA_DIRCTORY, + TAOS_CFG_VTYPE_DOUBLE, }; enum { diff --git a/src/util/inc/tscompression.h b/src/util/inc/tscompression.h index 6f963bc3e1d6d78cb8139135de98be4776d88312..79b629007cb1e129153027272022c4cdc28c6094 100644 --- a/src/util/inc/tscompression.h +++ b/src/util/inc/tscompression.h @@ -23,7 +23,6 @@ extern "C" { #include "taosdef.h" #include "tutil.h" - #define COMP_OVERFLOW_BYTES 2 #define BITS_PER_BYTE 8 // Masks @@ -51,9 +50,6 @@ extern "C" { #define HEAD_MODE(x) x%2 #define HEAD_ALGO(x) x/2 - -extern bool gOpenLossy; - extern int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type); extern int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type); extern int tsCompressBoolImp(const char *const input, const int nelements, char *const output); @@ -67,13 +63,17 @@ extern int tsDecompressDoubleImp(const char *const input, const int nelements, c extern int tsCompressFloatImp(const char *const input, const int nelements, char *const output); extern int tsDecompressFloatImp(const char *const input, const int nelements, char *const output); // lossy -int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output); -int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output); -int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output); -int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output); - -// init -bool tsLossyInit(); +extern int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output); +extern int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output); +extern int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output); +extern int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output); + +extern bool lossyFloat; +extern bool lossyDouble; +// init call +int tsCompressInit(); +// exit call +void tsCompressExit(); void cost_start(); double cost_end(const char* tag); @@ -223,7 +223,7 @@ static FORCE_INLINE int tsDecompressString(const char *const input, int compress static FORCE_INLINE int tsCompressFloat(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, char *const buffer, int bufferSize) { // lossy mode - if(gOpenLossy) { + if(lossyFloat) { return tsCompressFloatLossyImp(input, nelements, output); // lossless mode } else { @@ -269,7 +269,7 @@ static FORCE_INLINE int tsDecompressFloat(const char *const input, int compresse static FORCE_INLINE int tsCompressDouble(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, char *const buffer, int bufferSize) { - if(gOpenLossy){ + if(lossyDouble){ // lossy mode return tsCompressDoubleLossyImp(input, nelements, output); } else { diff --git a/src/util/src/tcompression.c b/src/util/src/tcompression.c index fb96c55268071e988ba61032430634d899a1fdef..a8c9bc535c8b586a372c2f3965a9b790c9fe2ae8 100644 --- a/src/util/src/tcompression.c +++ b/src/util/src/tcompression.c @@ -53,6 +53,8 @@ #include "taosdef.h" #include "tscompression.h" #include "tulog.h" +#include "tglobal.h" + static const int TEST_NUMBER = 1; #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) @@ -62,7 +64,33 @@ static const int TEST_NUMBER = 1; #define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode #define ZIGZAG_DECODE(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode -bool gOpenLossy = true; +bool lossyFloat = false; +bool lossyDouble = false; + +// init call +int tsCompressInit(){ + // config + if(lossyColumns[0] == 0){ + lossyFloat = false; + lossyDouble = false; + return 0; + } + + lossyFloat = strstr(lossyColumns, "float") != NULL; + lossyDouble = strstr(lossyColumns, "double") != NULL; + + if(lossyFloat == false && lossyDouble == false) + return 0; + + tdszInit(fPrecision, dPrecision, maxIntervals, intervals, Compressor); + uInfo("lossy compression is opened. columns = %s \n", lossyColumns); + return 1; +} +// exit call +void tsCompressExit(){ + tdszExit(); +} + /* * Compress Integer (Simple8B). @@ -891,20 +919,6 @@ int tsDecompressFloatImp(const char *const input, const int nelements, char *con return nelements * FLOAT_BYTES; } -// -// ----------- global init and exit resource ------ -// -int SZ_Init(const char *configFilePath); //declare deps/sz/include/sz.h - -bool gLossyInited = false; -bool tsLossyInit() { - // init compress init - if(!gLossyInited){ - gLossyInited = true; - SZ_Init("./sz.config"); - } - return true; -} // // ---------- float double lossy ----------- diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 80071986d6d2396ef2aec8f7841b0897cb3d7b26..ea83debfdc2cba27e1f78d1ae7e7a7c29dbd6aec 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -61,6 +61,24 @@ static void taosReadFloatConfig(SGlobalCfg *cfg, char *input_value) { } } +static void taosReadDoubleConfig(SGlobalCfg *cfg, char *input_value) { + double value = atof(input_value); + double *option = (double *)cfg->ptr; + if (value < cfg->minValue || value > cfg->maxValue) { + uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f", + cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); + } else { + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { + *option = value; + cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; + } else { + uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value, + tsCfgStatusStr[cfg->cfgStatus], *option); + } + } +} + + static void taosReadInt32Config(SGlobalCfg *cfg, char *input_value) { int32_t value = atoi(input_value); int32_t *option = (int32_t *)cfg->ptr; @@ -262,6 +280,9 @@ static void taosReadConfigOption(const char *option, char *value, char *value2, case TAOS_CFG_VTYPE_FLOAT: taosReadFloatConfig(cfg, value); break; + case TAOS_CFG_VTYPE_DOUBLE: + taosReadDoubleConfig(cfg, value); + break; case TAOS_CFG_VTYPE_STRING: taosReadStringConfig(cfg, value); break;