From bb28e3a5f8c6c696c705516b35ee400ff439f43b Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 19 Aug 2021 23:43:02 +0800 Subject: [PATCH] [TD-6233]: make column compression threshold user configurable --- src/client/src/tscServer.c | 7 +++++-- src/common/src/tglobal.c | 19 +++++++++---------- src/query/inc/qExecutor.h | 4 +--- src/query/src/queryMain.c | 8 ++++++-- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 29c0eb693f..783c7a3dbe 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2678,7 +2678,7 @@ int tscProcessQueryRsp(SSqlObj *pSql) { return 0; } -static void decompressQueryColData(SSqlRes *pRes, SQueryInfo* pQueryInfo, char **data, int8_t compressed, int compLen) { +static void decompressQueryColData(SSqlObj *pSql, SSqlRes *pRes, SQueryInfo* pQueryInfo, char **data, int8_t compressed, int32_t compLen) { int32_t decompLen = 0; int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; int32_t *compSizes; @@ -2715,6 +2715,9 @@ static void decompressQueryColData(SSqlRes *pRes, SQueryInfo* pQueryInfo, char * pData = *data + compLen + numOfCols * sizeof(int32_t); } + tscDebug("0x%"PRIx64" decompress col data, compressed size:%d, decompressed size:%d", + pSql->self, (int32_t)(compLen + numOfCols * sizeof(int32_t)), decompLen); + int32_t tailLen = pRes->rspLen - sizeof(SRetrieveTableRsp) - decompLen; memmove(*data + decompLen, pData, tailLen); memmove(*data, outputBuf, decompLen); @@ -2749,7 +2752,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { //Decompress col data if compressed from server if (pRetrieve->compressed) { int32_t compLen = htonl(pRetrieve->compLen); - decompressQueryColData(pRes, pQueryInfo, &pRes->data, pRetrieve->compressed, compLen); + decompressQueryColData(pSql, pRes, pQueryInfo, &pRes->data, pRetrieve->compressed, compLen); } STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index b7930dd43e..3c84f16d4f 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -76,12 +76,11 @@ int32_t tsMaxBinaryDisplayWidth = 30; int32_t tsCompressMsgSize = -1; /* denote if server needs to compress the retrieved column data before adding to the rpc response message body. - * 0: disable column data compression - * 1: enable column data compression - * This option is default to disabled. Once enabled, compression will be conducted if any column has size more - * than QUERY_COMP_THRESHOLD. Otherwise, no further compression is needed. + * 0: all data are compressed + * -1: all data are not compressed + * other values: if the any retrieved column size is greater than the tsCompressColData, all data will be compressed. */ -int32_t tsCompressColData = 0; +int32_t tsCompressColData = -1; // client int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN; @@ -95,7 +94,7 @@ int32_t tsMaxNumOfOrderedResults = 100000; // 10 ms for sliding time, the value will changed in case of time precision changed int32_t tsMinSlidingTime = 10; -// the maxinum number of distict query result +// the maxinum number of distict query result int32_t tsMaxNumOfDistinctResults = 1000 * 10000; // 1 us for interval time range, changed accordingly @@ -1006,10 +1005,10 @@ static void doInitGlobalConfig(void) { cfg.option = "compressColData"; cfg.ptr = &tsCompressColData; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = -1; + cfg.maxValue = 100000000.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 6e8eec2456..b54bead94a 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -43,9 +43,7 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int #define GET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL? 0:((_r)->outputBuf)->info.rows) -//TODO: may need to fine tune this threshold -#define QUERY_COMP_THRESHOLD (1024 * 512) -#define NEEDTO_COMPRESS_QUERY(size) ((size) > QUERY_COMP_THRESHOLD ? 1 : 0) +#define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData? 1 : 0) enum { // when query starts to execute, this status will set diff --git a/src/query/src/queryMain.c b/src/query/src/queryMain.c index d25f5eab7a..64e3e67843 100644 --- a/src/query/src/queryMain.c +++ b/src/query/src/queryMain.c @@ -357,7 +357,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co } (*pRsp)->precision = htons(pQueryAttr->precision); - (*pRsp)->compressed = (int8_t)(tsCompressColData && checkNeedToCompressQueryCol(pQInfo)); + (*pRsp)->compressed = (int8_t)((tsCompressColData != -1) && checkNeedToCompressQueryCol(pQInfo)); if (GET_NUM_OF_RESULTS(&(pQInfo->runtimeEnv)) > 0 && pQInfo->code == TSDB_CODE_SUCCESS) { doDumpQueryResult(pQInfo, (*pRsp)->data, (*pRsp)->compressed, &compLen); @@ -367,8 +367,12 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co if ((*pRsp)->compressed && compLen != 0) { int32_t numOfCols = pQueryAttr->pExpr2 ? pQueryAttr->numOfExpr2 : pQueryAttr->numOfOutput; - *contLen = *contLen - pQueryAttr->resultRowSize * s + compLen + numOfCols * sizeof(int32_t); + int32_t origSize = pQueryAttr->resultRowSize * s; + int32_t compSize = compLen + numOfCols * sizeof(int32_t); + *contLen = *contLen - origSize + compSize; *pRsp = (SRetrieveTableRsp *)rpcReallocCont(*pRsp, *contLen); + qDebug("QInfo:0x%"PRIx64" compress col data, uncompressed size:%d, compressed size:%d, ratio:%.2f\n", + pQInfo->qId, origSize, compSize, (float)origSize / (float)compSize); } (*pRsp)->compLen = htonl(compLen); -- GitLab