From 2e03fd72477fb6e55858dce913d4bc482b854a3b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 1 Jun 2020 16:02:39 +0800 Subject: [PATCH] [td-225] support microsecond database options. --- src/common/inc/tglobal.h | 2 +- src/common/src/tglobal.c | 5 +++-- src/inc/taosdef.h | 13 ++++--------- src/inc/tsdb.h | 5 +++-- src/query/inc/qExecutor.h | 2 +- src/query/src/qExecutor.c | 11 +++++++---- src/tsdb/src/tsdbMain.c | 8 ++++++-- src/tsdb/src/tsdbRead.c | 8 +++++--- 8 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 0af8430ab1..64a99b1a16 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -49,7 +49,7 @@ extern int32_t tsTotalMemoryMB; extern int32_t tsVersion; extern int32_t tscEmbedded; -extern int64_t tsMsPerDay[2]; +extern int64_t tsMsPerDay[3]; extern char tsFirst[]; extern char tsSecond[]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 2b4e59e92a..bef069fb6e 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -55,11 +55,12 @@ int32_t tsEnableCoreFile = 0; int32_t tscEmbedded = 0; /* - * minmum scale for whole system, millisecond by default + * minimum scale for whole system, millisecond by default * for TSDB_TIME_PRECISION_MILLI: 86400000L * TSDB_TIME_PRECISION_MICRO: 86400000000L + * TSDB_TIME_PRECISION_NANO: 86400000000000L */ -int64_t tsMsPerDay[] = {86400000L, 86400000000L}; +int64_t tsMsPerDay[] = {86400000L, 86400000000L, 86400000000000L}; char tsFirst[TSDB_EP_LEN] = {0}; char tsSecond[TSDB_EP_LEN] = {0}; diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index def30ed8cb..bdf61208a3 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -91,6 +91,7 @@ extern const int32_t TYPE_BYTES[11]; #define TSDB_TIME_PRECISION_MILLI 0 #define TSDB_TIME_PRECISION_MICRO 1 +#define TSDB_TIME_PRECISION_NANO 2 #define TSDB_TIME_PRECISION_MILLI_STR "ms" #define TSDB_TIME_PRECISION_MICRO_STR "us" @@ -285,9 +286,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_COMMIT_TIME 40960 #define TSDB_DEFAULT_COMMIT_TIME 3600 -#define TSDB_MIN_PRECISION TSDB_PRECISION_MILLI -#define TSDB_MAX_PRECISION TSDB_PRECISION_NANO -#define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI +#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI +#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO +#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI #define TSDB_MIN_COMP_LEVEL 0 #define TSDB_MAX_COMP_LEVEL 2 @@ -356,12 +357,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TAOS_QTYPE_WAL 2 #define TAOS_QTYPE_CQ 3 -typedef enum { - TSDB_PRECISION_MILLI, - TSDB_PRECISION_MICRO, - TSDB_PRECISION_NANO -} EPrecisionType; - typedef enum { TSDB_SUPER_TABLE = 0, // super table TSDB_CHILD_TABLE = 1, // table created from super table diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 927b3d08f6..f291da4665 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -64,13 +64,14 @@ typedef struct { int8_t compression; } STsdbCfg; +typedef void TsdbRepoT; // use void to hide implementation details from outside + void tsdbSetDefaultCfg(STsdbCfg *pCfg); STsdbCfg *tsdbCreateDefaultCfg(); void tsdbFreeCfg(STsdbCfg *pCfg); +STsdbCfg *tsdbGetCfg(const TsdbRepoT *repo); // --------- TSDB REPOSITORY DEFINITION -typedef void TsdbRepoT; // use void to hide implementation details from outside - int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter); int32_t tsdbDropRepo(TsdbRepoT *repo); TsdbRepoT *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 09ac1b6cd7..722a625526 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -136,7 +136,7 @@ typedef struct SQuery { int64_t intervalTime; int64_t slidingTime; // sliding time for sliding window query char slidingTimeUnit; // interval data type, used for daytime revise - int8_t precision; + int16_t precision; int16_t numOfOutput; int16_t fillType; int16_t checkBuffer; // check if the buffer is full during scan each block diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index b487c790a9..51d166ccb4 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3964,10 +3964,12 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { } int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool isSTableQuery) { + int32_t code = TSDB_CODE_SUCCESS; + SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - int32_t code = TSDB_CODE_SUCCESS; + pQuery->precision = tsdbGetCfg(tsdb)->precision; setScanLimitationByResultBuffer(pQuery); changeExecuteScanOrder(pQuery, false); @@ -5422,7 +5424,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, pQuery->slidingTimeUnit = pQueryMsg->slidingTimeUnit; pQuery->fillType = pQueryMsg->fillType; pQuery->numOfTags = pQueryMsg->numOfTags; - + // todo do not allocate ?? pQuery->colList = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); if (pQuery->colList == NULL) { @@ -5491,7 +5493,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, int tableIndex = 0; STimeWindow window = pQueryMsg->window; - taosArraySort( pTableIdList, compareTableIdInfo ); + taosArraySort(pTableIdList, compareTableIdInfo); for(int32_t i = 0; i < numOfGroups; ++i) { SArray* pa = taosArrayGetP(groupInfo->pGroupList, i); size_t s = taosArrayGetSize(pa); @@ -5960,7 +5962,8 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co (*pRsp)->offset = 0; (*pRsp)->useconds = 0; } - + + (*pRsp)->precision = htons(pQuery->precision); if (pQuery->rec.rows > 0 && code == TSDB_CODE_SUCCESS) { code = doDumpQueryResult(pQInfo, (*pRsp)->data); } else { diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 9c8e57d18a..197718e154 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -9,8 +9,7 @@ #include "ttime.h" #include -#define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI // default precision -#define IS_VALID_PRECISION(precision) (((precision) >= TSDB_PRECISION_MILLI) && ((precision) <= TSDB_PRECISION_NANO)) +#define IS_VALID_PRECISION(precision) (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO)) #define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP #define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) #define TSDB_MIN_ID 0 @@ -79,6 +78,11 @@ void tsdbFreeCfg(STsdbCfg *pCfg) { if (pCfg != NULL) free(pCfg); } +STsdbCfg *tsdbGetCfg(const TsdbRepoT *repo) { + assert(repo != NULL); + return &((STsdbRepo*)repo)->config; +} + /** * Create a new TSDB repository * @param rootDir the TSDB repository root directory diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 08804f6ba1..1e03794045 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -364,12 +364,13 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { return true; } -static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile) { +static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precision) { + assert(precision >= TSDB_TIME_PRECISION_MICRO || precision <= TSDB_TIME_PRECISION_NANO); if (key == TSKEY_INITIAL_VAL) { return INT32_MIN; } - int64_t fid = (int64_t)(key / (daysPerFile * tsMsPerDay[0])); // set the starting fileId + int64_t fid = (int64_t)(key / (daysPerFile * tsMsPerDay[precision])); // set the starting fileId if (fid < 0L && llabs(fid) > INT32_MAX) { // data value overflow for INT32 fid = INT32_MIN; } @@ -1297,7 +1298,8 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { // find the start data block in file if (!pQueryHandle->locateStart) { pQueryHandle->locateStart = true; - int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pQueryHandle->pTsdb->config.daysPerFile); + STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; + int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order); tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); -- GitLab