Merge pull request #2097 from taosdata/feature/query

[td-225] fix variable overflow bug.

src/common/inc/tglobal.h

@@ -49,7 +49,7 @@ extern int32_t tsTotalMemoryMB;
 extern int32_t tsVersion;
 
 extern int32_t tscEmbedded;
-extern int64_t tsMsPerDay[2];
+extern int64_t tsMsPerDay[3];
 
 extern char  tsFirst[];
 extern char  tsSecond[];

src/common/src/tglobal.c

@@ -55,11 +55,12 @@ int32_t tsEnableCoreFile = 0;
 int32_t tscEmbedded = 0;
 
 /*
- * minmum scale for whole system, millisecond by default
+ * minimum scale for whole system, millisecond by default
  * for TSDB_TIME_PRECISION_MILLI: 86400000L
  *     TSDB_TIME_PRECISION_MICRO: 86400000000L
+ *     TSDB_TIME_PRECISION_NANO:  86400000000000L
  */
-int64_t tsMsPerDay[] = {86400000L, 86400000000L};
+int64_t tsMsPerDay[] = {86400000L, 86400000000L, 86400000000000L};
 
 char  tsFirst[TSDB_EP_LEN] = {0};  
 char  tsSecond[TSDB_EP_LEN] = {0};

src/inc/taosdef.h

@@ -91,6 +91,7 @@ extern const int32_t TYPE_BYTES[11];
 
 #define TSDB_TIME_PRECISION_MILLI 0
 #define TSDB_TIME_PRECISION_MICRO 1
+#define TSDB_TIME_PRECISION_NANO  2
 
 #define TSDB_TIME_PRECISION_MILLI_STR "ms"
 #define TSDB_TIME_PRECISION_MICRO_STR "us"
@@ -285,9 +286,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_MAX_COMMIT_TIME            40960
 #define TSDB_DEFAULT_COMMIT_TIME        3600
 
-#define TSDB_MIN_PRECISION              TSDB_PRECISION_MILLI
-#define TSDB_MAX_PRECISION              TSDB_PRECISION_NANO
-#define TSDB_DEFAULT_PRECISION          TSDB_PRECISION_MILLI
+#define TSDB_MIN_PRECISION              TSDB_TIME_PRECISION_MILLI
+#define TSDB_MAX_PRECISION              TSDB_TIME_PRECISION_NANO
+#define TSDB_DEFAULT_PRECISION          TSDB_TIME_PRECISION_MILLI
 
 #define TSDB_MIN_COMP_LEVEL             0
 #define TSDB_MAX_COMP_LEVEL             2
@@ -356,12 +357,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TAOS_QTYPE_WAL      2 
 #define TAOS_QTYPE_CQ       3
 
-typedef enum {
-  TSDB_PRECISION_MILLI,
-  TSDB_PRECISION_MICRO,
-  TSDB_PRECISION_NANO
-} EPrecisionType;
-
 typedef enum {
   TSDB_SUPER_TABLE        = 0,  // super table
   TSDB_CHILD_TABLE        = 1,  // table created from super table

src/inc/tsdb.h

@@ -64,13 +64,14 @@ typedef struct {
   int8_t  compression;
 } STsdbCfg;
 
+typedef void TsdbRepoT;  // use void to hide implementation details from outside
+
 void      tsdbSetDefaultCfg(STsdbCfg *pCfg);
 STsdbCfg *tsdbCreateDefaultCfg();
 void      tsdbFreeCfg(STsdbCfg *pCfg);
+STsdbCfg *tsdbGetCfg(const TsdbRepoT *repo);
 
 // --------- TSDB REPOSITORY DEFINITION
-typedef void TsdbRepoT;  // use void to hide implementation details from outside
-
 int        tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter);
 int32_t    tsdbDropRepo(TsdbRepoT *repo);
 TsdbRepoT *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH);

src/query/inc/qExecutor.h

@@ -136,7 +136,7 @@ typedef struct SQuery {
   int64_t          intervalTime;
   int64_t          slidingTime;      // sliding time for sliding window query
   char             slidingTimeUnit;  // interval data type, used for daytime revise
-  int8_t           precision;
+  int16_t          precision;
   int16_t          numOfOutput;
   int16_t          fillType;
   int16_t          checkBuffer;  // check if the buffer is full during scan each block

src/query/inc/qpercentile.h

@@ -49,7 +49,7 @@ typedef struct tMemBucket {
   int32_t nTotalBufferSize;
   int32_t maxElemsCapacity;
   
-  int16_t pageSize;
+  int32_t pageSize;
   int16_t numOfTotalPages;
   int16_t numOfAvailPages; /* remain available buffer pages */
   

src/query/src/qExecutor.c

@@ -3964,10 +3964,12 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) {
 }
 
 int32_t doInitQInfo(SQInfo *pQInfo, void *param, void *tsdb, int32_t vgId, bool isSTableQuery) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
 
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
-  int32_t code = TSDB_CODE_SUCCESS;
+  pQuery->precision = tsdbGetCfg(tsdb)->precision;
 
   setScanLimitationByResultBuffer(pQuery);
   changeExecuteScanOrder(pQuery, false);
@@ -5422,7 +5424,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
   pQuery->slidingTimeUnit = pQueryMsg->slidingTimeUnit;
   pQuery->fillType        = pQueryMsg->fillType;
   pQuery->numOfTags       = pQueryMsg->numOfTags;
-
+  
   // todo do not allocate ??
   pQuery->colList = calloc(numOfCols, sizeof(SSingleColumnFilterInfo));
   if (pQuery->colList == NULL) {
@@ -5491,7 +5493,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
   
   int tableIndex = 0;
   STimeWindow window = pQueryMsg->window;
-  taosArraySort( pTableIdList, compareTableIdInfo );
+  taosArraySort(pTableIdList, compareTableIdInfo);
   for(int32_t i = 0; i < numOfGroups; ++i) {
     SArray* pa = taosArrayGetP(groupInfo->pGroupList, i);
     size_t s = taosArrayGetSize(pa);
@@ -5960,7 +5962,8 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
     (*pRsp)->offset = 0;
     (*pRsp)->useconds = 0;
   }
-
+  
+  (*pRsp)->precision = htons(pQuery->precision);
   if (pQuery->rec.rows > 0 && code == TSDB_CODE_SUCCESS) {
     code = doDumpQueryResult(pQInfo, (*pRsp)->data);
   } else {

src/tsdb/src/tsdbMain.c

@@ -9,8 +9,7 @@
 #include "ttime.h"
 #include <sys/stat.h>
 
-#define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI  // default precision
-#define IS_VALID_PRECISION(precision) (((precision) >= TSDB_PRECISION_MILLI) && ((precision) <= TSDB_PRECISION_NANO))
+#define IS_VALID_PRECISION(precision) (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO))
 #define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP
 #define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP))
 #define TSDB_MIN_ID 0
@@ -79,6 +78,11 @@ void tsdbFreeCfg(STsdbCfg *pCfg) {
   if (pCfg != NULL) free(pCfg);
 }
 
+STsdbCfg *tsdbGetCfg(const TsdbRepoT *repo) {
+  assert(repo != NULL);
+  return &((STsdbRepo*)repo)->config;
+}
+
 /**
  * Create a new TSDB repository
  * @param rootDir the TSDB repository root directory

src/tsdb/src/tsdbRead.c

@@ -364,12 +364,13 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
   return true;
 }
 
-static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile) {
+static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precision) {
+  assert(precision >= TSDB_TIME_PRECISION_MICRO || precision <= TSDB_TIME_PRECISION_NANO);
   if (key == TSKEY_INITIAL_VAL) {
     return INT32_MIN;
   }
   
-  int64_t fid = (int64_t)(key / (daysPerFile * tsMsPerDay[0]));  // set the starting fileId
+  int64_t fid = (int64_t)(key / (daysPerFile * tsMsPerDay[precision]));  // set the starting fileId
   if (fid < 0L && llabs(fid) > INT32_MAX) { // data value overflow for INT32
     fid = INT32_MIN;
   }
@@ -1297,7 +1298,8 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) {
   // find the start data block in file
   if (!pQueryHandle->locateStart) {
     pQueryHandle->locateStart = true;
-    int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pQueryHandle->pTsdb->config.daysPerFile);
+    STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
+    int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
     
     tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order);
     tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid);