Merge pull request #4884 from taosdata/feature/query

Feature/query

packaging/cfg/taos.cfg

@@ -72,9 +72,6 @@
 # time interval of heart beat from shell to dnode, seconds
 # shellActivityTimer        3
 
-# time of keeping table meta data in cache, seconds
-# tableMetaKeepTimer        7200
-
 # minimum sliding window time, milli-second
 # minSlidingTime            10
 

src/client/inc/tsclient.h

@@ -300,8 +300,8 @@ typedef struct STscObj {
   void *             pTimer;
   char               user[TSDB_USER_LEN];
   char               pass[TSDB_KEY_LEN];
-  char               acctId[TSDB_ACCT_LEN];
-  char               db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+  char               acctId[TSDB_ACCT_ID_LEN];
+  char               db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
   char               sversion[TSDB_VERSION_LEN];
   char               writeAuth : 1;
   char               superAuth : 1;

src/client/src/tscSQLParser.c

@@ -221,7 +221,7 @@ static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
   }
 
-  if (pPwd->n >= TSDB_PASSWORD_LEN) {
+  if (pPwd->n >= TSDB_KEY_LEN) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
   }
 
@@ -1242,7 +1242,7 @@ int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStr
 
   /* db name is not specified, the tableName dose not include db name */
   if (pDB != NULL) {
-    if (pDB->n >= TSDB_ACCT_LEN + TSDB_DB_NAME_LEN || pDB->n == 0) {
+    if (pDB->n >= TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN || pDB->n == 0) {
       return TSDB_CODE_TSC_INVALID_SQL;
     }
 

src/client/src/tscServer.c

@@ -1975,9 +1975,6 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
     //
     //    rsp += tagLen;
     //    int32_t size = (int32_t)(rsp - ((char *)pMeta));  // Consistent with STableMeta in cache
-    //
-    //    pMeta->index = 0;
-    //    (void)taosCachePut(tscMetaCache, pMeta->tableId, (char *)pMeta, size, tsTableMetaKeepTimer);
     //  }
   }
   

src/client/src/tscSql.c

@@ -47,7 +47,7 @@ static bool validUserName(const char* user) {
 }
 
 static bool validPassword(const char* passwd) {
-  return validImpl(passwd, TSDB_PASSWORD_LEN - 1);
+  return validImpl(passwd, TSDB_KEY_LEN - 1);
 }
 
 static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *auth, const char *db,
@@ -238,11 +238,11 @@ TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t us
                      uint8_t passLen, const char *db, uint8_t dbLen, uint16_t port) {
   char ipBuf[TSDB_EP_LEN] = {0};
   char userBuf[TSDB_USER_LEN] = {0};
-  char passBuf[TSDB_PASSWORD_LEN] = {0};
+  char passBuf[TSDB_KEY_LEN] = {0};
   char dbBuf[TSDB_DB_NAME_LEN] = {0};
   strncpy(ipBuf, ip, MIN(TSDB_EP_LEN - 1, ipLen));
   strncpy(userBuf, user, MIN(TSDB_USER_LEN - 1, userLen));
-  strncpy(passBuf, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen));
+  strncpy(passBuf, pass, MIN(TSDB_KEY_LEN - 1, passLen));
   strncpy(dbBuf, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen));
   return taos_connect(ipBuf, userBuf, passBuf, dbBuf, port);
 }
@@ -978,7 +978,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t
       return code;
     }
 
-    if (++pCmd->count > TSDB_MULTI_METERMETA_MAX_NUM) {
+    if (++pCmd->count > TSDB_MULTI_TABLEMETA_MAX_NUM) {
       code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
       sprintf(pCmd->payload, "tables over the max number");
       return code;

src/client/tests/timeParseTest.cpp

@@ -162,6 +162,10 @@ TEST(testCase, parse_time) {
   
   taosParseTime(t13, &time, strlen(t13), TSDB_TIME_PRECISION_MILLI, 0);
   EXPECT_EQ(time, -28800 * MILLISECOND_PER_SECOND);
+
+  char* t = "2021-01-08T02:11:40.000+00:00";
+  taosParseTime(t, &time, strlen(t), TSDB_TIME_PRECISION_MILLI, 0);
+  printf("%ld\n", time);
 }
 
 

src/common/inc/tglobal.h

@@ -63,7 +63,6 @@ extern int32_t  tsRetrieveBlockingModel;// retrieve threads will be blocked
 extern int8_t   tsKeepOriginalColumnName;
 
 // client
-extern int32_t tsTableMetaKeepTimer;
 extern int32_t tsMaxSQLStringLen;
 extern int8_t  tsTscEnableRecordSql;
 extern int32_t tsMaxNumOfOrderedResults;

src/common/src/tglobal.c

@@ -72,7 +72,6 @@ char    tsTempDir[TSDB_FILENAME_LEN] = "/tmp/";
 int32_t tsCompressMsgSize = -1;
 
 // client
-int32_t tsTableMetaKeepTimer = 7200;  // second
 int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN;
 int8_t  tsTscEnableRecordSql = 0;
 
@@ -596,16 +595,6 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_SECOND;
   taosInitConfigOption(cfg);
 
-  cfg.option = "tableMetaKeepTimer";
-  cfg.ptr = &tsTableMetaKeepTimer;
-  cfg.valType = TAOS_CFG_VTYPE_INT32;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
-  cfg.minValue = 1;
-  cfg.maxValue = 8640000;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_SECOND;
-  taosInitConfigOption(cfg);
-
   cfg.option = "minSlidingTime";
   cfg.ptr = &tsMinSlidingTime;
   cfg.valType = TAOS_CFG_VTYPE_INT32;

src/cq/src/cqMain.c

@@ -43,7 +43,7 @@ typedef struct {
   int32_t  master;
   int32_t  num;      // number of continuous streams
   char     user[TSDB_USER_LEN];
-  char     pass[TSDB_PASSWORD_LEN];
+  char     pass[TSDB_KEY_LEN];
   char     db[TSDB_DB_NAME_LEN];
   FCqWrite cqWrite;
   struct SCqObj *pHead;

src/inc/taosdef.h

@@ -256,10 +256,8 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo
 #define TSDB_USER_LEN             TSDB_UNI_LEN
 
 // ACCOUNT is a 32 bit positive integer
-// this is the length of its string representation
-// including the terminator zero
-#define TSDB_ACCT_LEN             11
-#define TSDB_PASSWORD_LEN         TSDB_UNI_LEN
+// this is the length of its string representation, including the terminator zero
+#define TSDB_ACCT_ID_LEN          11
 
 #define TSDB_MAX_COLUMNS          1024
 #define TSDB_MIN_COLUMNS          2       //PRIMARY COLUMN(timestamp) + other columns
@@ -267,7 +265,7 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo
 #define TSDB_NODE_NAME_LEN        64
 #define TSDB_TABLE_NAME_LEN       193     // it is a null-terminated string
 #define TSDB_DB_NAME_LEN          33
-#define TSDB_TABLE_FNAME_LEN      (TSDB_ACCT_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN)
+#define TSDB_TABLE_FNAME_LEN      (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN)
 #define TSDB_COL_NAME_LEN         65
 #define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN          TSDB_PAYLOAD_SIZE
@@ -293,11 +291,6 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo
 #define TSDB_EP_LEN               (TSDB_FQDN_LEN+6)
 #define TSDB_IPv4ADDR_LEN      	  16
 #define TSDB_FILENAME_LEN         128
-#define TSDB_METER_VNODE_BITS     20
-#define TSDB_METER_SID_MASK       0xFFFFF
-#define TSDB_SHELL_VNODE_BITS     24
-#define TSDB_SHELL_SID_MASK       0xFF
-#define TSDB_HTTP_TOKEN_LEN       20
 #define TSDB_SHOW_SQL_LEN         512
 #define TSDB_SLOW_QUERY_SQL_LEN   512
 
@@ -311,9 +304,6 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo
 #define TSDB_MQTT_TOPIC_LEN       64
 #define TSDB_MQTT_CLIENT_ID_LEN   32
 
-#define TSDB_METER_STATE_OFFLINE  0
-#define TSDB_METER_STATE_ONLLINE  1
-
 #define TSDB_DEFAULT_PKT_SIZE     65480  //same as RPC_MAX_UDP_SIZE
 
 #define TSDB_PAYLOAD_SIZE         TSDB_DEFAULT_PKT_SIZE
@@ -333,7 +323,7 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo
 
 #define TSDB_TBNAME_COLUMN_INDEX        (-1)
 #define TSDB_UD_COLUMN_INDEX            (-100)
-#define TSDB_MULTI_METERMETA_MAX_NUM    100000  // maximum batch size allowed to load metermeta
+#define TSDB_MULTI_TABLEMETA_MAX_NUM    100000  // maximum batch size allowed to load table meta
 
 #define TSDB_MIN_CACHE_BLOCK_SIZE       1
 #define TSDB_MAX_CACHE_BLOCK_SIZE       128     // 128MB for each vnode

src/inc/taosmsg.h

@@ -268,7 +268,7 @@ typedef struct {
 typedef struct {
   int32_t len;  // one create table message
   char    tableId[TSDB_TABLE_FNAME_LEN];
-  char    db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+  char    db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
   int8_t  igExists;
   int8_t  getMeta;
   int16_t numOfTags;
@@ -290,7 +290,7 @@ typedef struct {
 
 typedef struct {
   char    tableId[TSDB_TABLE_FNAME_LEN];
-  char    db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+  char    db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
   int16_t type; /* operation type   */
   int16_t numOfCols; /* number of schema */
   int32_t tagValLen;
@@ -322,7 +322,7 @@ typedef struct {
 } SConnectMsg;
 
 typedef struct {
-  char      acctId[TSDB_ACCT_LEN];
+  char      acctId[TSDB_ACCT_ID_LEN];
   char      serverVersion[TSDB_VERSION_LEN];
   char      clusterId[TSDB_CLUSTER_ID_LEN];
   int8_t    writeAuth;
@@ -534,7 +534,7 @@ typedef struct {
 } SVnodeLoad;
 
 typedef struct {
-  char     db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+  char     db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
   int32_t  cacheBlockSize; //MB
   int32_t  totalBlocks;
   int32_t  maxTables;
@@ -682,7 +682,7 @@ typedef struct {
 } SVnodeDesc;
 
 typedef struct {
-  char       db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+  char       db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
   SVnodeCfg  cfg;
   SVnodeDesc nodes[TSDB_MAX_REPLICA];
 } SCreateVnodeMsg, SAlterVnodeMsg;
@@ -761,7 +761,7 @@ typedef struct {
  */
 typedef struct {
   int8_t   type;
-  char     db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+  char     db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
   uint16_t payloadLen;
   char     payload[];
 } SShowMsg;

src/inc/tcq.h

@@ -26,7 +26,7 @@ typedef int32_t (*FCqWrite)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg
 typedef struct {
   int32_t  vgId;
   char     user[TSDB_USER_LEN];
-  char     pass[TSDB_PASSWORD_LEN];
+  char     pass[TSDB_KEY_LEN];
   char     db[TSDB_DB_NAME_LEN];
   FCqWrite cqWrite;
 } SCqCfg;

src/mnode/inc/mnodeDef.h

@@ -138,7 +138,7 @@ typedef struct SVgObj {
   int64_t        createdTime;
   int32_t        lbDnodeId;
   int32_t        lbTime;
-  char           dbName[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+  char           dbName[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
   int8_t         inUse;
   int8_t         accessState;
   int8_t         status;
@@ -179,7 +179,7 @@ typedef struct {
 } SDbCfg;
 
 typedef struct SDbObj {
-  char    name[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+  char    name[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
   int8_t  reserved0[4];
   char    acct[TSDB_USER_LEN];
   int64_t createdTime;

src/mnode/src/mnodeVgroup.c

@@ -565,7 +565,7 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) {
   SDbObj *pDb = pMsg->pDb;
 
   SVgObj *pVgroup = (SVgObj *)calloc(1, sizeof(SVgObj));
-  tstrncpy(pVgroup->dbName, pDb->name, TSDB_ACCT_LEN + TSDB_DB_NAME_LEN);
+  tstrncpy(pVgroup->dbName, pDb->name, TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN);
   pVgroup->numOfVnodes = pDb->cfg.replications;
   pVgroup->createdTime = taosGetTimestampMs();
   pVgroup->accessState = TSDB_VN_ALL_ACCCESS;

src/os/CMakeLists.txt

@@ -10,3 +10,5 @@ ELSEIF (TD_WINDOWS)
 ENDIF ()
 
 ADD_SUBDIRECTORY(src/detail)
+ADD_SUBDIRECTORY(tests)
+

src/os/src/detail/osTime.c

@@ -501,9 +501,24 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio
       start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision));
     }
 
-    int64_t end = start + pInterval->interval - 1;
-    if (end < t) {
+    int64_t end = 0;
+
+    // not enough time range
+    if (INT64_MAX - start > pInterval->interval - 1) {
+      end = start + pInterval->interval - 1;
+
+      while(end < t && ((start + pInterval->sliding) <= INT64_MAX)) { // move forward to the correct time window
         start += pInterval->sliding;
+
+        if (INT64_MAX - start > pInterval->interval - 1) {
+          end = start + pInterval->interval - 1;
+        } else {
+          end = INT64_MAX;
+          break;
+        }
+      }
+    } else {
+      end = INT64_MAX;
     }
   }
 

src/os/tests/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest)
+FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib)
+
+IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR)
+    MESSAGE(STATUS "gTest library found, build unit test")
+
+    INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
+    AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+
+    ADD_EXECUTABLE(osTest ${SOURCE_LIST})
+    TARGET_LINK_LIBRARIES(osTest taos osdetail tutil common gtest pthread)
+ENDIF()
\ No newline at end of file

src/os/tests/test.cpp

+#include "os.h"
+#include <gtest/gtest.h>
+#include <cassert>
+#include <iostream>
+
+#include "taos.h"
+#include "tstoken.h"
+#include "tutil.h"
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+// test function in os module
+TEST(testCase, parse_time) {
+  taos_options(TSDB_OPTION_TIMEZONE, "GMT-8");
+  deltaToUtcInitOnce();
+
+  // window: 1500000001000, 1500002000000
+  // pQuery->interval: interval: 86400000, sliding:3600000
+  int64_t key = 1500000001000;
+  SInterval interval = {0};
+  interval.interval = 86400000;
+  interval.intervalUnit = 'd';
+  interval.sliding = 3600000;
+  interval.slidingUnit = 'h';
+
+  int64_t s = taosTimeTruncate(key, &interval, TSDB_TIME_PRECISION_MILLI);
+  ASSERT_TRUE(s + interval.interval >= key);
+}
+
+
+

src/plugins/http/inc/httpInt.h

@@ -39,7 +39,7 @@
 #define HTTP_GC_TARGET_SIZE         512
 #define HTTP_WRITE_RETRY_TIMES      500
 #define HTTP_WRITE_WAIT_TIME_MS     5
-#define HTTP_SESSION_ID_LEN         (TSDB_USER_LEN + TSDB_PASSWORD_LEN)
+#define HTTP_SESSION_ID_LEN         (TSDB_USER_LEN + TSDB_KEY_LEN)
 
 typedef enum HttpReqType {
   HTTP_REQTYPE_OTHERS = 0,
@@ -147,7 +147,7 @@ typedef struct HttpContext {
   uint8_t      parsed;
   char         ipstr[22];
   char         user[TSDB_USER_LEN];  // parsed from auth token or login message
-  char         pass[TSDB_PASSWORD_LEN];
+  char         pass[TSDB_KEY_LEN];
   TAOS *       taos;
   void *       ppContext;
   HttpSession *session;

src/plugins/http/src/httpAuth.c

@@ -51,7 +51,7 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len)
 
   char *password = user + 1;
   int32_t pass_len = (int32_t)((base64 + outlen) - password);
-  if (pass_len < 1 || pass_len >= TSDB_PASSWORD_LEN) {
+  if (pass_len < 1 || pass_len >= TSDB_KEY_LEN) {
     httpError("context:%p, fd:%d, basic token:%s parse password error", pContext, pContext->fd, token);
     free(base64);
     return -1;
@@ -73,7 +73,7 @@ int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len)
     if (base64) free(base64);
     return 01;
   }
-  if (outlen != (TSDB_USER_LEN + TSDB_PASSWORD_LEN)) {
+  if (outlen != (TSDB_USER_LEN + TSDB_KEY_LEN)) {
     httpError("context:%p, fd:%d, taosd token:%s length error", pContext, pContext->fd, token);
     free(base64);
     return -1;
@@ -103,8 +103,8 @@ int32_t httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen
   size = sizeof(pContext->pass);
   tstrncpy(buffer + sizeof(pContext->user), pContext->pass, size);
 
-  char *encrypt = taosDesEncode(KEY_DES_4, buffer, TSDB_USER_LEN + TSDB_PASSWORD_LEN);
-  char *base64 = base64_encode((const unsigned char *)encrypt, TSDB_USER_LEN + TSDB_PASSWORD_LEN);
+  char *encrypt = taosDesEncode(KEY_DES_4, buffer, TSDB_USER_LEN + TSDB_KEY_LEN);
+  char *base64 = base64_encode((const unsigned char *)encrypt, TSDB_USER_LEN + TSDB_KEY_LEN);
 
   size_t len = strlen(base64);
   tstrncpy(token, base64, len + 1);

src/plugins/http/src/httpGcHandle.c

@@ -59,11 +59,11 @@ bool gcGetUserFromUrl(HttpContext* pContext) {
 
 bool gcGetPassFromUrl(HttpContext* pContext) {
   HttpParser* pParser = pContext->parser;
-  if (pParser->path[GC_PASS_URL_POS].pos >= TSDB_PASSWORD_LEN || pParser->path[GC_PASS_URL_POS].pos <= 0) {
+  if (pParser->path[GC_PASS_URL_POS].pos >= TSDB_KEY_LEN || pParser->path[GC_PASS_URL_POS].pos <= 0) {
     return false;
   }
 
-  tstrncpy(pContext->pass, pParser->path[GC_PASS_URL_POS].str, TSDB_PASSWORD_LEN);
+  tstrncpy(pContext->pass, pParser->path[GC_PASS_URL_POS].str, TSDB_KEY_LEN);
   return true;
 }
 

src/plugins/http/src/httpRestHandle.c

@@ -72,11 +72,11 @@ bool restGetUserFromUrl(HttpContext* pContext) {
 
 bool restGetPassFromUrl(HttpContext* pContext) {
   HttpParser* pParser = pContext->parser;
-  if (pParser->path[REST_PASS_URL_POS].pos >= TSDB_PASSWORD_LEN || pParser->path[REST_PASS_URL_POS].pos <= 0) {
+  if (pParser->path[REST_PASS_URL_POS].pos >= TSDB_KEY_LEN || pParser->path[REST_PASS_URL_POS].pos <= 0) {
     return false;
   }
 
-  tstrncpy(pContext->pass, pParser->path[REST_PASS_URL_POS].str, TSDB_PASSWORD_LEN);
+  tstrncpy(pContext->pass, pParser->path[REST_PASS_URL_POS].str, TSDB_KEY_LEN);
   return true;
 }
 

src/plugins/http/src/httpTgHandle.c

@@ -324,7 +324,7 @@ bool tgGetUserFromUrl(HttpContext *pContext) {
 
 bool tgGetPassFromUrl(HttpContext *pContext) {
   HttpParser *pParser = pContext->parser;
-  if (pParser->path[TG_PASS_URL_POS].pos >= TSDB_PASSWORD_LEN || pParser->path[TG_PASS_URL_POS].pos <= 0) {
+  if (pParser->path[TG_PASS_URL_POS].pos >= TSDB_KEY_LEN || pParser->path[TG_PASS_URL_POS].pos <= 0) {
     return false;
   }
 

src/query/inc/qPercentile.h

@@ -16,6 +16,10 @@
 #ifndef TDENGINE_QPERCENTILE_H
 #define TDENGINE_QPERCENTILE_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #include "qExtbuffer.h"
 #include "qResultbuf.h"
 #include "qTsbuf.h"
@@ -23,13 +27,13 @@
 typedef struct MinMaxEntry {
   union {
     double   dMinVal;
-    int32_t iMinVal;
     int64_t  i64MinVal;
+    uint64_t u64MinVal;
   };
   union {
     double  dMaxVal;
-    int32_t iMaxVal;
     int64_t i64MaxVal;
+    int64_t u64MaxVal;
   };
 } MinMaxEntry;
 
@@ -59,7 +63,7 @@ typedef struct tMemBucket {
   int32_t       times;        // count that has been checked for deciding the correct data value buckets.
   __compar_fn_t comparFn;
 
-  tMemBucketSlot *pSlots;
+  tMemBucketSlot *     pSlots;
   SDiskbasedResultBuf *pBuffer;
   __perc_hash_func_t   hashFunc;
 } tMemBucket;
@@ -73,3 +77,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size);
 double getPercentile(tMemBucket *pMemBucket, double percent);
 
 #endif  // TDENGINE_QPERCENTILE_H
+
+#ifdef __cplusplus
+}
+#endif
\ No newline at end of file

src/query/src/qAggMain.c

@@ -2545,7 +2545,7 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) {
     if (pInfo->numOfElems == 0) {
       pResInfo->complete = true;
     } else {
-      pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, GET_DOUBLE_VAL(&pInfo->minval), GET_DOUBLE_VAL(&pInfo->maxval));
+      pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval);
     }
 
     pInfo->stage += 1;

src/query/tests/percentileTest.cpp

+#include <gtest/gtest.h>
+#include <iostream>
+
+#include "qResultbuf.h"
+#include "taos.h"
+#include "taosdef.h"
+
+#include "qPercentile.h"
+
+namespace {
+tMemBucket *createBigIntDataBucket(int32_t start, int32_t end) {
+  tMemBucket *pBucket = tMemBucketCreate(sizeof(int64_t), TSDB_DATA_TYPE_BIGINT, start, end);
+  for (int32_t i = start; i <= end; ++i) {
+    int64_t val = i;
+    tMemBucketPut(pBucket, &val, 1);
+  }
+
+  return pBucket;
+}
+
+tMemBucket *createIntDataBucket(int32_t start, int32_t end) {
+  tMemBucket *pBucket = tMemBucketCreate(sizeof(int32_t), TSDB_DATA_TYPE_INT, start, end);
+
+  for (int32_t i = start; i <= end; ++i) {
+    int32_t val = i;
+    tMemBucketPut(pBucket, &val, 1);
+  }
+
+  return pBucket;
+}
+
+tMemBucket *createDoubleDataBucket(int32_t start, int32_t end) {
+  tMemBucket *pBucket = tMemBucketCreate(sizeof(double), TSDB_DATA_TYPE_DOUBLE, start, end);
+  for (int32_t i = start; i <= end; ++i) {
+    double  val = i;
+    int32_t ret = tMemBucketPut(pBucket, &val, 1);
+    if (ret != 0) {
+      printf("value out of range:%f", val);
+    }
+  }
+
+  return pBucket;
+}
+
+tMemBucket *createUnsignedDataBucket(int32_t start, int32_t end, int32_t type) {
+  tMemBucket *pBucket = tMemBucketCreate(tDataTypeDesc[type].nSize, type, start, end);
+  for (int32_t i = start; i <= end; ++i) {
+    uint64_t k = i;
+    int32_t ret = tMemBucketPut(pBucket, &k, 1);
+    if (ret != 0) {
+      printf("value out of range:%f", k);
+    }
+  }
+
+  return pBucket;
+}
+
+void intDataTest() {
+  printf("running %s\n", __FUNCTION__);
+
+  tMemBucket *pBucket = NULL;
+  double      result = 0.;
+
+  pBucket = createIntDataBucket(0, 0);
+  result = getPercentile(pBucket, 0);
+  ASSERT_DOUBLE_EQ(result, 0);
+  tMemBucketDestroy(pBucket);
+
+  pBucket = createIntDataBucket(0, 1);
+  result = getPercentile(pBucket, 100);
+  ASSERT_DOUBLE_EQ(result, 1);
+
+  result = getPercentile(pBucket, 0);
+  ASSERT_DOUBLE_EQ(result, 0);
+  tMemBucketDestroy(pBucket);
+
+  pBucket = createIntDataBucket(-1, 1);
+
+  result = getPercentile(pBucket, 50);
+  ASSERT_DOUBLE_EQ(result, 0);
+
+  result = getPercentile(pBucket, 0);
+  ASSERT_DOUBLE_EQ(result, -1);
+
+  result = getPercentile(pBucket, 75);
+  ASSERT_DOUBLE_EQ(result, 0.5);
+
+  result = getPercentile(pBucket, 100);
+  ASSERT_DOUBLE_EQ(result, 1);
+  tMemBucketDestroy(pBucket);
+
+  pBucket = createIntDataBucket(0, 99999);
+  result = getPercentile(pBucket, 50);
+  ASSERT_DOUBLE_EQ(result, 49999.5);
+
+  tMemBucketDestroy(pBucket);
+}
+
+void bigintDataTest() {
+  printf("running %s\n", __FUNCTION__);
+
+  tMemBucket *pBucket = NULL;
+  double      result = 0.0;
+
+  pBucket = createBigIntDataBucket(-1000, 1000);
+  result = getPercentile(pBucket, 50);
+  ASSERT_DOUBLE_EQ(result, 0.);
+  tMemBucketDestroy(pBucket);
+
+  pBucket = createBigIntDataBucket(-10000, 10000);
+  result = getPercentile(pBucket, 100);
+  ASSERT_DOUBLE_EQ(result, 10000.0);
+  tMemBucketDestroy(pBucket);
+
+  pBucket = createBigIntDataBucket(-10000, 10000);
+  result = getPercentile(pBucket, 75);
+  ASSERT_DOUBLE_EQ(result, 5000.0);
+
+  tMemBucketDestroy(pBucket);
+}
+
+void doubleDataTest() {
+  printf("running %s\n", __FUNCTION__);
+
+  tMemBucket *pBucket = NULL;
+  double      result = 0;
+
+  pBucket = createDoubleDataBucket(-10, 10);
+  result = getPercentile(pBucket, 0);
+  ASSERT_DOUBLE_EQ(result, -10.0);
+
+  printf("result is: %lf\n", result);
+  tMemBucketDestroy(pBucket);
+
+  pBucket = createDoubleDataBucket(-100000, 100000);
+  result = getPercentile(pBucket, 25);
+  ASSERT_DOUBLE_EQ(result, -50000);
+
+  printf("result is: %lf\n", result);
+
+  tMemBucketDestroy(pBucket);
+
+  pBucket = createDoubleDataBucket(-100000, 100000);
+  result = getPercentile(pBucket, 50);
+  ASSERT_DOUBLE_EQ(result, 0);
+
+  tMemBucketDestroy(pBucket);
+
+  pBucket = createDoubleDataBucket(-100000, 100000);
+  result = getPercentile(pBucket, 75);
+  ASSERT_DOUBLE_EQ(result, 50000);
+  tMemBucketDestroy(pBucket);
+
+  pBucket = createDoubleDataBucket(-100000, 100000);
+
+  result = getPercentile(pBucket, 100);
+  ASSERT_DOUBLE_EQ(result, 100000.0);
+
+  printf("result is: %lf\n", result);
+  tMemBucketDestroy(pBucket);
+}
+
+/*
+ * large data test, we employ 0.1billion double data to calculated the percentile
+ * which is 800MB data
+ */
+void largeDataTest() {
+  printf("running : %s\n", __FUNCTION__);
+
+  tMemBucket *pBucket = NULL;
+  double      result = 0;
+
+  struct timeval tv;
+  gettimeofday(&tv, NULL);
+
+  int64_t start = tv.tv_sec;
+  printf("start time: %" PRId64 "\n", tv.tv_sec);
+  pBucket = createDoubleDataBucket(0, 100000000);
+  result = getPercentile(pBucket, 50);
+  ASSERT_DOUBLE_EQ(result, 50000000);
+
+  gettimeofday(&tv, NULL);
+
+  printf("total elapsed time: %" PRId64 " sec.", -start + tv.tv_sec);
+  printf("the result of %d is: %lf\n", 50, result);
+  tMemBucketDestroy(pBucket);
+}
+
+void qsortTest() {
+  printf("running : %s\n", __FUNCTION__);
+
+  SSchema field[1] = {
+      {TSDB_DATA_TYPE_INT, "k", sizeof(int32_t)},
+  };
+
+  const int32_t num = 2000;
+
+  int32_t *d = (int32_t *)malloc(sizeof(int32_t) * num);
+  for (int32_t i = 0; i < num; ++i) {
+    d[i] = i % 4;
+  }
+
+  const int32_t     numOfOrderCols = 1;
+  int32_t           orderColIdx = 0;
+  SColumnModel *    pModel = createColumnModel(field, 1, 1000);
+  tOrderDescriptor *pDesc = tOrderDesCreate(&orderColIdx, numOfOrderCols, pModel, 1);
+
+  tColDataQSort(pDesc, num, 0, num - 1, (char *)d, 1);
+
+  for (int32_t i = 0; i < num; ++i) {
+    printf("%d\t", d[i]);
+  }
+  printf("\n");
+
+  destroyColumnModel(pModel);
+}
+
+void unsignedDataTest() {
+  printf("running %s\n", __FUNCTION__);
+
+  tMemBucket *pBucket = NULL;
+  double      result = 0.0;
+
+  pBucket = createUnsignedDataBucket(0, 1000, TSDB_DATA_TYPE_UINT);
+  result = getPercentile(pBucket, 50);
+  ASSERT_DOUBLE_EQ(result, 500.0);
+  tMemBucketDestroy(pBucket);
+
+  pBucket = createUnsignedDataBucket(0, 10000, TSDB_DATA_TYPE_UBIGINT);
+  result = getPercentile(pBucket, 100);
+  ASSERT_DOUBLE_EQ(result, 10000.0);
+
+  result = getPercentile(pBucket, 0);
+  ASSERT_DOUBLE_EQ(result, 0.0);
+
+  result = getPercentile(pBucket, 50);
+  ASSERT_DOUBLE_EQ(result, 5000);
+
+  result = getPercentile(pBucket, 75);
+  ASSERT_DOUBLE_EQ(result, 7500);
+  tMemBucketDestroy(pBucket);
+
+}
+
+}  // namespace
+
+TEST(testCase, percentileTest) {
+  //  qsortTest();
+  intDataTest();
+  bigintDataTest();
+  doubleDataTest();
+  unsignedDataTest();
+  largeDataTest();
+}

src/util/src/tcompare.c

@@ -30,24 +30,32 @@ int32_t compareInt8Val(const void *pLeft, const void *pRight) {
   return 0;
 }
 
-int32_t compareIntDoubleVal(const void *pLeft, const void *pRight) {
-  int64_t lhs = GET_INT64_VAL(pLeft);
-  double  rhs = GET_DOUBLE_VAL(pRight);
-  if (fabs(lhs - rhs) < FLT_EPSILON) {
+int32_t compareUint32Val(const void *pLeft, const void *pRight) {
+  int32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
   return 0;
-  } else {
-    return (lhs > rhs) ? 1 : -1;
-  }
 }
 
-int32_t compareDoubleIntVal(const void *pLeft, const void *pRight) {
-  double  lhs = GET_DOUBLE_VAL(pLeft);
-  int64_t rhs = GET_INT64_VAL(pRight);
-  if (fabs(lhs - rhs) < FLT_EPSILON) {
+int32_t compareUint64Val(const void *pLeft, const void *pRight) {
+  int64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareUint16Val(const void *pLeft, const void *pRight) {
+  int16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
+  return 0;
+}
+
+int32_t compareUint8Val(const void* pLeft, const void* pRight) {
+  uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight);
+  if (left > right) return 1;
+  if (left < right) return -1;
   return 0;
-  } else {
-    return (lhs > rhs) ? 1 : -1;
-  }
 }
 
 int32_t compareFloatVal(const void *pLeft, const void *pRight) {
@@ -369,15 +377,24 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) {
     case TSDB_DATA_TYPE_DOUBLE:
       comparFn = compareDoubleVal;
       break;
-    
+    case TSDB_DATA_TYPE_UTINYINT:
+      comparFn = compareUint8Val;
+      break;
+    case TSDB_DATA_TYPE_USMALLINT:
+      comparFn = compareUint16Val;
+      break;
+    case TSDB_DATA_TYPE_UINT:
+      comparFn = compareUint32Val;
+      break;
+    case TSDB_DATA_TYPE_UBIGINT:
+      comparFn = compareUint64Val;
+      break;
     case TSDB_DATA_TYPE_BINARY:
       comparFn = compareLenPrefixedStr;
       break;
-    
     case TSDB_DATA_TYPE_NCHAR:
       comparFn = compareLenPrefixedWStr;
       break;
-    
     default:
       comparFn = compareInt32Val;
       break;

src/vnode/inc/vnodeInt.h

@@ -67,7 +67,7 @@ typedef struct {
   void *   qMgmt;
   char *   rootDir;
   tsem_t   sem;
-  char     db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
+  char     db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
   pthread_mutex_t statusMutex;
 } SVnodeObj;
 

tests/script/general/cache/new_metrics.sim

@@ -3,7 +3,6 @@ system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 0
-system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 10
 system sh/exec.sh -n dnode1 -s start
 
 sleep 3000

tests/script/general/cache/restart_metrics.sim

@@ -3,7 +3,6 @@ system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 0
-system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 10
 system sh/exec.sh -n dnode1 -s start
 
 sleep 3000
@@ -53,7 +52,6 @@ system sh/exec.sh -n dnode1 -s stop
 sleep 5000
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 0
-system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 10
 system sh/exec.sh -n dnode1 -s start
 
 print =============== step3

tests/script/general/cache/restart_table.sim

@@ -3,7 +3,6 @@ system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 0
-system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 10
 system sh/exec.sh -n dnode1 -s start
 
 sleep 3000
@@ -37,7 +36,6 @@ system sh/exec.sh -n dnode1 -s stop
 sleep 5000
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 0
-system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 10
 system sh/exec.sh -n dnode1 -s start
 
 print =============== step3

tests/script/general/parser/alter.sim

@@ -2,7 +2,6 @@ system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 0
-system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3
 system sh/exec.sh -n dnode1 -s start
 sleep 100
 sql connect

tests/script/general/parser/alter_stable.sim

@@ -2,7 +2,6 @@ system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 0
-system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3
 system sh/exec.sh -n dnode1 -s start
 sleep 100
 sql connect

tests/script/general/parser/binary_escapeCharacter.sim

@@ -2,7 +2,6 @@ system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 0
-system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3
 system sh/exec.sh -n dnode1 -s start
 sleep 100
 sql connect

tests/script/general/parser/columnValue.sim

@@ -3,7 +3,6 @@ system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 0
-system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3
 system sh/exec.sh -n dnode1 -s start
 sleep 100
 sql connect

tests/script/general/parser/columnValue_unsign.sim

@@ -107,3 +107,69 @@ if $data01 != 1 then
 endi
 
 sql select a+b+c from mt_unsigned_1 where a is null;
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 6.000000000 then
+  return -1
+endi
+
+sql select count(*), a from mt_unsigned_1 group by a;
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 1 then
+  return -1
+endi
+
+if $data01 != 1 then
+  return -1
+endi
+
+sql select count(*), b from mt_unsigned_1 group by b;
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 1 then
+  return -1
+endi
+
+if $data01 != 2 then
+  return -1
+endi
+
+sql select count(*), c from mt_unsigned_1 group by c;
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 1 then
+  return -1
+endi
+
+if $data01 != 3 then
+  return -1
+endi
+
+sql select count(*), d from mt_unsigned_1 group by d;
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 1 then
+  return -1
+endi
+
+if $data01 != 4 then
+  return -1
+endi
+
+// todo insert more rows and chec it
+sql select first(a),count(b),last(c),sum(b),spread(d),avg(c),min(b),max(a),stddev(a) from mt_unsigned_1;
+if $rows != 1 then
+  return -1
+endi
+

tests/script/general/parser/function.sim

@@ -2,7 +2,6 @@ system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 0
-system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3
 system sh/exec.sh -n dnode1 -s start
 sleep 100
 sql connect
@@ -228,7 +227,7 @@ sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<
 sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00' interval(10m) order by ts desc
 
 
-#todo add test case while column filter exists.
+#todo add test case while column filter exists for twa query
 
 #sql select count(*),TWA(k) from tm0 where ts>='1970-1-1 13:43:00' and ts<='1970-1-1 13:44:10' interval(9s)
 
@@ -373,3 +372,14 @@ sql select twa(k) from tm2 where ts='2020-12-29 18:46:19.109'
 if $rows != 0 then
   return -1
 endi
+
+print ========================> TD-1787
+sql create table cars(ts timestamp, c int) tags(id int);
+sql create table car1 using cars tags(1);
+sql create table car2 using cars tags(2);
+sql insert into car1 (ts, c) values (now,1) car2(ts, c) values(now, 2);
+sql drop table cars;
+sql create table cars(ts timestamp, c int) tags(id int);
+sql create table car1 using cars tags(1);
+sql create table car2 using cars tags(2);
+sql insert into car1 (ts, c) values (now,1) car2(ts, c) values(now, 2);
\ No newline at end of file

tests/script/general/parser/null_char.sim

@@ -3,7 +3,6 @@ system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 0
-system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3
 system sh/exec.sh -n dnode1 -s start
 
 sleep 100

tests/script/general/stream/restart_stream.sim

@@ -3,7 +3,6 @@ system sh/stop_dnodes.sh
 
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 0
-system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 1
 system sh/exec.sh -n dnode1 -s start
 
 sleep 3000
@@ -98,7 +97,6 @@ print =============== step4
 system sh/exec.sh -n dnode1 -s stop
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 0
-system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 1
 system sh/exec.sh -n dnode1 -s start
 
 print =============== step5

tests/script/test.sh

@@ -113,7 +113,6 @@ echo "rpcDebugFlag       143"                     >> $TAOS_CFG
 echo "tmrDebugFlag       131"                     >> $TAOS_CFG
 echo "cDebugFlag         143"                     >> $TAOS_CFG
 echo "udebugFlag         143"                     >> $TAOS_CFG
-echo "tablemetakeeptimer 5"                       >> $TAOS_CFG
 echo "wal                0"                       >> $TAOS_CFG
 echo "asyncLog           0"                       >> $TAOS_CFG
 echo "locale             en_US.UTF-8"             >> $TAOS_CFG

tests/script/unique/stream/metrics_balance.sim

@@ -14,8 +14,6 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 0
 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 0
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 4
-system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 5
-system sh/cfg.sh -n dnode2 -c tableMetaKeepTimer -v 5
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2