Merge remote-tracking branch 'origin/master' into feature/m1

.drone.yml

@@ -15,7 +15,7 @@ steps:
   - mkdir debug
   - cd debug
   - cmake .. 
-  - make 
+  - make -j4 
   trigger:
     event:
       - pull_request
@@ -23,6 +23,7 @@ steps:
     branch:
     - develop
     - master
+    - 2.0
 ---
 kind: pipeline
 name: test_arm64_bionic
@@ -39,7 +40,7 @@ steps:
   - mkdir debug
   - cd debug
   - cmake .. -DCPUTYPE=aarch64 > /dev/null
-  - make 
+  - make -j4 
   trigger:
     event:
       - pull_request
@@ -66,7 +67,7 @@ steps:
   - mkdir debug
   - cd debug
   - cmake .. -DCPUTYPE=aarch64 > /dev/null
-  - make 
+  - make -j4 
   trigger:
     event:
       - pull_request
@@ -91,7 +92,7 @@ steps:
   - mkdir debug
   - cd debug
   - cmake .. -DCPUTYPE=aarch64 > /dev/null
-  - make 
+  - make -j4 
   trigger:
     event:
       - pull_request
@@ -116,7 +117,7 @@ steps:
   - mkdir debug
   - cd debug
   - cmake .. -DCPUTYPE=aarch64 > /dev/null
-  - make 
+  - make -j4 
   trigger:
     event:
       - pull_request
@@ -142,7 +143,7 @@ steps:
   - mkdir debug
   - cd debug
   - cmake .. -DCPUTYPE=aarch32 > /dev/null
-  - make 
+  - make -j4 
   trigger:
     event:
       - pull_request
@@ -150,6 +151,7 @@ steps:
     branch:
     - develop
     - master
+    - 2.0
 ---
 kind: pipeline
 name: build_trusty
@@ -168,7 +170,7 @@ steps:
   - mkdir debug
   - cd debug
   - cmake .. 
-  - make 
+  - make -j4 
   trigger:
     event:
       - pull_request
@@ -176,6 +178,7 @@ steps:
     branch:
     - develop
     - master
+    - 2.0
 ---
 kind: pipeline
 name: build_xenial
@@ -193,7 +196,7 @@ steps:
   - mkdir debug
   - cd debug
   - cmake .. 
-  - make 
+  - make -j4 
   trigger:
     event:
       - pull_request
@@ -201,7 +204,7 @@ steps:
     branch:
     - develop
     - master
-
+    - 2.0
 ---
 kind: pipeline
 name: build_bionic
@@ -218,7 +221,7 @@ steps:
   - mkdir debug
   - cd debug
   - cmake .. 
-  - make 
+  - make -j4 
   trigger:
     event:
       - pull_request
@@ -226,6 +229,7 @@ steps:
     branch:
     - develop
     - master
+    - 2.0
 ---
 kind: pipeline
 name: build_centos7
@@ -241,7 +245,7 @@ steps:
   - mkdir debug
   - cd debug
   - cmake .. 
-  - make 
+  - make -j4 
   trigger:
     event:
       - pull_request
@@ -249,4 +253,4 @@ steps:
     branch:
     - develop
     - master
-
+    - 2.0
\ No newline at end of file

Jenkinsfile

@@ -160,7 +160,6 @@ pipeline {
             skipbuild='2'     
             skipbuild=sh(script: "git log -2 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' && echo 1 || echo 2", returnStdout:true)
             println skipbuild
-
           }
           sh'''
           rm -rf ${WORKSPACE}.tes
@@ -225,6 +224,26 @@ pipeline {
           steps {     
             timeout(time: 55, unit: 'MINUTES'){       
               pre_test()
+              sh '''
+                rm -rf /var/lib/taos/*
+                rm -rf /var/log/taos/*
+                nohup taosd >/dev/null &
+                sleep 10
+              '''
+              sh '''
+              cd ${WKC}/tests/examples/nodejs
+              npm install td2.0-connector > /dev/null 2>&1
+              node nodejsChecker.js host=localhost
+              '''
+              sh '''
+                cd ${WKC}/tests/examples/C#/taosdemo
+                mcs -out:taosdemo *.cs > /dev/null 2>&1
+                echo '' |./taosdemo
+              '''
+              sh '''
+                cd ${WKC}/tests/gotest
+                bash batchtest.sh
+              '''
               sh '''
               cd ${WKC}/tests
               ./test-all.sh b1fq

cmake/version.inc

@@ -4,7 +4,7 @@ PROJECT(TDengine)
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "2.1.6.0")
+  SET(TD_VER_NUMBER "2.1.7.1")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

deps/MsvcLibX/src/iconv.c

@@ -98,6 +98,7 @@ int ConvertString(char *buf, size_t nBytes, UINT cpFrom, UINT cpTo, LPCSTR lpDef
 char *DupAndConvert(const char *string, UINT cpFrom, UINT cpTo, LPCSTR lpDefaultChar) {
   int nBytes;
   char *pBuf;
+  char *pBuf1;
   nBytes = 4 * ((int)lstrlen(string) + 1); /* Worst case for the size needed */
   pBuf = (char *)malloc(nBytes);
   if (!pBuf) {
@@ -110,8 +111,9 @@ char *DupAndConvert(const char *string, UINT cpFrom, UINT cpTo, LPCSTR lpDefault
     free(pBuf);
     return NULL;
   }
-  pBuf = realloc(pBuf, nBytes+1);
-  return pBuf;
+  pBuf1 = realloc(pBuf, nBytes+1);
+  if(pBuf1 == NULL && pBuf != NULL)  free(pBuf);
+  return pBuf1;
 }
 
 int CountCharacters(const char *string, UINT cp) {

deps/MsvcLibX/src/main.c

@@ -68,6 +68,7 @@ int BreakArgLine(LPSTR pszCmdLine, char ***pppszArg) {
   int iString = FALSE;	/* TRUE = string mode; FALSE = non-string mode */
   int nBackslash = 0;
   char **ppszArg;
+  char **ppszArg1;
   int iArg = FALSE;	/* TRUE = inside an argument; FALSE = between arguments */
 
   ppszArg = (char **)malloc((argc+1)*sizeof(char *));
@@ -89,7 +90,10 @@ int BreakArgLine(LPSTR pszCmdLine, char ***pppszArg) {
     if ((!iArg) && (c != ' ') && (c != '\t')) { /* Beginning of a new argument */
       iArg = TRUE;
       ppszArg[argc++] = pszCopy+j;
-      ppszArg = (char **)realloc(ppszArg, (argc+1)*sizeof(char *));
+      ppszArg1 = (char **)realloc(ppszArg, (argc+1)*sizeof(char *));
+      if(ppszArg1 == NULL && ppszArg != NULL)
+            free(ppszArg); 
+      ppszArg = ppszArg1;
       if (!ppszArg) return -1;
       pszCopy[j] = c0 = '\0';
     }
@@ -212,7 +216,7 @@ int _initU(void) {
     fprintf(stderr, "Warning: Can't convert the argument line to UTF-8\n");
     _acmdln[0] = '\0';
   }
-  realloc(_acmdln, n+1); /* Resize the memory block to fit the UTF-8 line */
+  //realloc(_acmdln, n+1); /* Resize the memory block to fit the UTF-8 line */
   /* Should not fail since we make it smaller */
 
   /* Record the console code page, to allow converting the output accordingly */

deps/MsvcLibX/src/realpath.c

@@ -196,6 +196,7 @@ not_compact_enough:
 /* Normally defined in stdlib.h. Output buf must contain PATH_MAX bytes */
 char *realpath(const char *path, char *outbuf) {
   char *pOutbuf = outbuf;
+  char *pOutbuf1 = NULL;
   int iErr;
   const char *pc;
 
@@ -242,8 +243,11 @@ realpath_failed:
     return NULL;
   }
 
-  if (!outbuf) pOutbuf = realloc(pOutbuf, strlen(pOutbuf) + 1);
-  return pOutbuf;
+  if (!outbuf) {
+    pOutbuf1 = realloc(pOutbuf, strlen(pOutbuf) + 1);
+    if(pOutbuf1 == NULL && pOutbuf) free(pOutbuf);
+  }
+  return pOutbuf1;
 }
 
 #endif
@@ -517,6 +521,7 @@ int ResolveLinksA(const char *path, char *buf, size_t bufsize) {
 /* Normally defined in stdlib.h. Output buf must contain PATH_MAX bytes */
 char *realpathU(const char *path, char *outbuf) {
   char *pOutbuf = outbuf;
+  char *pOutbuf1 = NULL;
   char *pPath1 = NULL;
   char *pPath2 = NULL;
   int iErr;
@@ -590,10 +595,13 @@ realpathU_failed:
   }
 
   DEBUG_LEAVE(("return 0x%p; // \"%s\"\n", pOutbuf, pOutbuf));
-  if (!outbuf) pOutbuf = realloc(pOutbuf, strlen(pOutbuf) + 1);
+  if (!outbuf) {
+    pOutbuf1 = realloc(pOutbuf, strlen(pOutbuf) + 1);
+    if(pOutbuf1 == NULL && pOutbuf) free(pOutbuf);
+  }  
   free(pPath1);
   free(pPath2);
-  return pOutbuf;
+  return pOutbuf1;
 }
 
 #endif /* defined(_WIN32) */

snap/snapcraft.yaml

 name: tdengine
 base: core18
-version: '2.1.6.0'
+version: '2.1.7.1'
 icon: snap/gui/t-dengine.svg
 summary: an open-source big data platform designed and optimized for IoT. 
 description: |
@@ -72,7 +72,7 @@ parts:
       - usr/bin/taosd
       - usr/bin/taos
       - usr/bin/taosdemo
-      - usr/lib/libtaos.so.2.1.6.0
+      - usr/lib/libtaos.so.2.1.7.1
       - usr/lib/libtaos.so.1
       - usr/lib/libtaos.so
 

src/balance/src/bnScore.c

@@ -116,8 +116,17 @@ void bnCleanupDnodes() {
 
 static void bnCheckDnodesSize(int32_t dnodesNum) {
   if (tsBnDnodes.maxSize <= dnodesNum) {
-    tsBnDnodes.maxSize = dnodesNum * 2;
-    tsBnDnodes.list = realloc(tsBnDnodes.list, tsBnDnodes.maxSize * sizeof(SDnodeObj *));
+    int32_t maxSize = dnodesNum * 2;
+    SDnodeObj** list1 = NULL;
+    int32_t retry = 0;
+    
+    while(list1 == NULL && retry++ < 3) {
+      list1 = realloc(tsBnDnodes.list, maxSize * sizeof(SDnodeObj *));
+    }    
+    if(list1) {
+      tsBnDnodes.list = list1;
+      tsBnDnodes.maxSize = maxSize;
+    }
   }
 }
 

src/client/inc/tscUtil.h

@@ -29,15 +29,16 @@ extern "C" {
 #include "tsched.h"
 #include "tsclient.h"
 
-#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo)  \
+#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo) \
   (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE))
+
 #define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \
   (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE))
-  
-#define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo)\
-  (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo)))
 
-#define UTIL_TABLE_IS_TMP_TABLE(metaInfo)  \
+#define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo) \
+  (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo) || UTIL_TABLE_IS_TMP_TABLE(metaInfo)))
+
+#define UTIL_TABLE_IS_TMP_TABLE(metaInfo) \
   (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_TEMP_TABLE))
 
 #pragma pack(push,1)
@@ -142,6 +143,7 @@ bool tscIsSessionWindowQuery(SQueryInfo* pQueryInfo);
 bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo);
 bool tsIsArithmeticQueryOnAggResult(SQueryInfo* pQueryInfo);
 bool tscGroupbyColumn(SQueryInfo* pQueryInfo);
+int32_t tscGetTopBotQueryExprIndex(SQueryInfo* pQueryInfo);
 bool tscIsTopBotQuery(SQueryInfo* pQueryInfo);
 bool hasTagValOutput(SQueryInfo* pQueryInfo);
 bool timeWindowInterpoRequired(SQueryInfo *pQueryInfo);

src/client/src/tscPrepare.c

@@ -1527,8 +1527,9 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
   pCmd->insertParam.insertType = TSDB_QUERY_TYPE_STMT_INSERT;
   pCmd->insertParam.objectId = pSql->self;
 
-  pSql->sqlstr = realloc(pSql->sqlstr, sqlLen + 1);
-
+  char* sqlstr = realloc(pSql->sqlstr, sqlLen + 1);
+  if(sqlstr == NULL && pSql->sqlstr) free(pSql->sqlstr);  
+  pSql->sqlstr = sqlstr;
   if (pSql->sqlstr == NULL) {
     tscError("%p failed to malloc sql string buffer", pSql);
     STMT_RET(TSDB_CODE_TSC_OUT_OF_MEMORY);

src/client/src/tscServer.c

@@ -409,7 +409,7 @@ static void doProcessMsgFromServer(SSchedMsg* pSchedMsg) {
     if ((TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY |
                                                TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) &&
                                                !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) ||
-                                               (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY))) {
+                                               (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY)) || (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) &&  pQueryInfo->distinct)) {
       // do nothing in case of super table subquery
     }  else {
       pSql->retry += 1;
@@ -880,16 +880,16 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
   }
 
   SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
+  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+  STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
 
   SQueryAttr query = {{0}};
   tscCreateQueryFromQueryInfo(pQueryInfo, &query, pSql);
+  query.vgId = pTableMeta->vgId;
 
   SArray* tableScanOperator = createTableScanPlan(&query);
   SArray* queryOperator = createExecOperatorPlan(&query);
 
-  STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
-  STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
-
   SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload;
   tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version));
 

src/client/src/tscSql.c

@@ -887,7 +887,9 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
     return TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
   }
 
-  pSql->sqlstr = realloc(pSql->sqlstr, sqlLen + 1);
+  char* sqlstr = realloc(pSql->sqlstr, sqlLen + 1);
+  if(sqlstr == NULL && pSql->sqlstr) free(pSql->sqlstr);
+  pSql->sqlstr = sqlstr;
   if (pSql->sqlstr == NULL) {
     tscError("0x%"PRIx64" failed to malloc sql string buffer", pSql->self);
     tfree(pSql);

src/client/src/tscSubquery.c

@@ -2397,6 +2397,10 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) {
         } else {
           SSchema ss = {.type = (uint8_t)pCol->info.type, .bytes = pCol->info.bytes, .colId = (int16_t)pCol->columnIndex};
           tscColumnListInsert(pNewQueryInfo->colList, pCol->columnIndex, pCol->tableUid, &ss);
+          int32_t ti = tscColumnExists(pNewQueryInfo->colList, pCol->columnIndex, pCol->tableUid);
+          assert(ti >= 0);
+          SColumn* x = taosArrayGetP(pNewQueryInfo->colList, ti);
+          tscColumnCopy(x, pCol);
         }
       }
     }
@@ -2812,7 +2816,6 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
       pParentSql->self, pState->numOfSub, pState->numOfRetrievedRows);
   
   SQueryInfo *pPQueryInfo = tscGetQueryInfo(&pParentSql->cmd);
-  tscClearInterpInfo(pPQueryInfo);
   
   code = tscCreateGlobalMerger(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, pPQueryInfo, &pParentSql->res.pMerger, pParentSql->self);
   pParentSql->res.code = code;

src/client/src/tscUtil.c

@@ -369,6 +369,27 @@ bool tscGroupbyColumn(SQueryInfo* pQueryInfo) {
   return false;
 }
 
+int32_t tscGetTopBotQueryExprIndex(SQueryInfo* pQueryInfo) {
+  size_t numOfExprs = tscNumOfExprs(pQueryInfo);
+
+  for (int32_t i = 0; i < numOfExprs; ++i) {
+    SExprInfo* pExpr = tscExprGet(pQueryInfo, i);
+    if (pExpr == NULL) {
+      continue;
+    }
+
+    if (pExpr->base.functionId == TSDB_FUNC_TS) {
+      continue;
+    }
+
+    if (pExpr->base.functionId == TSDB_FUNC_TOP || pExpr->base.functionId == TSDB_FUNC_BOTTOM) {
+      return i;
+    }
+  }
+
+  return -1;
+}
+
 bool tscIsTopBotQuery(SQueryInfo* pQueryInfo) {
   size_t numOfExprs = tscNumOfExprs(pQueryInfo);
 
@@ -625,8 +646,10 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo
 
   } else if (convertNchar && pInfo->field.type == TSDB_DATA_TYPE_NCHAR) {
     // convert unicode to native code in a temporary buffer extra one byte for terminated symbol
-    pRes->buffer[i] = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
-
+    char* buffer = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
+    if(buffer == NULL)
+       return ;
+    pRes->buffer[i] = buffer;
     // string terminated char for binary data
     memset(pRes->buffer[i], 0, pInfo->field.bytes * pRes->numOfRows);
 
@@ -1206,7 +1229,6 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue
     createInputDataFilterInfo(px, numOfCol1, &numOfFilterCols, &pFilterInfo);
 
     SOperatorInfo* pSourceOperator = createDummyInputOperator(pSqlObjList[0], pSchema, numOfCol1, pFilterInfo, numOfFilterCols);
-
     pOutput->precision = pSqlObjList[0]->res.precision;
 
     SSchema* schema = NULL;
@@ -1502,7 +1524,6 @@ void tscFreeSqlObj(SSqlObj* pSql) {
   tscFreeSqlResult(pSql);
   tscResetSqlCmd(pCmd, false);
 
-  memset(pCmd->payload, 0, (size_t)pCmd->allocSize);
   tfree(pCmd->payload);
   pCmd->allocSize = 0;
 
@@ -3532,8 +3553,10 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t
   pNewQueryInfo->numOfTables = 0;
   pNewQueryInfo->pTableMetaInfo = NULL;
   pNewQueryInfo->bufLen = pQueryInfo->bufLen;
-
   pNewQueryInfo->buf = malloc(pQueryInfo->bufLen);
+
+
+  pNewQueryInfo->distinct =  pQueryInfo->distinct;
   if (pNewQueryInfo->buf == NULL) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     goto _error;
@@ -3830,6 +3853,9 @@ void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) {
       pNew->sqlstr    = strdup(pSql->sqlstr);
       pNew->fp        = tscSubqueryCompleteCallback;
       pNew->maxRetry  = pSql->maxRetry;
+
+      pNew->cmd.resColumnId = TSDB_RES_COL_ID;
+      
       tsem_init(&pNew->rspSem, 0, 0);
 
       SRetrieveSupport* ps = calloc(1, sizeof(SRetrieveSupport));  // todo use object id
@@ -4364,6 +4390,7 @@ int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name,
   STableMeta* p    = NULL;
   size_t      sz   = 0;
   STableMeta* pChild = *ppChild;
+  STableMeta* pChild1;
    
   taosHashGetCloneExt(tscTableMetaMap, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, (void **)&p, &sz);
 
@@ -4374,7 +4401,10 @@ int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name,
     int32_t totalBytes    = (p->tableInfo.numOfColumns + p->tableInfo.numOfTags) * sizeof(SSchema);
     int32_t tableMetaSize =  sizeof(STableMeta)  + totalBytes;
     if (*tableMetaCapacity < tableMetaSize) {
-      pChild = realloc(pChild, tableMetaSize); 
+      pChild1 = realloc(pChild, tableMetaSize);
+      if(pChild1 == NULL) 
+        return -1;
+      pChild = pChild1; 
       *tableMetaCapacity = (size_t)tableMetaSize;
     }
 

src/common/inc/tdataformat.h

@@ -547,8 +547,9 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
 static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, void *value) {
   if (pBuilder->nCols >= pBuilder->tCols) {
     pBuilder->tCols *= 2;
-    pBuilder->pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);
-    if (pBuilder->pColIdx == NULL) return -1;
+    SColIdx* pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);
+    if (pColIdx == NULL) return -1;
+    pBuilder->pColIdx = pColIdx;
   }
 
   pBuilder->pColIdx[pBuilder->nCols].colId = colId;
@@ -561,8 +562,9 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId,
     while (tlen > pBuilder->alloc - pBuilder->size) {
       pBuilder->alloc *= 2;
     }
-    pBuilder->buf = realloc(pBuilder->buf, pBuilder->alloc);
-    if (pBuilder->buf == NULL) return -1;
+    void* buf = realloc(pBuilder->buf, pBuilder->alloc);
+    if (buf == NULL) return -1;
+    pBuilder->buf = buf;
   }
 
   memcpy(POINTER_SHIFT(pBuilder->buf, pBuilder->size), value, tlen);

src/common/inc/tglobal.h

@@ -59,6 +59,7 @@ extern char     tsLocale[];
 extern char     tsCharset[];            // default encode string
 extern int8_t   tsEnableCoreFile;
 extern int32_t  tsCompressMsgSize;
+extern int32_t  tsMaxNumOfDistinctResults;
 extern char     tsTempDir[];
 
 //query buffer management

src/common/src/tdataformat.c

@@ -138,8 +138,9 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int1
 
   if (pBuilder->nCols >= pBuilder->tCols) {
     pBuilder->tCols *= 2;
-    pBuilder->columns = (STColumn *)realloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols);
-    if (pBuilder->columns == NULL) return -1;
+    STColumn* columns = (STColumn *)realloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols);
+    if (columns == NULL) return -1;
+    pBuilder->columns = columns;
   }
 
   STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]);

src/common/src/tglobal.c

@@ -79,7 +79,7 @@ int32_t tsCompressMsgSize = -1;
 
 // client
 int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN;
-int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_MAX_LEN;
+int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN;
 int8_t  tsTscEnableRecordSql = 0;
 
 // the maximum number of results for projection query on super table that are returned from
@@ -89,6 +89,9 @@ int32_t tsMaxNumOfOrderedResults = 100000;
 // 10 ms for sliding time, the value will changed in case of time precision changed
 int32_t tsMinSlidingTime = 10;
 
+// the maxinum number of distict query result
+int32_t tsMaxNumOfDistinctResults  = 1000 * 10000;
+
 // 1 us for interval time range, changed accordingly
 int32_t tsMinIntervalTime = 1;
 
@@ -546,6 +549,16 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "maxNumOfDistinctRes";
+  cfg.ptr = &tsMaxNumOfDistinctResults;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.minValue = 10*10000;
+  cfg.maxValue = 10000*10000;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+  
   cfg.option = "numOfMnodes";
   cfg.ptr = &tsNumOfMnodes;
   cfg.valType = TAOS_CFG_VTYPE_INT32;

go @ 050667e5

-Subproject commit b8f76da4a708d158ec3cc4b844571dc4414e36b4
+Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f

grafanaplugin @ 32e2c97a

-Subproject commit a44ec1ca493ad01b2bf825b6418f69e11f548206
+Subproject commit 32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df

hivemq-tdengine-extension @ b62a26ec

-Subproject commit ce5201014136503d34fecbd56494b67b4961056c
+Subproject commit b62a26ecc164a310104df57691691b237e091c89

src/connector/nodejs/nodetaos/cinterface.js

@@ -109,6 +109,24 @@ function convertDouble(data, num_of_rows, nbytes = 0, offset = 0, precision = 0)
   return res;
 }
 
+function convertBinary(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) {
+  data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset);
+  let res = [];
+
+  let currOffset = 0;
+  while (currOffset < data.length) {
+    let len = data.readIntLE(currOffset, 2);
+    let dataEntry = data.slice(currOffset + 2, currOffset + len + 2); //one entry in a row under a column;
+    if (dataEntry[0] == 255) {
+      res.push(null)
+    } else {
+      res.push(dataEntry.toString("utf-8"));
+    }
+    currOffset += nbytes;
+  }
+  return res;
+}
+
 function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, precision = 0) {
   data = ref.reinterpret(data.deref(), nbytes * num_of_rows, offset);
   let res = [];
@@ -117,7 +135,11 @@ function convertNchar(data, num_of_rows, nbytes = 0, offset = 0, precision = 0)
   while (currOffset < data.length) {
     let len = data.readIntLE(currOffset, 2);
     let dataEntry = data.slice(currOffset + 2, currOffset + len + 2); //one entry in a row under a column;
-    res.push(dataEntry.toString("utf-8"));
+    if (dataEntry[0] == 255 && dataEntry[1] == 255) {
+      res.push(null)
+    } else {
+      res.push(dataEntry.toString("utf-8"));
+    }
     currOffset += nbytes;
   }
   return res;
@@ -132,7 +154,7 @@ let convertFunctions = {
   [FieldTypes.C_BIGINT]: convertBigint,
   [FieldTypes.C_FLOAT]: convertFloat,
   [FieldTypes.C_DOUBLE]: convertDouble,
-  [FieldTypes.C_BINARY]: convertNchar,
+  [FieldTypes.C_BINARY]: convertBinary,
   [FieldTypes.C_TIMESTAMP]: convertTimestamp,
   [FieldTypes.C_NCHAR]: convertNchar
 }

src/connector/nodejs/test/testnchar.js

+const taos = require('../tdengine');
+var conn = taos.connect({ host: "localhost" });
+var c1 = conn.cursor();
+
+
+function checkData(data, row, col, expect) {
+	let checkdata = data[row][col];
+	if (checkdata == expect) {
+		// console.log('check pass')
+	}
+	else {
+		console.log('check failed, expect ' + expect + ', but is ' + checkdata)
+	}
+}
+
+c1.execute('drop database if exists testnodejsnchar')
+c1.execute('create database testnodejsnchar')
+c1.execute('use testnodejsnchar');
+c1.execute('create table tb (ts timestamp, value float, text binary(200))')
+c1.execute("insert into tb values('2021-06-10 00:00:00', 24.7, '中文10000000000000000000000');") -
+c1.execute('insert into tb values(1623254400150, 24.7, NULL);')
+c1.execute('import into tb values(1623254400300, 24.7, "中文3中文10000000000000000000000中文10000000000000000000000中文10000000000000000000000中文10000000000000000000000");')
+sql = 'select * from tb;'
+
+console.log('*******************************************')
+
+c1.execute(sql);
+data = c1.fetchall();
+console.log(data)
+//check data about insert data
+checkData(data, 0, 2, '中文10000000000000000000000')
+checkData(data, 1, 2, null)
+checkData(data, 2, 2, '中文3中文10000000000000000000000中文10000000000000000000000中文10000000000000000000000中文10000000000000000000000')
\ No newline at end of file

src/inc/twal.h

@@ -32,7 +32,7 @@ typedef enum {
 
 typedef struct {
   int8_t   msgType;
-  int8_t   sver;
+  int8_t   sver;  // sver 2 for WAL SDataRow/SMemRow compatibility
   int8_t   reserved[2];
   int32_t  len;
   uint64_t version;

src/kit/shell/src/shellCheck.c

@@ -72,12 +72,13 @@ static int32_t shellShowTables(TAOS *con, char *db) {
       int32_t tbIndex = tbNum++;
       if (tbMallocNum < tbNum) {
         tbMallocNum = (tbMallocNum * 2 + 1);
-        tbNames = realloc(tbNames, tbMallocNum * sizeof(char *));
-        if (tbNames == NULL) {
+        char** tbNames1 = realloc(tbNames, tbMallocNum * sizeof(char *));
+        if (tbNames1 == NULL) {
           fprintf(stdout, "failed to malloc tablenames, num:%d\n", tbMallocNum);
           code = TSDB_CODE_TSC_OUT_OF_MEMORY;
           break;
         }
+        tbNames = tbNames1;
       }
 
       tbNames[tbIndex] = malloc(TSDB_TABLE_NAME_LEN);

src/kit/taospack/taospack.c

@@ -149,7 +149,10 @@ float* read_float(const char* inFile, int* pcount){
     //printf(" buff=%s float=%.50f \n ", buf, floats[fi]);
     if ( ++fi == malloc_cnt ) {
       malloc_cnt += 100000;
-      floats = realloc(floats, malloc_cnt*sizeof(float));
+      float* floats1 = realloc(floats, malloc_cnt*sizeof(float));
+      if(floats1 == NULL)
+         break;
+      floats = floats1;   
     }
     memset(buf, 0, sizeof(buf));
   }

src/mnode/src/mnodeTable.c

@@ -2921,10 +2921,11 @@ static SMultiTableMeta* ensureMsgBufferSpace(SMultiTableMeta *pMultiMeta, SArray
       (*totalMallocLen) *= 2;
     }
 
-    pMultiMeta = realloc(pMultiMeta, *totalMallocLen);
-    if (pMultiMeta == NULL) {
+    SMultiTableMeta* pMultiMeta1 = realloc(pMultiMeta, *totalMallocLen);
+    if (pMultiMeta1 == NULL) {
       return NULL;
     }
+    pMultiMeta = pMultiMeta1;
   }
 
   return pMultiMeta;

src/os/src/detail/osMemory.c

@@ -504,8 +504,9 @@ void * taosTRealloc(void *ptr, size_t size) {
 
   void * tptr = (void *)((char *)ptr - sizeof(size_t));
   size_t tsize = size + sizeof(size_t);
-  tptr = realloc(tptr, tsize);
-  if (tptr == NULL) return NULL;
+  void* tptr1 = realloc(tptr, tsize);
+  if (tptr1 == NULL) return NULL;
+  tptr = tptr1;
 
   *(size_t *)tptr = size;
 

src/os/src/windows/wGetline.c

@@ -81,11 +81,13 @@ int32_t getstr(char **lineptr, size_t *n, FILE *stream, char terminator, int32_t
         *n += MIN_CHUNK;
 
       nchars_avail = (int32_t)(*n + *lineptr - read_pos);
-      *lineptr = realloc(*lineptr, *n);
-      if (!*lineptr) {
+      char* lineptr1 = realloc(*lineptr, *n);
+      if (!lineptr1) {
         errno = ENOMEM;
         return -1;
       }
+      *lineptr = lineptr1;
+
       read_pos = *n - nchars_avail + *lineptr;
       assert((*lineptr + *n) == (read_pos + nchars_avail));
     }

src/query/inc/qExecutor.h

@@ -333,6 +333,7 @@ enum OPERATOR_TYPE_E {
   OP_StateWindow       = 22,
   OP_AllTimeWindow     = 23,
   OP_AllMultiTableTimeInterval = 24,
+  OP_Order             = 25,
 };
 
 typedef struct SOperatorInfo {
@@ -417,7 +418,6 @@ typedef struct STableScanInfo {
   int32_t        *rowCellInfoOffset;
   SExprInfo      *pExpr;
   SSDataBlock     block;
-  bool            loadExternalRows; // load external rows (prev & next rows)
   int32_t         numOfOutput;
   int64_t         elapsedTime;
 
@@ -510,13 +510,21 @@ typedef struct SStateWindowOperatorInfo {
   bool           reptScan;
 } SStateWindowOperatorInfo ;
 
+typedef struct SDistinctDataInfo {
+  int32_t index;
+  int32_t type;
+  int32_t bytes;
+} SDistinctDataInfo;
+
 typedef struct SDistinctOperatorInfo {
   SHashObj         *pSet;
   SSDataBlock      *pRes;
   bool              recordNullVal;  //has already record the null value, no need to try again
   int64_t           threshold;
   int64_t           outputCapacity;
-  int32_t           colIndex; 
+  int32_t           totalBytes;
+  char*             buf;
+  SArray*           pDistinctDataInfo;
 } SDistinctOperatorInfo;
 
 struct SGlobalMerger;
@@ -541,6 +549,13 @@ typedef struct SMultiwayMergeInfo {
   SArray              *udfInfo;
 } SMultiwayMergeInfo;
 
+// todo support the disk-based sort
+typedef struct SOrderOperatorInfo {
+  int32_t      colIndex;
+  int32_t      order;
+  SSDataBlock *pDataBlock;
+} SOrderOperatorInfo;
+
 void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream);
 
 SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime);
@@ -570,6 +585,7 @@ SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperator
                                         int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter);
 
 SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput);
+SOperatorInfo* createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal);
 
 SSDataBlock* doGlobalAggregate(void* param, bool* newgroup);
 SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup);

src/query/inc/qExtbuffer.h

@@ -220,6 +220,8 @@ tOrderDescriptor *tOrderDesCreate(const int32_t *orderColIdx, int32_t numOfOrder
 
 void tOrderDescDestroy(tOrderDescriptor *pDesc);
 
+void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn);
+
 void tColModelAppend(SColumnModel *dstModel, tFilePage *dstPage, void *srcData, int32_t srcStartRows,
                      int32_t numOfRowsToWrite, int32_t srcCapacity);
 

src/query/src/qAggMain.c

@@ -3670,6 +3670,8 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) {
     return;
   }
 
+  bool ascQuery = (pCtx->order == TSDB_ORDER_ASC);
+
   if (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) {
     *(TSKEY *)pCtx->pOutput = pCtx->startTs;
   } else if (type == TSDB_FILL_NULL) {
@@ -3677,7 +3679,7 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) {
   } else if (type == TSDB_FILL_SET_VALUE) {
     tVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true);
   } else {
-    if (pCtx->start.key != INT64_MIN && pCtx->start.key < pCtx->startTs && pCtx->end.key > pCtx->startTs) {
+    if (pCtx->start.key != INT64_MIN && ((ascQuery && pCtx->start.key <= pCtx->startTs && pCtx->end.key >= pCtx->startTs) || ((!ascQuery) && pCtx->start.key >= pCtx->startTs && pCtx->end.key <= pCtx->startTs))) {
       if (type == TSDB_FILL_PREV) {
         if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) {
           SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->start.val);
@@ -3716,13 +3718,14 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) {
       TSKEY skey = GET_TS_DATA(pCtx, 0);
 
       if (type == TSDB_FILL_PREV) {
-        if (skey > pCtx->startTs) {
+        if ((ascQuery && skey > pCtx->startTs) || ((!ascQuery) && skey < pCtx->startTs)) {
           return;
         }
 
         if (pCtx->size > 1) {
           TSKEY ekey = GET_TS_DATA(pCtx, 1);
-          if (ekey > skey && ekey <= pCtx->startTs) {
+          if ((ascQuery &&  ekey > skey && ekey <= pCtx->startTs) ||
+             ((!ascQuery) && ekey < skey && ekey >= pCtx->startTs)){
             skey = ekey;
           }
         }
@@ -3731,10 +3734,10 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) {
         TSKEY ekey = skey;
         char* val = NULL;
         
-        if (ekey < pCtx->startTs) {
+        if ((ascQuery && ekey < pCtx->startTs) || ((!ascQuery) && ekey > pCtx->startTs)) {
           if (pCtx->size > 1) {
             ekey = GET_TS_DATA(pCtx, 1);
-            if (ekey < pCtx->startTs) {
+            if ((ascQuery && ekey < pCtx->startTs) || ((!ascQuery) && ekey > pCtx->startTs)) {
               return;
             }
 
@@ -3755,12 +3758,11 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) {
         TSKEY ekey = GET_TS_DATA(pCtx, 1);
       
         // no data generated yet
-        if (!(skey < pCtx->startTs && ekey > pCtx->startTs)) {
+        if ((ascQuery && !(skey <= pCtx->startTs && ekey >= pCtx->startTs))
+           || ((!ascQuery) && !(skey >= pCtx->startTs && ekey <= pCtx->startTs))) {
           return;
         }
         
-        assert(pCtx->start.key == INT64_MIN && skey < pCtx->startTs && ekey > pCtx->startTs);
-        
         char *start = GET_INPUT_DATA(pCtx, 0);
         char *end = GET_INPUT_DATA(pCtx, 1);
 
@@ -3788,11 +3790,37 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) {
 static void interp_function(SQLFunctionCtx *pCtx) {
   // at this point, the value is existed, return directly
   if (pCtx->size > 0) {
-    // impose the timestamp check
-    TSKEY key = GET_TS_DATA(pCtx, 0);
+    bool ascQuery = (pCtx->order == TSDB_ORDER_ASC);
+    TSKEY key;
+    char *pData;
+    int32_t typedData = 0;
+    
+    if (ascQuery) {
+      key = GET_TS_DATA(pCtx, 0);
+      pData = GET_INPUT_DATA(pCtx, 0);
+    } else {
+      key = pCtx->start.key;
+      if (key == INT64_MIN) {
+        key = GET_TS_DATA(pCtx, 0);
+        pData = GET_INPUT_DATA(pCtx, 0);
+      } else {        
+        if (!(IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL)) {
+          pData = pCtx->start.ptr;
+        } else {
+          typedData = 1;
+          pData = (char *)&pCtx->start.val;
+        }
+      }
+    }
+    
+    //if (key == pCtx->startTs && (ascQuery || !(IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL))) {
     if (key == pCtx->startTs) {
-      char *pData = GET_INPUT_DATA(pCtx, 0);
-      assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType);
+      if (typedData) {
+        SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, *(double *)pData);
+      } else {
+        assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType);
+      }
+      
       SET_VAL(pCtx, 1, 1);
     } else {
       interp_function_impl(pCtx);

src/query/src/qExtbuffer.c

@@ -1102,3 +1102,57 @@ void tOrderDescDestroy(tOrderDescriptor *pDesc) {
   destroyColumnModel(pDesc->pColumnModel);
   tfree(pDesc);
 }
+
+void taoscQSort(void** pCols, SSchema* pSchema, int32_t numOfCols, int32_t numOfRows, int32_t index, __compar_fn_t compareFn) {
+  assert(numOfRows > 0 && numOfCols > 0 && index >= 0 && index < numOfCols);
+
+  int32_t bytes = pSchema[index].bytes;
+  int32_t size = bytes + sizeof(int32_t);
+
+  char* buf = calloc(1, size * numOfRows);
+
+  for(int32_t i = 0; i < numOfRows; ++i) {
+    char* dest = buf + size * i;
+    memcpy(dest, ((char*) pCols[index]) + bytes * i, bytes);
+    *(int32_t*)(dest+bytes) = i;
+  }
+
+  qsort(buf, numOfRows, size, compareFn);
+
+  int32_t prevLength = 0;
+  char* p = NULL;
+
+  for(int32_t i = 0; i < numOfCols; ++i) {
+    int32_t bytes1 = pSchema[i].bytes;
+
+    if (i == index) {
+      for(int32_t j = 0; j < numOfRows; ++j){
+        char* src  = buf + (j * size);
+        char* dest = ((char*)pCols[i]) + (j * bytes1);
+        memcpy(dest, src, bytes1);
+      }
+    } else {
+      // make sure memory buffer is enough
+      if (prevLength < bytes1) {
+        char *tmp = realloc(p, bytes1 * numOfRows);
+        assert(tmp);
+
+        p = tmp;
+        prevLength = bytes1;
+      }
+
+      memcpy(p, pCols[i], bytes1 * numOfRows);
+
+      for(int32_t j = 0; j < numOfRows; ++j){
+        char* dest = ((char*)pCols[i]) + bytes1 * j;
+
+        int32_t newPos = *(int32_t*)(buf + (j * size) + bytes);
+        char* src = p + (newPos * bytes1);
+        memcpy(dest, src, bytes1);
+      }
+    }
+  }
+
+  tfree(buf);
+  tfree(p);
+}
\ No newline at end of file

src/query/src/qPercentile.c

@@ -237,7 +237,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval,
   }
 
   pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(tFilePage))/pBucket->bytes;
-  pBucket->comparFn = getKeyComparFunc(pBucket->type);
+  pBucket->comparFn = getKeyComparFunc(pBucket->type, TSDB_ORDER_ASC);
 
   pBucket->hashFunc = getHashFunc(pBucket->type);
   if (pBucket->hashFunc == NULL) {

src/query/src/qPlan.c

@@ -557,10 +557,9 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) {
   int32_t op = 0;
 
   if (onlyQueryTags(pQueryAttr)) {  // do nothing for tags query
-    if (onlyQueryTags(pQueryAttr)) {
-      op = OP_TagScan;
-      taosArrayPush(plan, &op);
-    }
+    op = OP_TagScan;
+    taosArrayPush(plan, &op);
+
     if (pQueryAttr->distinct) {
       op = OP_Distinct;
       taosArrayPush(plan, &op);
@@ -651,8 +650,14 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) {
         taosArrayPush(plan, &op);
       }
     }
+
+    // outer query order by support
+    int32_t orderColId = pQueryAttr->order.orderColId;
+    if (pQueryAttr->vgId == 0 && orderColId != PRIMARYKEY_TIMESTAMP_COL_INDEX && orderColId != INT32_MIN) {
+      op = OP_Order;
+      taosArrayPush(plan, &op);
+    }
   }
- 
 
   if (pQueryAttr->limit.limit > 0 || pQueryAttr->limit.offset > 0) {
     op = OP_Limit;
@@ -693,7 +698,7 @@ SArray* createGlobalMergePlan(SQueryAttr* pQueryAttr) {
   }
 
   // fill operator
-  if (pQueryAttr->fillType != TSDB_FILL_NONE && (!pQueryAttr->pointInterpQuery)) {
+  if (pQueryAttr->fillType != TSDB_FILL_NONE && pQueryAttr->interval.interval > 0) {
     op = OP_Fill;
     taosArrayPush(plan, &op);
   }

src/query/src/qTsbuf.c

@@ -223,8 +223,11 @@ static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) {
 static void shrinkBuffer(STSList* ptsData) {
   // shrink tmp buffer size if it consumes too many memory compared to the pre-defined size
   if (ptsData->allocSize >= ptsData->threshold * 2) {
-    ptsData->rawBuf = realloc(ptsData->rawBuf, MEM_BUF_SIZE);
-    ptsData->allocSize = MEM_BUF_SIZE;
+    char* rawBuf = realloc(ptsData->rawBuf, MEM_BUF_SIZE);
+    if(rawBuf) {
+      ptsData->rawBuf = rawBuf;
+      ptsData->allocSize = MEM_BUF_SIZE;
+    }
   }
 }
 

src/util/inc/tcompare.h

@@ -22,10 +22,10 @@ extern "C" {
 
 #include "os.h"
 
-#define TSDB_PATTERN_MATCH 0
-#define TSDB_PATTERN_NOMATCH 1
-#define TSDB_PATTERN_NOWILDCARDMATCH 2
-#define TSDB_PATTERN_STRING_MAX_LEN 100
+#define TSDB_PATTERN_MATCH               0
+#define TSDB_PATTERN_NOMATCH             1
+#define TSDB_PATTERN_NOWILDCARDMATCH     2
+#define TSDB_PATTERN_STRING_DEFAULT_LEN  100
 
 #define FLT_COMPAR_TOL_FACTOR    4
 #define FLT_EQUAL(_x, _y)        (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON))
@@ -47,7 +47,7 @@ int WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size, con
 
 int32_t doCompare(const char* a, const char* b, int32_t type, size_t size);
 
-__compar_fn_t getKeyComparFunc(int32_t keyType);
+__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
 
 __compar_fn_t getComparFunc(int32_t type, int32_t optr);
 

src/util/src/tcompare.c

@@ -16,58 +16,101 @@
 #include "os.h"
 #include "ttype.h"
 #include "tcompare.h"
-#include "tarray.h"
 #include "hash.h"
 
-int32_t compareInt32Val(const void *pLeft, const void *pRight) {
-  int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);
+int32_t setCompareBytes1(const void *pLeft, const void *pRight) {
+  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
+}
+
+int32_t setCompareBytes2(const void *pLeft, const void *pRight) {
+  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 2) ? 1 : 0;
+}
+
+int32_t setCompareBytes4(const void *pLeft, const void *pRight) {
+  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 4) ? 1 : 0;
+}
+
+int32_t setCompareBytes8(const void *pLeft, const void *pRight) {
+  return NULL != taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0;
+}
+
+int32_t compareInt8Val(const void *pLeft, const void *pRight) {
+  int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight);
   if (left > right) return 1;
   if (left < right) return -1;
   return 0;
 }
 
-int32_t compareInt64Val(const void *pLeft, const void *pRight) {
-  int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight);
+int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) {
+  return compareInt8Val(pRight, pLeft);
+}
+
+int32_t compareInt16Val(const void *pLeft, const void *pRight) {
+  int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight);
   if (left > right) return 1;
   if (left < right) return -1;
   return 0;
 }
 
-int32_t compareInt16Val(const void *pLeft, const void *pRight) {
-  int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight);
+int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) {
+  return compareInt16Val(pRight, pLeft);
+}
+
+int32_t compareInt32Val(const void *pLeft, const void *pRight) {
+  int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);
   if (left > right) return 1;
   if (left < right) return -1;
   return 0;
 }
 
-int32_t compareInt8Val(const void *pLeft, const void *pRight) {
-  int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight);
+int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) {
+  return compareInt32Val(pRight, pLeft);
+}
+
+int32_t compareInt64Val(const void *pLeft, const void *pRight) {
+  int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight);
   if (left > right) return 1;
   if (left < right) return -1;
   return 0;
 }
 
+int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) {
+  return compareInt64Val(pRight, pLeft);
+}
+
 int32_t compareUint32Val(const void *pLeft, const void *pRight) {
-  int32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight);
+  uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight);
   if (left > right) return 1;
   if (left < right) return -1;
   return 0;
 }
 
+int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) {
+  return compareUint32Val(pRight, pLeft);
+}
+
 int32_t compareUint64Val(const void *pLeft, const void *pRight) {
-  int64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight);
+  uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight);
   if (left > right) return 1;
   if (left < right) return -1;
   return 0;
 }
 
+int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) {
+  return compareUint64Val(pRight, pLeft);
+}
+
 int32_t compareUint16Val(const void *pLeft, const void *pRight) {
-  int16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight);
+  uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight);
   if (left > right) return 1;
   if (left < right) return -1;
   return 0;
 }
 
+int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) {
+  return compareUint16Val(pRight, pLeft);
+}
+
 int32_t compareUint8Val(const void* pLeft, const void* pRight) {
   uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight);
   if (left > right) return 1;
@@ -75,6 +118,10 @@ int32_t compareUint8Val(const void* pLeft, const void* pRight) {
   return 0;
 }
 
+int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) {
+  return compareUint8Val(pRight, pLeft);
+}
+
 int32_t compareFloatVal(const void *pLeft, const void *pRight) {
   float p1 = GET_FLOAT_VAL(pLeft);
   float p2 = GET_FLOAT_VAL(pRight);
@@ -92,8 +139,12 @@ int32_t compareFloatVal(const void *pLeft, const void *pRight) {
   }
   if (FLT_EQUAL(p1, p2)) {
     return 0;
-  } 
-  return FLT_GREATER(p1, p2) ? 1: -1; 
+  }
+  return FLT_GREATER(p1, p2) ? 1: -1;
+}
+
+int32_t compareFloatValDesc(const void* pLeft, const void* pRight) {
+  return compareFloatVal(pRight, pLeft);
 }
 
 int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
@@ -113,14 +164,18 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) {
   }
   if (FLT_EQUAL(p1, p2)) {
     return 0;
-  } 
-  return FLT_GREATER(p1, p2) ? 1: -1; 
+  }
+  return FLT_GREATER(p1, p2) ? 1: -1;
+}
+
+int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) {
+  return compareDoubleVal(pRight, pLeft);
 }
 
 int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {
   int32_t len1 = varDataLen(pLeft);
   int32_t len2 = varDataLen(pRight);
-  
+
   if (len1 != len2) {
     return len1 > len2? 1:-1;
   } else {
@@ -133,14 +188,18 @@ int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) {
   }
 }
 
+int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) {
+  return compareLenPrefixedStr(pRight, pLeft);
+}
+
 int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
   int32_t len1 = varDataLen(pLeft);
   int32_t len2 = varDataLen(pRight);
-  
+
   if (len1 != len2) {
     return len1 > len2? 1:-1;
   } else {
-    int32_t ret = wcsncmp(varDataVal(pLeft), varDataVal(pRight), len1/TSDB_NCHAR_SIZE);
+    int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1);
     if (ret == 0) {
       return 0;
     } else {
@@ -149,6 +208,10 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
   }
 }
 
+int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) {
+  return compareLenPrefixedWStr(pRight, pLeft);
+}
+
 /*
  * Compare two strings
  *    TSDB_MATCH:            Match
@@ -161,33 +224,33 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
  */
 int patternMatch(const char *patterStr, const char *str, size_t size, const SPatternCompareInfo *pInfo) {
   char c, c1;
-  
+
   int32_t i = 0;
   int32_t j = 0;
-  
+
   while ((c = patterStr[i++]) != 0) {
     if (c == pInfo->matchAll) { /* Match "*" */
-      
+
       while ((c = patterStr[i++]) == pInfo->matchAll || c == pInfo->matchOne) {
         if (c == pInfo->matchOne && (j > size || str[j++] == 0)) {
           // empty string, return not match
           return TSDB_PATTERN_NOWILDCARDMATCH;
         }
       }
-      
+
       if (c == 0) {
         return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */
       }
-      
+
       char next[3] = {toupper(c), tolower(c), 0};
       while (1) {
         size_t n = strcspn(str, next);
         str += n;
-        
+
         if (str[0] == 0 || (n >= size)) {
           break;
         }
-        
+
         int32_t ret = patternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
         if (ret != TSDB_PATTERN_NOMATCH) {
           return ret;
@@ -195,18 +258,18 @@ int patternMatch(const char *patterStr, const char *str, size_t size, const SPat
       }
       return TSDB_PATTERN_NOWILDCARDMATCH;
     }
-    
+
     c1 = str[j++];
-    
+
     if (j <= size) {
       if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) {
         continue;
       }
     }
-    
+
     return TSDB_PATTERN_NOMATCH;
   }
-  
+
   return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
 }
 
@@ -214,13 +277,13 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c
   wchar_t c, c1;
   wchar_t matchOne = L'_';  // "_"
   wchar_t matchAll = L'%';  // "%"
-  
+
   int32_t i = 0;
   int32_t j = 0;
-  
+
   while ((c = patterStr[i++]) != 0) {
     if (c == matchAll) { /* Match "%" */
-      
+
       while ((c = patterStr[i++]) == matchAll || c == matchOne) {
         if (c == matchOne && (j > size || str[j++] == 0)) {
           return TSDB_PATTERN_NOWILDCARDMATCH;
@@ -229,40 +292,40 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c
       if (c == 0) {
         return TSDB_PATTERN_MATCH;
       }
-      
+
       wchar_t accept[3] = {towupper(c), towlower(c), 0};
       while (1) {
         size_t n = wcscspn(str, accept);
-        
+
         str += n;
         if (str[0] == 0 || (n >= size)) {
           break;
         }
-        
+
         int32_t ret = WCSPatternMatch(&patterStr[i], ++str, size - n - 1, pInfo);
         if (ret != TSDB_PATTERN_NOMATCH) {
           return ret;
         }
       }
-      
+
       return TSDB_PATTERN_NOWILDCARDMATCH;
     }
-    
+
     c1 = str[j++];
-    
+
     if (j <= size) {
       if (c == c1 || towlower(c) == towlower(c1) || (c == matchOne && c1 != 0)) {
         continue;
       }
     }
-    
+
     return TSDB_PATTERN_NOMATCH;
   }
   
   return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH;
 }
 
-static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
+int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
 
   assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN);
@@ -283,34 +346,54 @@ static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
 int32_t taosArrayCompareString(const void* a, const void* b) {
   const char* x = *(const char**)a;
   const char* y = *(const char**)b;
-  
+
   return compareLenPrefixedStr(x, y);
 }
 
-//static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) {
-//  const SArray* arr = (const SArray*) pRight;
-//  return taosArraySearchString(arr, pLeft, taosArrayCompareString, TD_EQ) == NULL ? 0 : 1;
-//}
-static int32_t compareFindItemInSet(const void *pLeft, const void* pRight)  {
-  return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;    
+int32_t compareFindItemInSet(const void *pLeft, const void* pRight)  {
+  return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0;
 }
 
-static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
+int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
   SPatternCompareInfo pInfo = {'%', '_'};
 
   assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE);
-  wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t));
 
+  wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t));
   memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
 
   int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo);
   free(pattern);
+
   return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
 }
 
 __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
   __compar_fn_t comparFn = NULL;
-  
+
+  if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) {
+    switch (type) {
+      case TSDB_DATA_TYPE_BOOL:
+      case TSDB_DATA_TYPE_TINYINT:
+      case TSDB_DATA_TYPE_UTINYINT:
+        return setCompareBytes1;
+      case TSDB_DATA_TYPE_SMALLINT:
+      case TSDB_DATA_TYPE_USMALLINT:
+        return setCompareBytes2;
+      case TSDB_DATA_TYPE_INT:
+      case TSDB_DATA_TYPE_UINT:
+      case TSDB_DATA_TYPE_FLOAT:
+        return setCompareBytes4;
+      case TSDB_DATA_TYPE_BIGINT:
+      case TSDB_DATA_TYPE_UBIGINT:
+      case TSDB_DATA_TYPE_DOUBLE:
+      case TSDB_DATA_TYPE_TIMESTAMP:
+        return setCompareBytes8;
+      default:
+        assert(0);
+    }
+  }
+
   switch (type) {
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_TINYINT:   comparFn = compareInt8Val;   break;
@@ -328,13 +411,15 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
       } else { /* normal relational comparFn */
         comparFn = compareLenPrefixedStr;
       }
-    
+
       break;
     }
-  
+
     case TSDB_DATA_TYPE_NCHAR: {
       if (optr == TSDB_RELATION_LIKE) {
         comparFn = compareWStrPatternComp;
+      } else if (optr == TSDB_RELATION_IN) {
+        comparFn = compareFindItemInSet;
       } else {
         comparFn = compareLenPrefixedWStr;
       }
@@ -350,57 +435,57 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
       comparFn = compareInt32Val;
       break;
   }
-  
+
   return comparFn;
 }
 
-__compar_fn_t getKeyComparFunc(int32_t keyType) {
+__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) {
   __compar_fn_t comparFn = NULL;
-  
+
   switch (keyType) {
     case TSDB_DATA_TYPE_TINYINT:
     case TSDB_DATA_TYPE_BOOL:
-      comparFn = compareInt8Val;
+      comparFn = (order == TSDB_ORDER_ASC)? compareInt8Val:compareInt8ValDesc;
       break;
     case TSDB_DATA_TYPE_SMALLINT:
-      comparFn = compareInt16Val;
+      comparFn = (order == TSDB_ORDER_ASC)? compareInt16Val:compareInt16ValDesc;
       break;
     case TSDB_DATA_TYPE_INT:
-      comparFn = compareInt32Val;
+      comparFn = (order == TSDB_ORDER_ASC)? compareInt32Val:compareInt32ValDesc;
       break;
     case TSDB_DATA_TYPE_BIGINT:
     case TSDB_DATA_TYPE_TIMESTAMP:
-      comparFn = compareInt64Val;
+      comparFn = (order == TSDB_ORDER_ASC)? compareInt64Val:compareInt64ValDesc;
       break;
     case TSDB_DATA_TYPE_FLOAT:
-      comparFn = compareFloatVal;
+      comparFn = (order == TSDB_ORDER_ASC)? compareFloatVal:compareFloatValDesc;
       break;
     case TSDB_DATA_TYPE_DOUBLE:
-      comparFn = compareDoubleVal;
+      comparFn = (order == TSDB_ORDER_ASC)? compareDoubleVal:compareDoubleValDesc;
       break;
     case TSDB_DATA_TYPE_UTINYINT:
-      comparFn = compareUint8Val;
+      comparFn = (order == TSDB_ORDER_ASC)? compareUint8Val:compareUint8ValDesc;
       break;
     case TSDB_DATA_TYPE_USMALLINT:
-      comparFn = compareUint16Val;
+      comparFn = (order == TSDB_ORDER_ASC)? compareUint16Val:compareUint16ValDesc;
       break;
     case TSDB_DATA_TYPE_UINT:
-      comparFn = compareUint32Val;
+      comparFn = (order == TSDB_ORDER_ASC)? compareUint32Val:compareUint32ValDesc;
       break;
     case TSDB_DATA_TYPE_UBIGINT:
-      comparFn = compareUint64Val;
+      comparFn = (order == TSDB_ORDER_ASC)? compareUint64Val:compareUint64ValDesc;
       break;
     case TSDB_DATA_TYPE_BINARY:
-      comparFn = compareLenPrefixedStr;
+      comparFn = (order == TSDB_ORDER_ASC)? compareLenPrefixedStr:compareLenPrefixedStrDesc;
       break;
     case TSDB_DATA_TYPE_NCHAR:
-      comparFn = compareLenPrefixedWStr;
+      comparFn = (order == TSDB_ORDER_ASC)? compareLenPrefixedWStr:compareLenPrefixedWStrDesc;
       break;
     default:
-      comparFn = compareInt32Val;
+      comparFn = (order == TSDB_ORDER_ASC)? compareInt32Val:compareInt32ValDesc;
       break;
   }
-  
+
   return comparFn;
 }
 
@@ -424,8 +509,7 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) {
       if (t1->len != t2->len) {
         return t1->len > t2->len? 1:-1;
       }
-
-      int32_t ret = wcsncmp((wchar_t*) t1->data, (wchar_t*) t2->data, t2->len/TSDB_NCHAR_SIZE);
+      int32_t ret = memcmp((wchar_t*) t1, (wchar_t*) t2, t2->len);
       if (ret == 0) {
         return ret;
       }
@@ -434,7 +518,7 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) {
     default: {  // todo refactor
       tstr* t1 = (tstr*) f1;
       tstr* t2 = (tstr*) f2;
-      
+
       if (t1->len != t2->len) {
         return t1->len > t2->len? 1:-1;
       } else {