Merge branch '3.0_udfd' of github.com:taosdata/TDengine into 3.0_udfd

Jenkinsfile2

@@ -88,6 +88,12 @@ def pre_test(){
     cmake .. > /dev/null
     make -j4> /dev/null
     '''
+    sh'''
+    cd ${WKPY}
+    git reset --hard
+    git pull
+    pip3 install .
+    '''
     return 1
 }
 
@@ -97,6 +103,7 @@ pipeline {
   environment{
       WK = '/var/lib/jenkins/workspace/TDinternal'
       WKC= '/var/lib/jenkins/workspace/TDengine'
+      WKPY= '/var/lib/jenkins/workspace/taos-connector-python'
   }
   stages {
       stage('pre_build'){
@@ -113,13 +120,18 @@ pipeline {
             timeout(time: 45, unit: 'MINUTES'){
               pre_test()
               sh'''
-              cd ${WKC}/tests
-              ./test-all.sh b1fq
-              '''
-              sh'''
               cd ${WKC}/debug
               ctest
               '''
+              sh'''
+              export LD_LIBRARY_PATH=${WKC}/debug/build/lib
+              cd ${WKC}/tests/system-test
+              ./fulltest.sh
+              '''
+              sh'''
+              cd ${WKC}/tests
+              ./test-all.sh b1fq
+              '''
             }
           }
       }

cmake/cmake.define

 cmake_minimum_required(VERSION 3.16)
 
-set(CMAKE_VERBOSE_MAKEFILE ON)
+set(CMAKE_VERBOSE_MAKEFILE OFF)
 
 #set output directory
 SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)

examples/c/asyncdemo.c

@@ -45,7 +45,7 @@ typedef struct {
 
 void taos_insert_call_back(void *param, TAOS_RES *tres, int code);
 void taos_select_call_back(void *param, TAOS_RES *tres, int code);
-void taos_error(TAOS *taos);
+void shellPrintError(TAOS *taos);
 
 static void queryDB(TAOS *taos, char *command) {
   int i;
@@ -102,7 +102,7 @@ int main(int argc, char *argv[])
 
   taos = taos_connect(argv[1], "root", "taosdata", NULL, 0);
   if (taos == NULL)
-    taos_error(taos);
+    shellPrintError(taos);
 
   printf("success to connect to server\n");
 
@@ -193,7 +193,7 @@ int main(int argc, char *argv[])
   return 0;
 }
 
-void taos_error(TAOS *con)
+void shellPrintError(TAOS *con)
 {
   fprintf(stderr, "TDengine error: %s\n", taos_errstr(con));
   taos_close(con);

include/common/tglobal.h

@@ -34,13 +34,9 @@ extern int32_t  tsVersion;
 extern int32_t  tsStatusInterval;
 
 // common
-extern int32_t tsRpcTimer;
-extern int32_t tsRpcMaxTime;
-extern bool    tsRpcForceTcp;  // all commands go to tcp protocol if this is enabled
 extern int32_t tsMaxConnections;
 extern int32_t tsMaxShellConns;
 extern int32_t tsShellActivityTimer;
-extern int32_t tsMaxTmrCtrl;
 extern int32_t tsCompressMsgSize;
 extern int32_t tsCompressColData;
 extern int32_t tsMaxNumOfDistinctResults;
@@ -98,9 +94,6 @@ extern bool    tsDeadLockKillQuery;
 extern int32_t tsQueryPolicy;
 
 // client
-extern int32_t tsMaxWildCardsLen;
-extern int32_t tsMaxRegexStringLen;
-extern int32_t tsMaxNumOfOrderedResults;
 extern int32_t tsMinSlidingTime;
 extern int32_t tsMinIntervalTime;
 extern int32_t tsMaxStreamComputDelay;

include/common/tmsgdef.h

@@ -85,6 +85,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "dnode-server-status", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "dnode-net-test", NULL, NULL)
 
   // Requests handled by MNODE
   TD_NEW_MSG_SEG(TDMT_MND_MSG)

include/libs/nodes/plannodes.h

@@ -46,7 +46,7 @@ typedef struct SScanLogicNode {
   struct STableMeta* pMeta;
   SVgroupsInfo* pVgroupList;
   EScanType scanType;
-  uint8_t scanFlag;         // denotes reversed scan of data or not
+  uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count
   STimeWindow scanRange;
   SName tableName;
   bool showRewrite;
@@ -189,9 +189,6 @@ typedef struct SScanPhysiNode {
   SNodeList* pScanCols;
   uint64_t uid;           // unique id of the table
   int8_t tableType;
-  int32_t order;         // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC
-  int32_t count;         // repeat count
-  int32_t reverse;       // reverse scan count
   SName tableName;
 } SScanPhysiNode;
 
@@ -207,7 +204,7 @@ typedef struct SSystemTableScanPhysiNode {
 
 typedef struct STableScanPhysiNode {
   SScanPhysiNode scan;
-  uint8_t scanFlag;         // denotes reversed scan of data or not
+  uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count
   STimeWindow scanRange;
   double ratio;
   int32_t dataRequired;

include/libs/planner/planner.h

@@ -37,6 +37,8 @@ typedef struct SPlanContext {
   bool isStmtQuery;
   void* pTransporter;
   struct SCatalog* pCatalog;
+  char* pMsg;
+  int32_t msgLen;
 } SPlanContext;
 
 // Create the physical plan for the query, according to the AST.

include/os/osSystem.h

@@ -31,19 +31,19 @@ extern "C" {
 
 typedef struct TdCmd *TdCmdPtr;
 
-TdCmdPtr taosOpenCmd(const char *cmd);
-int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf);
+TdCmdPtr taosOpenCmd(const char* cmd);
+int64_t  taosGetLineCmd(TdCmdPtr pCmd, char** __restrict ptrBuf);
 int32_t  taosEOFCmd(TdCmdPtr pCmd);
-int64_t taosCloseCmd(TdCmdPtr *ppCmd);
+int64_t  taosCloseCmd(TdCmdPtr* ppCmd);
 
 void* taosLoadDll(const char* filename);
 void* taosLoadSym(void* handle, char* name);
 void  taosCloseDll(void* handle);
 
 int32_t taosSetConsoleEcho(bool on);
-void setTerminalMode();
-int32_t getOldTerminalMode();
-void resetTerminalMode();
+void    taosSetTerminalMode();
+int32_t taosGetOldTerminalMode();
+void    taosResetTerminalMode();
 
 #ifdef __cplusplus
 }

include/util/taoserror.h

@@ -623,6 +623,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_DAYS_VALUE        TAOS_DEF_ERROR_CODE(0, 0x2636)
 #define TSDB_CODE_PAR_OFFSET_LESS_ZERO          TAOS_DEF_ERROR_CODE(0, 0x2637)
 #define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY  TAOS_DEF_ERROR_CODE(0, 0x2638)
+#define TSDB_CODE_PAR_INVALID_TOPIC_QUERY       TAOS_DEF_ERROR_CODE(0, 0x2639)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)

include/util/tdef.h

@@ -76,8 +76,6 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_DEFAULT_PASS "taosdata"
 #endif
 
-#define SHELL_MAX_PASSWORD_LEN 20
-
 #define TSDB_TRUE  1
 #define TSDB_FALSE 0
 #define TSDB_OK    0

source/client/src/clientImpl.c

@@ -232,7 +232,9 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
     .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
     .pAstRoot = pQuery->pRoot,
     .showRewrite = pQuery->showRewrite,
-    .pTransporter = pRequest->pTscObj->pAppInfo->pTransporter
+    .pTransporter = pRequest->pTscObj->pAppInfo->pTransporter,
+    .pMsg = pRequest->msgBuf,
+    .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE
   };
   int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
   if (TSDB_CODE_SUCCESS == code) {

source/common/src/tglobal.c

@@ -32,13 +32,9 @@ int32_t  tsVersion = 30000000;
 int32_t  tsStatusInterval = 1;  // second
 
 // common
-int32_t tsRpcTimer = 300;
-int32_t tsRpcMaxTime = 600;    // seconds;
-bool    tsRpcForceTcp = true;  // disable this, means query, show command use udp protocol as default
 int32_t tsMaxShellConns = 50000;
 int32_t tsMaxConnections = 50000;
 int32_t tsShellActivityTimer = 3;  // second
-int32_t tsMaxBinaryDisplayWidth = 30;
 bool    tsEnableSlaveQuery = true;
 bool    tsPrintAuth = false;
 
@@ -105,14 +101,6 @@ int32_t tsCompressColData = -1;
  */
 int32_t tsCompatibleModel = 1;
 
-// client
-int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN;
-int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN;
-
-// the maximum number of results for projection query on super table that are returned from
-// one virtual node, to order according to timestamp
-int32_t tsMaxNumOfOrderedResults = 100000;
-
 // 10 ms for sliding time, the value will changed in case of time precision changed
 int32_t tsMinSlidingTime = 10;
 
@@ -314,19 +302,10 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1;
   if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1;
   if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "maxTmrCtrl", tsMaxTmrCtrl, 8, 2048, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "rpcTimer", tsRpcTimer, 100, 3000, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "rpcMaxTime", tsRpcMaxTime, 100, 7200, 1) != 0) return -1;
-  if (cfgAddBool(pCfg, "rpcForceTcp", tsRpcForceTcp, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "maxWildCardsLength", tsMaxWildCardsLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "maxRegexStringLen", tsMaxRegexStringLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "maxNumOfOrderedRes", tsMaxNumOfOrderedResults, 128, TSDB_MAX_ALLOWED_SQL_LEN, 1) != 0)
-    return -1;
   if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1;
 
   tsNumOfTaskQueueThreads = tsNumOfCores / 4;
@@ -508,18 +487,10 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
     return -1;
   }
 
-  tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32;
-  tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32;
-  tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32;
-  tsRpcForceTcp = cfgGetItem(pCfg, "rpcForceTcp")->i32;
   tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
   tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32;
   tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32;
-  tsMaxWildCardsLen = cfgGetItem(pCfg, "maxWildCardsLength")->i32;
-  tsMaxRegexStringLen = cfgGetItem(pCfg, "maxRegexStringLen")->i32;
-  tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32;
   tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
-  tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32;
   tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
   tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32;
   return 0;
@@ -710,6 +681,6 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
 
   if (strcasecmp(option, "resetlog") == 0) {
     taosResetLog();
-    cfgDumpCfg(tsCfg, 1, false);
+    cfgDumpCfg(tsCfg, 0, false);
   }
 }

source/dnode/mgmt/exe/dmMain.c

@@ -106,7 +106,7 @@ static void dmPrintVersion() {
 
 static void dmDumpCfg() {
   SConfig *pCfg = taosGetCfg();
-  cfgDumpCfg(pCfg, 0, 1);
+  cfgDumpCfg(pCfg, 0, true);
 }
 
 static SDnodeOpt dmGetOpt() {

source/dnode/mgmt/implement/src/dmHandle.c

@@ -17,7 +17,7 @@
 #include "dmImp.h"
 
 static void dmUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) {
-  if (pDnode->data.dnodeId == 0) {
+  if (pDnode->data.dnodeId == 0 || pDnode->data.clusterId == 0) {
     dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId);
     taosWLockLatch(&pDnode->data.latch);
     pDnode->data.dnodeId = pCfg->dnodeId;
@@ -57,6 +57,7 @@ void dmSendStatusReq(SDnode *pDnode) {
   req.dnodeVer = pDnode->data.dnodeVer;
   req.dnodeId = pDnode->data.dnodeId;
   req.clusterId = pDnode->data.clusterId;
+  if (req.clusterId == 0) req.dnodeId = 0;
   req.rebootTime = pDnode->data.rebootTime;
   req.updateTime = pDnode->data.updateTime;
   req.numOfCores = tsNumOfCores;
@@ -145,10 +146,10 @@ int32_t dmProcessCreateNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMs
     dError("node:%s, failed to create since %s", pWrapper->name, terrstr());
   } else {
     dDebug("node:%s, has been created", pWrapper->name);
+    (void)dmOpenNode(pWrapper);
     pWrapper->required = true;
     pWrapper->deployed = true;
     pWrapper->procType = pDnode->ptype;
-    (void)dmOpenNode(pWrapper);
   }
 
   taosThreadMutexUnlock(&pDnode->mutex);
@@ -170,13 +171,13 @@ int32_t dmProcessDropNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg)
     dError("node:%s, failed to drop since %s", pWrapper->name, terrstr());
   } else {
     dDebug("node:%s, has been dropped", pWrapper->name);
+    pWrapper->required = false;
+    pWrapper->deployed = false;
   }
 
   dmReleaseWrapper(pWrapper);
 
   if (code == 0) {
-    pWrapper->required = false;
-    pWrapper->deployed = false;
     dmCloseNode(pWrapper);
     taosRemoveDir(pWrapper->path);
   }

source/dnode/mgmt/implement/src/dmTransport.c

@@ -71,12 +71,15 @@ static void dmProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSe
   SNodeMsg *pMsg = NULL;
   NodeMsgFp msgFp = NULL;
   uint16_t  msgType = pRpc->msgType;
+  bool      needRelease = false;
 
   if (pEpSet && pEpSet->numOfEps > 0 && msgType == TDMT_MND_STATUS_RSP) {
     dmSetMnodeEpSet(pWrapper->pDnode, pEpSet);
   }
 
   if (dmMarkWrapper(pWrapper) != 0) goto _OVER;
+
+  needRelease = true;
   if ((msgFp = dmGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER;
   if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER;
   if (dmBuildMsg(pMsg, pRpc) != 0) goto _OVER;
@@ -119,7 +122,9 @@ _OVER:
     rpcFreeCont(pRpc->pCont);
   }
 
+  if (needRelease) {
     dmReleaseWrapper(pWrapper);
+  }
 }
 
 static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
@@ -135,6 +140,12 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
     return;
   }
 
+  if (msgType == TDMT_DND_NET_TEST) {
+    dTrace("net test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle);
+    dmProcessServerStatusReq(pDnode, pMsg);
+    return;
+  }
+
   if (pDnode->status != DND_STAT_RUNNING) {
     dError("msg:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle);
     if (isReq) {

source/dnode/mgmt/interface/inc/dmInt.h

@@ -37,6 +37,7 @@ void   dmSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgF
 void   dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc);
 void   dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc);
 void   dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pMsg);
+void   dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pMsg);
 void   dmGetMonitorSysInfo(SMonSysInfo *pInfo);
 
 // dmFile.c

source/dnode/mgmt/interface/src/dmInt.c

@@ -171,6 +171,14 @@ static void dmGetServerStatus(SDnode *pDnode, SServerStatusRsp *pStatus) {
   }
 }
 
+void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pRpc) {
+  dDebug("net test req is received");
+  SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = 0};
+  rsp.pCont = rpcMallocCont(pRpc->contLen);
+  rsp.contLen = pRpc->contLen;
+  rpcSendResponse(&rsp);
+}
+
 void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pReq) {
   dDebug("server status req is received");
 

source/dnode/mgmt/mgmt_mnode/src/mmInt.c

@@ -161,9 +161,9 @@ static int32_t mmOpen(SMgmtWrapper *pWrapper) {
   SMnodeOpt option = {0};
   if (!deployed) {
     dInfo("mnode start to deploy");
-    if (pWrapper->procType == DND_PROC_CHILD) {
+    // if (pWrapper->procType == DND_PROC_CHILD) {
       pWrapper->pDnode->data.dnodeId = 1;
-    }
+    // }
     mmBuildOptionForDeploy(pMgmt, &option);
   } else {
     dInfo("mnode start to open");

source/dnode/mnode/impl/src/mnode.c

@@ -385,7 +385,11 @@ int32_t mndProcessMsg(SNodeMsg *pMsg) {
     terrno = code;
     mTrace("msg:%p, in progress, app:%p", pMsg, ahandle);
   } else if (code != 0) {
+    if (terrno != TSDB_CODE_OPS_NOT_SUPPORT) {
       mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
+    } else {
+      mTrace("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
+    }
   } else {
     mTrace("msg:%p, is processed, app:%p", pMsg, ahandle);
   }

source/dnode/mnode/impl/test/func/CMakeLists.txt

@@ -5,7 +5,7 @@ target_link_libraries(
     PUBLIC sut
 )
 
-#add_test(
-#    NAME funcTest
-#    COMMAND funcTest
-#)
+add_test(
+    NAME funcTest
+    COMMAND funcTest
+)

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -1510,7 +1510,6 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid,
   }
 
   STSma *pTSma = pItem->pSma;
-
 #endif
 
   STSmaReadH tReadH = {0};

source/libs/command/src/explain.c

@@ -344,11 +344,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
       EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
       EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->totalRowSize);
       EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
-      EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count);
-      if (pTagScanNode->reverse) {
-        EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
-        EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse);
-      }
       EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
       EXPLAIN_ROW_END();
       QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
@@ -361,10 +356,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
         EXPLAIN_ROW_END();
         QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
         
-        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order));
-        EXPLAIN_ROW_END();
-        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
-
         if (pResNode->pExecInfo) {
           QRY_ERR_RET(qExplainBufAppendVerboseExecInfo(pResNode->pExecInfo, tbuf, &tlen));
           if (tlen) {
@@ -388,11 +379,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
       EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
       EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize);
       EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
-      EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count);
-      if (pTblScanNode->scan.reverse) {
-        EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
-        EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse);
-      }
       EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);      
       EXPLAIN_ROW_END();
       QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
@@ -405,10 +391,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
         EXPLAIN_ROW_END();
         QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
 
-        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order));
-        EXPLAIN_ROW_END();
-        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
-        
         EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey);
         EXPLAIN_ROW_END();
         QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
@@ -434,11 +416,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
       EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
       EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize);
       EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
-      EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count);
-      if (pSTblScanNode->scan.reverse) {
-        EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
-        EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse);
-      }
       EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);      
       EXPLAIN_ROW_END();
       QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
@@ -451,10 +428,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
         EXPLAIN_ROW_END();
         QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
 
-        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order));
-        EXPLAIN_ROW_END();
-        QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));   
-
         if (pSTblScanNode->scan.node.pConditions) {
           EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
           QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); 

source/libs/executor/src/executorimpl.c

@@ -1121,6 +1121,10 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt
         // todo avoid case: top(k, 12), 12 is the value parameter.
         // sum(11), 11 is also the value parameter.
         if (createDummyCol && pOneExpr->base.numOfParams == 1) {
+          pInput->totalRows = pBlock->info.rows;
+          pInput->numOfRows = pBlock->info.rows;
+          pInput->startRowIndex = 0;
+
           code = doCreateConstantValColumnInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows);
           if (code != TSDB_CODE_SUCCESS) {
             return code;
@@ -6571,9 +6575,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
           .offset = pTableScanNode->offset,
       };
 
-      return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pTableScanNode->dataRequired,
-                                         pScanPhyNode->count, pScanPhyNode->reverse, pColList, pResBlock,
-                                         pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo);
+      return createTableScanOperatorInfo(pDataReader, pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC,
+          numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq[0], pTableScanNode->scanSeq[1], pColList,
+          pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo);
     } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) {
       SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode;
       SSDataBlock*        pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc);
@@ -6721,7 +6725,7 @@ static tsdbReaderT createDataReaderImpl(STableScanPhysiNode* pTableScanNode, STa
                                         void* readHandle, uint64_t queryId, uint64_t taskId) {
   STsdbQueryCond cond = {.loadExternalRows = false};
 
-  cond.order = pTableScanNode->scan.order;
+  cond.order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
   cond.numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols);
   cond.colList = taosMemoryCalloc(cond.numOfCols, sizeof(SColumnInfo));
   if (cond.colList == NULL) {

source/libs/function/inc/builtinsimpl.h

@@ -40,6 +40,11 @@ bool getMinmaxFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 int32_t minFunction(SqlFunctionCtx* pCtx);
 int32_t maxFunction(SqlFunctionCtx *pCtx);
 
+bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+bool avgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
+int32_t avgFunction(SqlFunctionCtx* pCtx);
+int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
+
 bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t stddevFunction(SqlFunctionCtx* pCtx);

source/libs/function/src/builtins.c

@@ -104,7 +104,7 @@ static int32_t translateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
   if (1 != LIST_LENGTH(pFunc->pParameterList)) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
-  pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT};
+  pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
   return TSDB_CODE_SUCCESS;
 }
 
@@ -355,7 +355,7 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  pFunc->node.resType = (SDataType) { .bytes = 64, .type = TSDB_DATA_TYPE_BINARY};
+  pFunc->node.resType = (SDataType) { .bytes = 24, .type = TSDB_DATA_TYPE_BINARY};
   return TSDB_CODE_SUCCESS;
 }
 
@@ -479,6 +479,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .processFunc  = stddevFunction,
     .finalizeFunc = stddevFinalize
   },
+  {
+    .name = "avg",
+    .type = FUNCTION_TYPE_AVG,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateInNumOutDou,
+    .getEnvFunc   = getAvgFuncEnv,
+    .initFunc     = avgFunctionSetup,
+    .processFunc  = avgFunction,
+    .finalizeFunc = avgFinalize
+  },
   {
     .name = "percentile",
     .type = FUNCTION_TYPE_PERCENTILE,

source/libs/function/src/builtinsimpl.c

@@ -20,6 +20,59 @@
 #include "tdatablock.h"
 #include "tpercentile.h"
 
+typedef struct SSumRes {
+  union {
+    int64_t  isum;
+    uint64_t usum;
+    double   dsum;
+  };
+} SSumRes;
+
+typedef struct SAvgRes {
+  double  result;
+  SSumRes sum;
+  int64_t count;
+} SAvgRes;
+
+typedef struct STopBotResItem {
+  SVariant v;
+  uint64_t uid;        // it is a table uid, used to extract tag data during building of the final result for the tag data
+  struct {
+   int32_t pageId;
+   int32_t offset;
+  } tuplePos;          // tuple data of this chosen row
+} STopBotResItem;
+
+typedef struct STopBotRes {
+  int32_t         pageId;
+//  int32_t   num;
+  STopBotResItem *pItems;
+} STopBotRes;
+
+typedef struct SStddevRes {
+  double  result;
+  int64_t count;
+  union  {double  quadraticDSum; int64_t quadraticISum;};
+  union  {double  dsum; int64_t isum;};
+} SStddevRes;
+
+typedef struct SPercentileInfo {
+  double      result;
+  tMemBucket *pMemBucket;
+  int32_t     stage;
+  double      minval;
+  double      maxval;
+  int64_t     numOfElems;
+} SPercentileInfo;
+
+typedef struct SDiffInfo {
+  bool  hasPrev;
+  bool  includeNull;
+  bool  ignoreNegative;
+  bool  firstOutput;
+  union { int64_t i64; double d64;} prev;
+} SDiffInfo;
+
 #define SET_VAL(_info, numOfElem, res)  \
   do {                                  \
     if ((numOfElem) <= 0) {             \
@@ -28,13 +81,50 @@
     (_info)->numOfRes = (res);          \
   } while (0)
 
-typedef struct SSumRes {
-  union {
-    int64_t  isum;
-    uint64_t usum;
-    double   dsum;
-  };
-} SSumRes;
+#define GET_TS_LIST(x)    ((TSKEY*)((x)->ptsList))
+#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)])
+
+#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx)                      \
+  do {                                                             \
+    for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \
+      SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i];      \
+      __ctx->fpSet.process(__ctx);                                 \
+    }                                                              \
+  } while (0);
+
+#define DO_UPDATE_SUBSID_RES(ctx, ts)                                 \
+  do {                                                                \
+    for (int32_t _i = 0; _i < (ctx)->subsidiaryRes.numOfCols; ++_i) { \
+      SqlFunctionCtx *__ctx = (ctx)->subsidiaryRes.pCtx[_i];          \
+      if (__ctx->functionId == FUNCTION_TS_DUMMY) {                   \
+        __ctx->tag.i = (ts);                                          \
+        __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT;                     \
+      }                                                               \
+      __ctx->fpSet.process(__ctx);                                    \
+    }                                                                 \
+  } while (0)
+
+#define UPDATE_DATA(ctx, left, right, num, sign, _ts) \
+  do {                                                \
+    if (((left) < (right)) ^ (sign)) {                \
+      (left) = (right);                               \
+      DO_UPDATE_SUBSID_RES(ctx, _ts);                 \
+      (num) += 1;                                     \
+    }                                                 \
+  } while (0)
+
+#define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num)        \
+  do {                                                                   \
+    _t *d = (_t *)((_col)->pData);                                       \
+    for (int32_t i = (_start); i < (_nrow) + (_start); ++i) {            \
+      if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \
+        continue;                                                        \
+      }                                                                  \
+      TSKEY ts = (ctx)->ptsList != NULL ? GET_TS_DATA(ctx, i) : 0;       \
+      UPDATE_DATA(ctx, val, d[i], num, sign, ts);                        \
+    }                                                                    \
+  } while (0)
+
 
 bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
   if (pResultInfo->initialized) {
@@ -190,6 +280,145 @@ bool getSumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   return true;
 }
 
+bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
+  pEnv->calcMemSize = sizeof(double);
+  return true;
+}
+
+bool avgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
+  if (!functionSetup(pCtx, pResultInfo)) {
+    return false;
+  }
+
+  SAvgRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo);
+  memset(pRes, 0, sizeof(SAvgRes));
+  return true;
+}
+
+int32_t avgFunction(SqlFunctionCtx* pCtx) {
+  int32_t numOfElem = 0;
+
+  // Only the pre-computing information loaded and actual data does not loaded
+  SInputColumnInfoData* pInput = &pCtx->input;
+  int32_t               type = pInput->pData[0]->info.type;
+
+  SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+
+  // computing based on the true data block
+  SColumnInfoData* pCol = pInput->pData[0];
+
+  int32_t start = pInput->startRowIndex;
+  int32_t numOfRows = pInput->numOfRows;
+
+  switch (type) {
+    case TSDB_DATA_TYPE_TINYINT: {
+        int8_t* plist = (int8_t*)pCol->pData;
+        for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
+          if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
+            continue;
+          }
+
+          numOfElem += 1;
+          pAvgRes->count += 1;
+          pAvgRes->sum.isum += plist[i];
+        }
+
+        break;
+      }
+
+      case TSDB_DATA_TYPE_SMALLINT: {
+      int16_t* plist = (int16_t*)pCol->pData;
+      for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
+        if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
+          continue;
+        }
+
+        numOfElem += 1;
+        pAvgRes->count += 1;
+        pAvgRes->sum.isum += plist[i];
+      }
+      break;
+    }
+
+    case TSDB_DATA_TYPE_INT: {
+      int32_t* plist = (int32_t*)pCol->pData;
+      for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
+        if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
+          continue;
+        }
+
+        numOfElem += 1;
+        pAvgRes->count += 1;
+        pAvgRes->sum.isum += plist[i];
+      }
+
+      break;
+    }
+
+    case TSDB_DATA_TYPE_BIGINT: {
+      int64_t* plist = (int64_t*)pCol->pData;
+      for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
+        if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
+          continue;
+        }
+
+        numOfElem += 1;
+        pAvgRes->count += 1;
+        pAvgRes->sum.isum += plist[i];
+      }
+      break;
+    }
+
+    case TSDB_DATA_TYPE_FLOAT: {
+      float* plist = (float*)pCol->pData;
+      for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
+        if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
+          continue;
+        }
+
+        numOfElem += 1;
+        pAvgRes->count += 1;
+        pAvgRes->sum.dsum += plist[i];
+      }
+      break;
+    }
+
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double* plist = (double*)pCol->pData;
+      for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
+        if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
+          continue;
+        }
+
+        numOfElem += 1;
+        pAvgRes->count += 1;
+        pAvgRes->sum.dsum += plist[i];
+      }
+      break;
+    }
+
+    default:
+      break;
+  }
+
+  // data in the check operation are all null, not output
+  SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
+  SInputColumnInfoData* pInput = &pCtx->input;
+  int32_t type = pInput->pData[0]->info.type;
+  SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+  if (IS_INTEGER_TYPE(type)) {
+    pAvgRes->result = pAvgRes->sum.isum / ((double) pAvgRes->count);
+  } else {
+    pAvgRes->result = pAvgRes->sum.dsum / ((double) pAvgRes->count);
+  }
+
+  return functionFinalize(pCtx, pBlock, slotId);
+}
+
 EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow){
   return FUNC_DATA_REQUIRED_STATIS_LOAD;
 }
@@ -292,49 +521,6 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   return true;
 }
 
-#define GET_TS_LIST(x)    ((TSKEY*)((x)->ptsList))
-#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)])
-
-#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx)                      \
-  do {                                                             \
-    for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \
-      SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i];      \
-      __ctx->fpSet.process(__ctx);                                 \
-    }                                                              \
-  } while (0);
-
-#define DO_UPDATE_SUBSID_RES(ctx, ts)                                 \
-  do {                                                                \
-    for (int32_t _i = 0; _i < (ctx)->subsidiaryRes.numOfCols; ++_i) { \
-      SqlFunctionCtx *__ctx = (ctx)->subsidiaryRes.pCtx[_i];          \
-      if (__ctx->functionId == FUNCTION_TS_DUMMY) {                   \
-        __ctx->tag.i = (ts);                                          \
-        __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT;                     \
-      }                                                               \
-      __ctx->fpSet.process(__ctx);                                    \
-    }                                                                 \
-  } while (0)
-
-#define UPDATE_DATA(ctx, left, right, num, sign, _ts) \
-  do {                                                \
-    if (((left) < (right)) ^ (sign)) {                \
-      (left) = (right);                               \
-      DO_UPDATE_SUBSID_RES(ctx, _ts);                 \
-      (num) += 1;                                     \
-    }                                                 \
-  } while (0)
-
-#define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num)        \
-  do {                                                                   \
-    _t *d = (_t *)((_col)->pData);                                       \
-    for (int32_t i = (_start); i < (_nrow) + (_start); ++i) {            \
-      if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \
-        continue;                                                        \
-      }                                                                  \
-      TSKEY ts = (ctx)->ptsList != NULL ? GET_TS_DATA(ctx, i) : 0;       \
-      UPDATE_DATA(ctx, val, d[i], num, sign, ts);                        \
-    }                                                                    \
-  } while (0)
 
 int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) {
   int32_t numOfElems = 0;
@@ -479,13 +665,6 @@ int32_t maxFunction(SqlFunctionCtx *pCtx) {
   return TSDB_CODE_SUCCESS;
 }
 
-typedef struct SStddevRes {
-  double  result;
-  int64_t count;
-  union  {double  quadraticDSum; int64_t quadraticISum;};
-  union  {double  dsum; int64_t isum;};
-} SStddevRes;
-
 bool getStddevFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(SStddevRes);
   return true;
@@ -588,8 +767,8 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
 
         numOfElem += 1;
         pStddevRes->count += 1;
-        pStddevRes->isum += plist[i];
-        pStddevRes->quadraticISum += plist[i] * plist[i];
+        pStddevRes->dsum += plist[i];
+        pStddevRes->quadraticDSum += plist[i] * plist[i];
       }
       break;
     }
@@ -603,8 +782,8 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
 
         numOfElem += 1;
         pStddevRes->count += 1;
-        pStddevRes->isum += plist[i];
-        pStddevRes->quadraticISum += plist[i] * plist[i];
+        pStddevRes->dsum += plist[i];
+        pStddevRes->quadraticDSum += plist[i] * plist[i];
       }
       break;
     }
@@ -619,21 +798,21 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
 }
 
 int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
+  SInputColumnInfoData* pInput = &pCtx->input;
+  int32_t type = pInput->pData[0]->info.type;
   SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
-  double avg = pStddevRes->isum / ((double) pStddevRes->count);
+  double avg;
+  if (IS_INTEGER_TYPE(type)) {
+    avg = pStddevRes->isum / ((double) pStddevRes->count);
     pStddevRes->result = sqrt(pStddevRes->quadraticISum/((double)pStddevRes->count) - avg*avg);
+  } else {
+    avg = pStddevRes->dsum / ((double) pStddevRes->count);
+    pStddevRes->result = sqrt(pStddevRes->quadraticDSum/((double)pStddevRes->count) - avg*avg);
+  }
+
   return functionFinalize(pCtx, pBlock, slotId);
 }
 
-typedef struct SPercentileInfo {
-  double      result;
-  tMemBucket *pMemBucket;
-  int32_t     stage;
-  double      minval;
-  double      maxval;
-  int64_t     numOfElems;
-} SPercentileInfo;
-
 bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(SPercentileInfo);
   return true;
@@ -928,14 +1107,6 @@ int32_t lastFunction(SqlFunctionCtx *pCtx) {
   return TSDB_CODE_SUCCESS;
 }
 
-typedef struct SDiffInfo {
-  bool  hasPrev;
-  bool  includeNull;
-  bool  ignoreNegative;
-  bool  firstOutput;
-  union { int64_t i64; double d64;} prev;
-} SDiffInfo;
-
 bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   pEnv->calcMemSize = sizeof(SDiffInfo);
   return true;
@@ -1168,21 +1339,6 @@ int32_t diffFunction(SqlFunctionCtx *pCtx) {
   }
 }
 
-typedef struct STopBotResItem {
-  SVariant v;
-  uint64_t uid;        // it is a table uid, used to extract tag data during building of the final result for the tag data
-  struct {
-   int32_t pageId;
-   int32_t offset;
-  } tuplePos;          // tuple data of this chosen row
-} STopBotResItem;
-
-typedef struct STopBotRes {
-  int32_t         pageId;
-//  int32_t   num;
-  STopBotResItem *pItems;
-} STopBotRes;
-
 bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
   SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1);
   pEnv->calcMemSize = sizeof(STopBotRes) + pkNode->datum.i * sizeof(STopBotResItem);

source/libs/nodes/src/nodesCodeFuncs.c

@@ -671,9 +671,6 @@ static int32_t jsonToName(const SJson* pJson, void* pObj) {
 static const char* jkScanPhysiPlanScanCols = "ScanCols";
 static const char* jkScanPhysiPlanTableId = "TableId";
 static const char* jkScanPhysiPlanTableType = "TableType";
-static const char* jkScanPhysiPlanScanOrder = "ScanOrder";
-static const char* jkScanPhysiPlanScanCount = "ScanCount";
-static const char* jkScanPhysiPlanReverseScanCount = "ReverseScanCount";
 static const char* jkScanPhysiPlanTableName = "TableName";
 
 static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) {
@@ -689,15 +686,6 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableType, pNode->tableType);
   }
-  if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanScanOrder, pNode->order);
-  }
-  if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanScanCount, pNode->count);
-  }
-  if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanReverseScanCount, pNode->reverse);
-  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddObject(pJson, jkScanPhysiPlanTableName, nameToJson, &pNode->tableName);
   }
@@ -718,15 +706,6 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetTinyIntValue(pJson, jkScanPhysiPlanTableType, &pNode->tableType);
   }
-  if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonGetIntValue(pJson, jkScanPhysiPlanScanOrder, &pNode->order);
-  }
-  if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonGetIntValue(pJson, jkScanPhysiPlanScanCount, &pNode->count);
-  }
-  if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonGetIntValue(pJson, jkScanPhysiPlanReverseScanCount, &pNode->reverse);
-  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonToObject(pJson, jkScanPhysiPlanTableName, jsonToName, &pNode->tableName);
   }
@@ -742,7 +721,8 @@ static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) {
   return jsonToPhysiScanNode(pJson, pObj);
 }
 
-static const char* jkTableScanPhysiPlanScanFlag = "ScanFlag";
+static const char* jkTableScanPhysiPlanScanCount = "ScanCount";
+static const char* jkTableScanPhysiPlanReverseScanCount = "ReverseScanCount";
 static const char* jkTableScanPhysiPlanStartKey = "StartKey";
 static const char* jkTableScanPhysiPlanEndKey = "EndKey";
 static const char* jkTableScanPhysiPlanRatio = "Ratio";
@@ -759,7 +739,10 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
 
   int32_t code = physiScanNodeToJson(pObj, pJson);
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanScanFlag, pNode->scanFlag);
+    code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanScanCount, pNode->scanSeq[0]);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanReverseScanCount, pNode->scanSeq[1]);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanStartKey, pNode->scanRange.skey);
@@ -800,7 +783,10 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
 
   int32_t code = jsonToPhysiScanNode(pJson, pObj);
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonGetUTinyIntValue(pJson, jkTableScanPhysiPlanScanFlag, &pNode->scanFlag);
+    code = tjsonGetUTinyIntValue(pJson, jkTableScanPhysiPlanScanCount, &pNode->scanSeq[0]);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetUTinyIntValue(pJson, jkTableScanPhysiPlanReverseScanCount, &pNode->scanSeq[1]);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetBigIntValue(pJson, jkTableScanPhysiPlanStartKey, &pNode->scanRange.skey);

source/libs/parser/inc/sql.y

@@ -611,6 +611,7 @@ function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D).
 function_expression(A) ::= star_func(B) NK_LP star_func_para_list(C) NK_RP(D).    { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
 function_expression(A) ::= CAST(B) NK_LP expression(C) AS type_name(D) NK_RP(E).  { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); }
 function_expression(A) ::= noarg_func(B) NK_LP NK_RP(C).                          { A = createRawExprNodeExt(pCxt, &B, &C, createFunctionNodeNoArg(pCxt, &B)); }
+//function_expression(A) ::= NOW(B).                                                { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
 
 %type noarg_func                                                                  { SToken }
 %destructor noarg_func                                                            { }

source/libs/parser/src/parCalcConst.c

@@ -83,7 +83,7 @@ static EDealRes calcConstOperator(SOperatorNode** pNode, void* pContext) {
 
 static EDealRes calcConstFunction(SFunctionNode** pNode, void* pContext) {
   SFunctionNode* pFunc = *pNode;
-  if (!fmIsScalarFunc(pFunc->funcId)) {
+  if (!fmIsScalarFunc(pFunc->funcId) || fmIsUserDefinedFunc(pFunc->funcId)) {
     return DEAL_RES_CONTINUE;
   }
   SNode* pParam = NULL;

source/libs/parser/src/parTranslater.c

@@ -543,7 +543,9 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
         TSDB_DATA_TYPE_BLOB == rdt.type) {
       return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
     }
-    if (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) {
+    if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) ||
+        (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && IS_VAR_DATA_TYPE(rdt.type)) ||
+        (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && IS_VAR_DATA_TYPE(ldt.type))) {
       return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
     }
 
@@ -2614,37 +2616,62 @@ static int32_t translateDropComponentNode(STranslateContext* pCxt, SDropComponen
                      (FSerializeFunc)tSerializeSCreateDropMQSBNodeReq, &dropReq);
 }
 
-static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) {
-  SCMCreateTopicReq createReq = {0};
+static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pStmt, SCMCreateTopicReq* pReq) {
+  SName name;
+  // tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName));
+  // tNameGetFullDbName(&name, pReq->name);
+  tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->topicName, &name), pReq->name);
+  pReq->igExists = pStmt->ignoreExists;
+  pReq->withTbName = pStmt->pOptions->withTable;
+  pReq->withSchema = pStmt->pOptions->withSchema;
+  pReq->withTag = pStmt->pOptions->withTag;
+
+  pReq->sql = strdup(pCxt->pParseCxt->pSql);
+  if (NULL == pReq->sql) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  int32_t code = TSDB_CODE_SUCCESS;
 
   if (NULL != pStmt->pQuery) {
+    strcpy(pReq->subscribeDbName, ((SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable))->table.dbName);
     pCxt->pParseCxt->topicQuery = true;
-    int32_t code = translateQuery(pCxt, pStmt->pQuery);
+    code = translateQuery(pCxt, pStmt->pQuery);
     if (TSDB_CODE_SUCCESS == code) {
-      code = nodesNodeToString(pStmt->pQuery, false, &createReq.ast, NULL);
-    }
-    if (TSDB_CODE_SUCCESS != code) {
-      return code;
+      code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL);
     }
   } else {
-    strcpy(createReq.subscribeDbName, pStmt->subscribeDbName);
+    strcpy(pReq->subscribeDbName, pStmt->subscribeDbName);
   }
 
-  createReq.sql = strdup(pCxt->pParseCxt->pSql);
-  if (NULL == createReq.sql) {
-    return TSDB_CODE_OUT_OF_MEMORY;
+  return code;
+}
+
+static int32_t checkCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) {
+  if (NULL == pStmt->pQuery) {
+    return TSDB_CODE_SUCCESS;
   }
 
-  SName name;
-  // tNameSetDbName(&name, pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, strlen(pCxt->pParseCxt->db));
-  // tNameGetFullDbName(&name, createReq.name);
-  tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->topicName, &name), createReq.name);
-  createReq.igExists = pStmt->ignoreExists;
-  createReq.withTbName = pStmt->pOptions->withTable;
-  createReq.withSchema = pStmt->pOptions->withSchema;
-  createReq.withTag = pStmt->pOptions->withTag;
+  if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) {
+    SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery;
+    if (!pSelect->isDistinct && QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable) && NULL == pSelect->pGroupByList &&
+        NULL == pSelect->pLimit && NULL == pSelect->pSlimit && NULL == pSelect->pOrderByList && NULL == pSelect->pPartitionByList) {
+      return TSDB_CODE_SUCCESS;
+    }
+  }
 
-  int32_t code = buildCmdMsg(pCxt, TDMT_MND_CREATE_TOPIC, (FSerializeFunc)tSerializeSCMCreateTopicReq, &createReq);
+  return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TOPIC_QUERY);
+}
+
+static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) {
+  SCMCreateTopicReq createReq = {0};
+  int32_t code = checkCreateTopic(pCxt, pStmt);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = buildCreateTopicReq(pCxt, pStmt, &createReq);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = buildCmdMsg(pCxt, TDMT_MND_CREATE_TOPIC, (FSerializeFunc)tSerializeSCMCreateTopicReq, &createReq);
+  }
   tFreeSCMCreateTopicReq(&createReq);
   return code;
 }

source/libs/parser/src/parUtil.c

@@ -128,6 +128,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
       return "soffset/offset can not be less than 0";
     case TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY:
       return "slimit/soffset only available for PARTITION BY query";
+    case TSDB_CODE_PAR_INVALID_TOPIC_QUERY:
+      return "Invalid topic query";
     case TSDB_CODE_OUT_OF_MEMORY:
       return "Out of memory";
     default:

source/libs/planner/src/planLogicCreater.c

@@ -199,7 +199,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
 
   TSWAP(pScan->pMeta, pRealTable->pMeta, STableMeta*);
   TSWAP(pScan->pVgroupList, pRealTable->pVgroupList, SVgroupsInfo*);
-  pScan->scanFlag = MAIN_SCAN;
+  pScan->scanSeq[0] = 1;
+  pScan->scanSeq[1] = 0;
   pScan->scanRange = TSWINDOW_INITIALIZER;  
   pScan->tableName.type = TSDB_TABLE_NAME_T;
   pScan->tableName.acctId = pCxt->pPlanCxt->acctId;

source/libs/planner/src/planOptimizer.c

@@ -20,11 +20,14 @@
 #define OPTIMIZE_FLAG_MASK(n)    (1 << n)
 
 #define OPTIMIZE_FLAG_OSD OPTIMIZE_FLAG_MASK(0)
+#define OPTIMIZE_FLAG_CPD OPTIMIZE_FLAG_MASK(1)
+#define OPTIMIZE_FLAG_OPK OPTIMIZE_FLAG_MASK(2)
 
 #define OPTIMIZE_FLAG_SET_MASK(val, mask) (val) |= (mask)
 #define OPTIMIZE_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0)
 
 typedef struct SOptimizeContext {
+  SPlanContext* pPlanCxt;
   bool optimized;
 } SOptimizeContext;
 
@@ -57,7 +60,23 @@ typedef enum ECondAction {
   // after supporting outer join, there are other possibilities
 } ECondAction;
 
-EDealRes haveNormalColImpl(SNode* pNode, void* pContext) {
+typedef bool (*FMayBeOptimized)(SLogicNode* pNode);
+
+static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func) {
+  if (func(pNode)) {
+    return pNode;
+  }
+  SNode* pChild;
+  FOREACH(pChild, pNode->pChildren) {
+    SLogicNode* pScanNode = optFindPossibleNode((SLogicNode*)pChild, func);
+    if (NULL != pScanNode) {
+      return pScanNode;
+    }
+  }
+  return NULL;
+}
+
+EDealRes osdHaveNormalColImpl(SNode* pNode, void* pContext) {
   if (QUERY_NODE_COLUMN == nodeType(pNode)) {
     *((bool*)pContext) = (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType);
     return *((bool*)pContext) ? DEAL_RES_END : DEAL_RES_IGNORE_CHILD;
@@ -65,9 +84,9 @@ EDealRes haveNormalColImpl(SNode* pNode, void* pContext) {
   return DEAL_RES_CONTINUE;
 }
 
-static bool haveNormalCol(SNodeList* pList) {
+static bool osdHaveNormalCol(SNodeList* pList) {
   bool res = false;
-  nodesWalkExprsPostOrder(pList, haveNormalColImpl, &res);
+  nodesWalkExprsPostOrder(pList, osdHaveNormalColImpl, &res);
   return res;
 }
 
@@ -89,21 +108,7 @@ static bool osdMayBeOptimized(SLogicNode* pNode) {
   if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent)) {
     return (WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType);
   }
-  return !haveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys);
-}
-
-static SLogicNode* osdFindPossibleScanNode(SLogicNode* pNode) {
-  if (osdMayBeOptimized(pNode)) {
-    return pNode;
-  }
-  SNode* pChild;
-  FOREACH(pChild, pNode->pChildren) {
-    SLogicNode* pScanNode = osdFindPossibleScanNode((SLogicNode*)pChild);
-    if (NULL != pScanNode) {
-      return pScanNode;
-    }
-  }
-  return NULL;
+  return !osdHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys);
 }
 
 static SNodeList* osdGetAllFuncs(SLogicNode* pNode) {
@@ -138,7 +143,7 @@ static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs,
 }
 
 static int32_t osdMatch(SOptimizeContext* pCxt, SLogicNode* pLogicNode, SOsdInfo* pInfo) {
-  pInfo->pScan = (SScanLogicNode*)osdFindPossibleScanNode(pLogicNode);
+  pInfo->pScan = (SScanLogicNode*)optFindPossibleNode(pLogicNode, osdMayBeOptimized);
   if (NULL == pInfo->pScan) {
     return TSDB_CODE_SUCCESS;
   }
@@ -345,7 +350,7 @@ static int32_t cpdCalcTimeRange(SScanLogicNode* pScan, SNode** pPrimaryKeyCond,
 }
 
 static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) {
-  if (NULL == pScan->node.pConditions) {
+  if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD)) {
     return TSDB_CODE_SUCCESS;
   }
 
@@ -359,7 +364,10 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode*
     pScan->node.pConditions = pOtherCond;
   }
 
-  if (TSDB_CODE_SUCCESS != code) {
+  if (TSDB_CODE_SUCCESS == code) {
+    OPTIMIZE_FLAG_SET_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD);
+    pCxt->optimized = true;
+  } else {
     nodesDestroyNode(pPrimaryKeyCond);
     nodesDestroyNode(pOtherCond);
   }
@@ -367,7 +375,7 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode*
   return code;
 }
 
-static bool belongThisTable(SNode* pCondCol, SNodeList* pTableCols) {
+static bool cpdBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) {
   SNode* pTableCol = NULL;
   FOREACH(pTableCol, pTableCols) {
     if (nodesEqualNode(pCondCol, pTableCol)) {
@@ -380,9 +388,9 @@ static bool belongThisTable(SNode* pCondCol, SNodeList* pTableCols) {
 static EDealRes cpdIsMultiTableCondImpl(SNode* pNode, void* pContext) {
   SCpdIsMultiTableCondCxt* pCxt = pContext;
   if (QUERY_NODE_COLUMN == nodeType(pNode)) {
-    if (belongThisTable(pNode, pCxt->pLeftCols)) {
+    if (cpdBelongThisTable(pNode, pCxt->pLeftCols)) {
       pCxt->havaLeftCol = true;
-    } else if (belongThisTable(pNode, pCxt->pRightCols)) {
+    } else if (cpdBelongThisTable(pNode, pCxt->pRightCols)) {
       pCxt->haveRightCol = true;
     }
     return pCxt->havaLeftCol && pCxt->haveRightCol ? DEAL_RES_END : DEAL_RES_CONTINUE;
@@ -508,11 +516,76 @@ static int32_t cpdPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SN
   return TSDB_CODE_PLAN_INTERNAL_ERROR;
 }
 
+static bool cpdIsPrimaryKey(SNode* pNode, SNodeList* pTableCols) {
+  if (QUERY_NODE_COLUMN != nodeType(pNode)) {
+    return false;
+  }
+  SColumnNode* pCol = (SColumnNode*)pNode;
+  if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) {
+    return false;
+  }
+  return cpdBelongThisTable(pNode, pTableCols);
+}
+
+static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) {
+  if (QUERY_NODE_OPERATOR != nodeType(pCond)) {
+    return false;
+  }
+  SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets;
+  SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets;
+  SOperatorNode* pOper = (SOperatorNode*)pJoin->pOnConditions;
+  if (cpdIsPrimaryKey(pOper->pLeft, pLeftCols)) {
+    return cpdIsPrimaryKey(pOper->pRight, pRightCols);
+  } else if (cpdIsPrimaryKey(pOper->pLeft, pRightCols)) {
+    return cpdIsPrimaryKey(pOper->pRight, pLeftCols);
+  }
+  return false;
+}
+
+static int32_t cpdCheckOpCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode* pOnCond) {
+  if (!cpdIsPrimaryKeyEqualCond(pJoin, pOnCond)) {
+    snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression");
+    return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SLogicConditionNode* pOnCond) {
+  if (LOGIC_COND_TYPE_AND != pOnCond->condType) {
+    snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression");
+    return TSDB_CODE_FAILED;
+  }
+  SNode* pCond = NULL;
+  FOREACH(pCond, pOnCond->pParameterList) {
+    if (!cpdIsPrimaryKeyEqualCond(pJoin, pCond)) {
+      snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression");
+      return TSDB_CODE_FAILED;
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
+  if (NULL == pJoin->pOnConditions) {
+    snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "not support cross join");
+    return TSDB_CODE_FAILED;
+  }
+  if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions)) {
+    return cpdCheckLogicCond(pCxt, pJoin, (SLogicConditionNode*)pJoin->pOnConditions);
+  } else {
+    return cpdCheckOpCond(pCxt, pJoin, pJoin->pOnConditions);
+  }
+}
+
 static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) {
-  if (NULL == pJoin->node.pConditions) {
+  if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_CPD)) {
     return TSDB_CODE_SUCCESS;
   }
 
+  if (NULL == pJoin->node.pConditions) {
+    return cpdCheckJoinOnCond(pCxt, pJoin);
+  }
+
   SNode* pOnCond = NULL;
   SNode* pLeftChildCond = NULL;
   SNode* pRightChildCond = NULL;
@@ -527,7 +600,11 @@ static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoi
     code = cpdPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond);
   }
 
-  if (TSDB_CODE_SUCCESS != code) {
+  if (TSDB_CODE_SUCCESS == code) {
+    OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_CPD);
+    pCxt->optimized = true;
+    code = cpdCheckJoinOnCond(pCxt, pJoin);
+  } else {
     nodesDestroyNode(pOnCond);
     nodesDestroyNode(pLeftChildCond);
     nodesDestroyNode(pRightChildCond);
@@ -572,15 +649,129 @@ static int32_t cpdOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
   return cpdPushCondition(pCxt, pLogicNode);
 }
 
+static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) {
+  if (1 != LIST_LENGTH(pSortKeys)) {
+    return false;
+  }
+  SNode* pNode = ((SOrderByExprNode*)nodesListGetNode(pSortKeys, 0))->pExpr;
+  return (QUERY_NODE_COLUMN == nodeType(pNode) ? (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) : false);
+}
+
+static bool opkSortMayBeOptimized(SLogicNode* pNode) {
+  if (QUERY_NODE_LOGIC_PLAN_SORT != nodeType(pNode)) {
+    return false;
+  }
+  if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_OPK)) {
+    return false;
+  }
+  return true;
+}
+
+static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeList** pScanNodes) {
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  switch (nodeType(pNode)) {
+    case QUERY_NODE_LOGIC_PLAN_SCAN:
+      return nodesListMakeAppend(pScanNodes, pNode);
+    case QUERY_NODE_LOGIC_PLAN_JOIN:
+      code = opkGetScanNodesImpl(nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes);
+      if (TSDB_CODE_SUCCESS == code) {
+        code = opkGetScanNodesImpl(nodesListGetNode(pNode->pChildren, 1), pNotOptimize, pScanNodes);
+      }
+      return code;
+    case QUERY_NODE_LOGIC_PLAN_AGG:
+      *pNotOptimize = true;
+      return code;
+    default:
+      break;
+  }
+
+  if (1 != LIST_LENGTH(pNode->pChildren)) {
+    *pNotOptimize = true;
+  }
+
+  return opkGetScanNodesImpl(nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes);
+}
+
+static int32_t opkGetScanNodes(SLogicNode* pNode, SNodeList** pScanNodes) {
+  bool notOptimize = false;
+  int32_t code = opkGetScanNodesImpl(pNode, &notOptimize, pScanNodes);
+  if (TSDB_CODE_SUCCESS != code || notOptimize) {
+    nodesClearList(*pScanNodes);
+  }
+  return code;
+}
+
+static EOrder opkGetPrimaryKeyOrder(SSortLogicNode* pSort) {
+  return ((SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0))->order;
+}
+
+static SNode* opkRewriteDownNode(SSortLogicNode* pSort) {
+  SNode* pDownNode = nodesListGetNode(pSort->node.pChildren, 0);
+  // todo
+  pSort->node.pChildren = NULL;
+  return pDownNode;
+}
+
+static int32_t opkDoOptimized(SOptimizeContext* pCxt, SSortLogicNode* pSort, SNodeList* pScanNodes) {
+  EOrder order = opkGetPrimaryKeyOrder(pSort);
+  if (ORDER_DESC == order) {
+    SNode* pScan = NULL;
+    FOREACH(pScan, pScanNodes) {
+      ((SScanLogicNode*)pScan)->scanSeq[0] = 0;
+      ((SScanLogicNode*)pScan)->scanSeq[1] = 1;
+    }
+  }
+
+  if (NULL == pSort->node.pParent) {
+    // todo
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SNode* pDownNode = opkRewriteDownNode(pSort);
+  SNode* pNode;
+  FOREACH(pNode, pSort->node.pParent->pChildren) {
+    if (nodesEqualNode(pNode, pSort)) {
+      REPLACE_NODE(pDownNode);
+      break;
+    }
+  }
+  nodesDestroyNode(pSort);
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t opkOptimizeImpl(SOptimizeContext* pCxt, SSortLogicNode* pSort) {
+  OPTIMIZE_FLAG_SET_MASK(pSort->node.optimizedFlag, OPTIMIZE_FLAG_OPK);
+  if (!opkIsPrimaryKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) {
+    return TSDB_CODE_SUCCESS;
+  }
+  SNodeList* pScanNodes = NULL;
+  int32_t code = opkGetScanNodes(nodesListGetNode(pSort->node.pChildren, 0), &pScanNodes);
+  if (TSDB_CODE_SUCCESS == code && NULL != pScanNodes) {
+    code = opkDoOptimized(pCxt, pSort, pScanNodes);
+  }
+  nodesClearList(pScanNodes);
+  return code;
+}
+
+static int32_t opkOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) {
+  SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicNode, opkSortMayBeOptimized);
+  if (NULL == pSort) {
+    return TSDB_CODE_SUCCESS;
+  }
+  return opkOptimizeImpl(pCxt, pSort);
+}
+
 static const SOptimizeRule optimizeRuleSet[] = {
   { .pName = "OptimizeScanData", .optimizeFunc = osdOptimize },
-  { .pName = "ConditionPushDown", .optimizeFunc = cpdOptimize }
+  { .pName = "ConditionPushDown", .optimizeFunc = cpdOptimize },
+  { .pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize }
 };
 
 static const int32_t optimizeRuleNum = (sizeof(optimizeRuleSet) / sizeof(SOptimizeRule));
 
-static int32_t applyOptimizeRule(SLogicNode* pLogicNode) {
-  SOptimizeContext cxt = { .optimized = false };
+static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicNode* pLogicNode) {
+  SOptimizeContext cxt = { .pPlanCxt = pCxt, .optimized = false };
   do {
     cxt.optimized = false;
     for (int32_t i = 0; i < optimizeRuleNum; ++i) {
@@ -594,5 +785,5 @@ static int32_t applyOptimizeRule(SLogicNode* pLogicNode) {
 }
 
 int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode) {
-  return applyOptimizeRule(pLogicNode);
+  return applyOptimizeRule(pCxt, pLogicNode);
 }

source/libs/planner/src/planPhysiCreater.c

@@ -398,9 +398,6 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SScanLogicNo
   if (TSDB_CODE_SUCCESS == code) {
     pScanPhysiNode->uid = pScanLogicNode->pMeta->uid;
     pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType;
-    pScanPhysiNode->order = TSDB_ORDER_ASC;
-    pScanPhysiNode->count = 1;
-    pScanPhysiNode->reverse = 0;
     memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName));
   }
 
@@ -432,7 +429,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  pTableScan->scanFlag = pScanLogicNode->scanFlag;
+  memcpy(pTableScan->scanSeq, pScanLogicNode->scanSeq, sizeof(pScanLogicNode->scanSeq));
   pTableScan->scanRange = pScanLogicNode->scanRange;
   pTableScan->ratio = pScanLogicNode->ratio;
   vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);

tools/shell/src/backup/tnettest.h → source/libs/planner/test/planOptTest.cpp

@@ -13,17 +13,20 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TNETTEST_H
-#define TDENGINE_TNETTEST_H
+#include "planTestUtil.h"
+#include "planner.h"
 
-#ifdef __cplusplus
-extern "C" {
-#endif
+using namespace std;
 
-void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType);
+class PlanOptimizeTest : public PlannerTestBase {
 
-#ifdef __cplusplus
-}
-#endif
+};
+
+TEST_F(PlanOptimizeTest, orderByPrimaryKey) {
+  useDb("root", "test");
 
-#endif  // TDENGINE_TNETTEST_H
+  run("select * from t1 order by ts");
+  run("select * from t1 order by ts desc");
+  run("select c1 from t1 order by ts");
+  run("select c1 from t1 order by ts desc");
+}

source/libs/qworker/src/qworkerMsg.c

@@ -346,7 +346,7 @@ int32_t qwRegisterHbBrokenLinkArg(SQWorkerMgmt *mgmt, uint64_t sId, SQWConnInfo
       .refId = pConn->refId,
       .msgType = TDMT_VND_QUERY_HEARTBEAT,
       .pCont = msg,
-      .contLen = sizeof(SSchedulerHbReq),
+      .contLen = msgSize,
       .code = TSDB_CODE_RPC_NETWORK_UNAVAIL,
   };
 

source/libs/scalar/src/sclfunc.c

@@ -32,8 +32,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
       float *in  = (float *)pInputData->pData;
       float *out = (float *)pOutputData->pData;
       for (int32_t i = 0; i < pInput->numOfRows; ++i) {
-        if (colDataIsNull_f(pInputData->nullbitmap, i)) {
-          colDataSetNull_f(pOutputData->nullbitmap, i);
+        if (colDataIsNull_s(pInputData, i)) {
+          colDataAppendNULL(pOutputData, i);
           continue;
         }
         out[i] = (in[i] >= 0)? in[i] : -in[i];
@@ -45,8 +45,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
       double *in  = (double *)pInputData->pData;
       double *out = (double *)pOutputData->pData;
       for (int32_t i = 0; i < pInput->numOfRows; ++i) {
-        if (colDataIsNull_f(pInputData->nullbitmap, i)) {
-          colDataSetNull_f(pOutputData->nullbitmap, i);
+        if (colDataIsNull_s(pInputData, i)) {
+          colDataAppendNULL(pOutputData, i);
           continue;
         }
         out[i] = (in[i] >= 0)? in[i] : -in[i];
@@ -58,8 +58,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
       int8_t *in  = (int8_t *)pInputData->pData;
       int8_t *out = (int8_t *)pOutputData->pData;
       for (int32_t i = 0; i < pInput->numOfRows; ++i) {
-        if (colDataIsNull_f(pInputData->nullbitmap, i)) {
-          colDataSetNull_f(pOutputData->nullbitmap, i);
+        if (colDataIsNull_s(pInputData, i)) {
+          colDataAppendNULL(pOutputData, i);
           continue;
         }
         out[i] = (in[i] >= 0)? in[i] : -in[i];
@@ -71,8 +71,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
       int16_t *in  = (int16_t *)pInputData->pData;
       int16_t *out = (int16_t *)pOutputData->pData;
       for (int32_t i = 0; i < pInput->numOfRows; ++i) {
-        if (colDataIsNull_f(pInputData->nullbitmap, i)) {
-          colDataSetNull_f(pOutputData->nullbitmap, i);
+        if (colDataIsNull_s(pInputData, i)) {
+          colDataAppendNULL(pOutputData, i);
           continue;
         }
         out[i] = (in[i] >= 0)? in[i] : -in[i];
@@ -84,8 +84,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
       int32_t *in  = (int32_t *)pInputData->pData;
       int32_t *out = (int32_t *)pOutputData->pData;
       for (int32_t i = 0; i < pInput->numOfRows; ++i) {
-        if (colDataIsNull_f(pInputData->nullbitmap, i)) {
-          colDataSetNull_f(pOutputData->nullbitmap, i);
+        if (colDataIsNull_s(pInputData, i)) {
+          colDataAppendNULL(pOutputData, i);
           continue;
         }
         out[i] = (in[i] >= 0)? in[i] : -in[i];
@@ -97,8 +97,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu
       int64_t *in  = (int64_t *)pInputData->pData;
       int64_t *out = (int64_t *)pOutputData->pData;
       for (int32_t i = 0; i < pInput->numOfRows; ++i) {
-        if (colDataIsNull_f(pInputData->nullbitmap, i)) {
-          colDataSetNull_f(pOutputData->nullbitmap, i);
+        if (colDataIsNull_s(pInputData, i)) {
+          colDataAppendNULL(pOutputData, i);
           continue;
         }
         out[i] = (in[i] >= 0)? in[i] : -in[i];
@@ -129,8 +129,8 @@ static int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SS
   double *out = (double *)pOutputData->pData;
 
   for (int32_t i = 0; i < pInput->numOfRows; ++i) {
-    if (colDataIsNull_f(pInputData->nullbitmap, i)) {
-      colDataSetNull_f(pOutputData->nullbitmap, i);
+    if (colDataIsNull_s(pInputData, i)) {
+      colDataAppendNULL(pOutputData, i);
       continue;
     }
     out[i] = valFn(getValueFn(pInputData->pData, i));
@@ -157,9 +157,9 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S
   double *out = (double *)pOutputData->pData;
 
   for (int32_t i = 0; i < pInput->numOfRows; ++i) {
-    if (colDataIsNull_f(pInputData[0]->nullbitmap, i) ||
-        colDataIsNull_f(pInputData[1]->nullbitmap, 0)) {
-      colDataSetNull_f(pOutputData->nullbitmap, i);
+    if (colDataIsNull_s(pInputData[0], i) ||
+        colDataIsNull_s(pInputData[1], 0)) {
+      colDataAppendNULL(pOutputData, i);
       continue;
     }
     out[i] = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, 0));
@@ -184,8 +184,8 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
       float *out = (float *)pOutputData->pData;
 
       for (int32_t i = 0; i < pInput->numOfRows; ++i) {
-        if (colDataIsNull_f(pInputData->nullbitmap, i)) {
-          colDataSetNull_f(pOutputData->nullbitmap, i);
+        if (colDataIsNull_s(pInputData, i)) {
+          colDataAppendNULL(pOutputData, i);
           continue;
         }
         out[i] = f1(in[i]);
@@ -198,8 +198,8 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
       double *out = (double *)pOutputData->pData;
 
       for (int32_t i = 0; i < pInput->numOfRows; ++i) {
-        if (colDataIsNull_f(pInputData->nullbitmap, i)) {
-          colDataSetNull_f(pOutputData->nullbitmap, i);
+        if (colDataIsNull_s(pInputData, i)) {
+          colDataAppendNULL(pOutputData, i);
           continue;
         }
         out[i] = d1(in[i]);
@@ -301,7 +301,7 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
 
   for (int32_t i = 0; i < pInput->numOfRows; ++i) {
     if (colDataIsNull_s(pInputData, i)) {
-      colDataSetNull_f(pOutputData->nullbitmap, i);
+      colDataAppendNULL(pOutputData, i);
       continue;
     }
 
@@ -1247,26 +1247,28 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
 }
 
 int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
-  if (inputNum != 1) {
-    return TSDB_CODE_FAILED;
+  int64_t ts = taosGetTimestamp(TSDB_TIME_PRECISION_MILLI);
+  for (int32_t i = 0; i < pInput->numOfRows; ++i) {
+    colDataAppendInt64(pOutput->columnData, i, &ts);
   }
-  colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 0));
+  pOutput->numOfRows = pInput->numOfRows;
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
-  if (inputNum != 1) {
-    return TSDB_CODE_FAILED;
+  int64_t ts = taosGetTimestampToday(TSDB_TIME_PRECISION_MILLI);
+  for (int32_t i = 0; i < pInput->numOfRows; ++i) {
+    colDataAppendInt64(pOutput->columnData, i, &ts);
   }
-  colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 0));
+  pOutput->numOfRows = pInput->numOfRows;
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
-  if (inputNum != 1) {
-    return TSDB_CODE_FAILED;
+  for (int32_t i = 0; i < pInput->numOfRows; ++i) {
+    colDataAppend(pOutput->columnData, i, tsTimezoneStr, false);
   }
-  colDataAppend(pOutput->columnData, pOutput->numOfRows, (char *)colDataGetData(pInput->columnData, 0), false);
+  pOutput->numOfRows = pInput->numOfRows;
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/sync/src/syncIO.c

@@ -170,7 +170,6 @@ static int32_t syncIOStartInternal(SSyncIO *io) {
   taosBlockSIGPIPE();
 
   rpcInit();
-  tsRpcForceTcp = 1;
 
   // cient rpc init
   {

source/os/src/osSystem.c

@@ -119,7 +119,7 @@ int taosSetConsoleEcho(bool on) {
 #endif
 }
 
-void setTerminalMode() {
+void taosSetTerminalMode() {
 #if defined(WINDOWS)
 
 #else
@@ -152,7 +152,7 @@ void setTerminalMode() {
 #endif
 }
 
-int32_t getOldTerminalMode() {
+int32_t taosGetOldTerminalMode() {
 #if defined(WINDOWS)
   
 #else
@@ -170,7 +170,7 @@ int32_t getOldTerminalMode() {
 #endif
 }
 
-void resetTerminalMode() {
+void taosResetTerminalMode() {
 #if defined(WINDOWS)
 
 #else

source/util/src/tconfig.c

@@ -513,6 +513,8 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
   for (int32_t i = 0; i < size; ++i) {
     SConfigItem *pItem = taosArrayGet(pCfg->array, i);
     if (tsc && !pItem->tsc) continue;
+    if (dump && strcmp(pItem->name, "scriptDir") == 0) continue;
+    if (dump && strcmp(pItem->name, "simDebugFlag") == 0) continue;
     tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN);
     for (int32_t i = 0; i < CFG_SRC_PRINT_LEN; ++i) {
       if (src[i] == 0) src[i] = ' ';
@@ -551,10 +553,10 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
         break;
       case CFG_DTYPE_FLOAT:
         if (dump) {
-          printf("%s %s %f", src, name, pItem->fval);
+          printf("%s %s %.2f", src, name, pItem->fval);
           printf("\n");
         } else {
-          uInfo("%s %s %f", src, name, pItem->fval);
+          uInfo("%s %s %.2f", src, name, pItem->fval);
         }
         break;
       case CFG_DTYPE_STRING:

source/util/src/ttimer.c

@@ -110,7 +110,7 @@ typedef struct time_wheel_t {
   tmr_obj_t**     slots;
 } time_wheel_t;
 
-int32_t tsMaxTmrCtrl = 512;
+static int32_t tsMaxTmrCtrl = 512;
 
 static TdThreadOnce  tmrModuleInit = PTHREAD_ONCE_INIT;
 static TdThreadMutex tmrCtrlMutex;

tests/script/jenkins/basic.txt

@@ -76,7 +76,7 @@
 ./test.sh -f tsim/insert/backquote.sim -m
 ./test.sh -f tsim/parser/fourArithmetic-basic.sim -m
 ./test.sh -f tsim/query/interval-offset.sim -m
-./test.sh -f tsim/tmq/basic1.sim -m
+#./test.sh -f tsim/tmq/basic1.sim -m
 ./test.sh -f tsim/stable/vnode3.sim -m
 ./test.sh -f tsim/qnode/basic1.sim -m
 ./test.sh -f tsim/mnode/basic1.sim -m
@@ -85,6 +85,6 @@
 ./test.sh -f tsim/sma/tsmaCreateInsertData.sim
 
 # --- valgrind
-./test.sh -f tsim/valgrind/checkError.sim -v
+#./test.sh -f tsim/valgrind/checkError.sim -v
 
 #======================b1-end===============

tests/script/tsim/query/crash_sql.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+    $loop_cnt = $loop_cnt + 1
+    sleep 200
+    if $loop_cnt == 10 then
+        print ====> dnode not ready!
+        return -1
+    endi
+
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+sql connect
+
+print =============== create database
+sql create database db
+sql show databases
+if $rows != 2 then
+  return -1
+endi
+
+sql use db
+
+print =============== create super table and child table
+sql create table stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)
+sql show stables
+print $rows $data00 $data01 $data02
+if $rows != 1 then
+  return -1
+endi
+
+sql create table ct1 using stb1 tags ( 1 )
+sql create table ct2 using stb1 tags ( 2 )
+sql create table ct3 using stb1 tags ( 3 )
+sql create table ct4 using stb1 tags ( 4 )
+sql show tables
+print $rows $data00 $data10 $data20
+if $rows != 4 then
+  return -1
+endi
+
+print =============== insert data into child table ct1 (s)
+sql insert into ct1 values ( '2022-01-01 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a )
+sql insert into ct1 values ( '2022-01-01 01:01:06.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a )
+sql insert into ct1 values ( '2022-01-01 01:01:10.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a )
+sql insert into ct1 values ( '2022-01-01 01:01:16.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a )
+sql insert into ct1 values ( '2022-01-01 01:01:20.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a )
+sql insert into ct1 values ( '2022-01-01 01:01:26.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a )
+sql insert into ct1 values ( '2022-01-01 01:01:30.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", now+7a )
+sql insert into ct1 values ( '2022-01-01 01:01:36.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", now+8a )
+
+print =============== insert data into child table ct2 (d)
+sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a )
+sql insert into ct2 values ( '2022-01-01 10:00:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a )
+sql insert into ct2 values ( '2022-01-01 20:00:01.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a )
+sql insert into ct2 values ( '2022-01-02 10:00:01.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a )
+sql insert into ct2 values ( '2022-01-02 20:00:01.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a )
+sql insert into ct2 values ( '2022-01-03 10:00:01.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", now+6a )
+sql insert into ct2 values ( '2022-01-03 20:00:01.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", now+7a )
+
+print =============== insert data into child table ct3 (n)
+sql insert into ct3 values ( '2021-12-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+sql insert into ct3 values ( '2021-12-31 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a )
+sql insert into ct3 values ( '2022-01-01 01:01:06.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a )
+sql insert into ct3 values ( '2022-01-07 01:01:10.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a )
+sql insert into ct3 values ( '2022-01-31 01:01:16.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a )
+sql insert into ct3 values ( '2022-02-01 01:01:20.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a )
+sql insert into ct3 values ( '2022-02-28 01:01:26.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a )
+sql insert into ct3 values ( '2022-03-01 01:01:30.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
+sql insert into ct3 values ( '2022-03-08 01:01:36.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
+
+print =============== insert data into child table ct4 (y)
+sql insert into ct4 values ( '2019-01-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+sql insert into ct4 values ( '2019-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a )
+sql insert into ct4 values ( '2019-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a )
+sql insert into ct4 values ( '2020-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a )
+sql insert into ct4 values ( '2020-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a )
+sql insert into ct4 values ( '2020-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a )
+sql insert into ct4 values ( '2020-12-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+sql insert into ct4 values ( '2021-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a )
+sql insert into ct4 values ( '2021-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
+sql insert into ct4 values ( '2021-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
+sql insert into ct4 values ( '2022-02-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
+sql insert into ct4 values ( '2022-05-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+
+
+print ================ start query ======================
+print ================ SQL used to cause taosd or taos shell  crash
+sql select sum(c1) ,count(c1) from ct4 group by c1 having  sum(c10)  between 0 and 1 ;
+
+
+
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file

tests/script/tsim/tmq/basic2Of2ConsOverlap.sim

@@ -292,7 +292,7 @@ print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $
 print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6]
 if $rows != 2 then
   sleep 1000
-  goto wait_consumer_end_from_ctb
+  goto wait_consumer_end_from_ntb
 endi
 if $data[0][1] == 0 then
   if $data[1][1] != 1 then

tests/system-test/2-query/cast.py

@@ -6,6 +6,7 @@ import inspect
 from util.log import *
 from util.sql import *
 from util.cases import *
+from util.dnodes import *
 
 
 
@@ -15,56 +16,50 @@ class TDTestCase:
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())
 
-    def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
-        tdSql.prepare()
+    def __cast_to_bigint(self, col_name, tbname):
+        __sql = f"select cast({col_name} as bigint), {col_name} from {tbname}"
+        tdSql.query(sql=__sql)
+        data_tb_col = [result[1] for result in tdSql.queryResult]
+        for i in range(len(tdSql.queryRows)):
+            tdSql.checkData( i, 0, None ) if data_tb_col[i] is None else tdSql.checkData( i, 0, int(data_tb_col[i]) )
+
+    def __range_to_bigint(self,cols,tables):
+        for col in cols:
+            for table in tables:
+                self.__cast_to_bigint(col_name=col, tbname=table)
+
+    def __cast_to_timestamp(self, col_name, tbname):
+        __sql = f"select cast({col_name} as timestamp), {col_name} from {tbname}"
+        tdSql.query(sql=__sql)
+        data_tb_col = [result[1] for result in tdSql.queryResult]
+        for i in range(len(tdSql.queryRows)):
+            if data_tb_col[i] is None:
+                tdSql.checkData( i, 0 , None )
+            if (col_name == "c2" or col_name == "double" ) and tbname == "t1" and i == 10:
+                continue
+            else:
+                utc_zone = datetime.timezone.utc
+                utc_8 = datetime.timezone(datetime.timedelta(hours=8))
+                date_init_stamp = datetime.datetime.utcfromtimestamp(data_tb_col[i]/1000)
+                date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f")
+                tdSql.checkData( i, 0, date_data)
 
-        tdLog.printNoPrefix("==========step1:create table")
-        tdSql.execute(
-            '''create table stb1
-            (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
-            tags (t1 int)
-            '''
-        )
-        tdSql.execute(
-            '''
-            create table t1
-            (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
-            '''
-        )
-        for i in range(4):
-            tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+    def __range_to_timestamp(self, cols, tables):
+        for col in cols:
+            for table in tables:
+                self.__cast_to_timestamp(col_name=col, tbname=table)
 
-        tdLog.printNoPrefix("==========step2:insert data")
-        for i in range(9):
-            tdSql.execute(
-                f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
-            )
-            tdSql.execute(
-                f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
-            )
-        tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
-        tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
+    def __test_bigint(self):
+        __table_list = ["ct1", "ct4", "t1"]
+        __col_list = ["c1","c2","c3","c4","c5","c6","c7","c10","c1+c2"]
+        self.__range_to_bigint(cols=__col_list, tables=__table_list)
 
-        tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
-        tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
-        tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL  ) ")
+    def __test_timestamp(self):
+        __table_list = ["ct1", "ct4", "t1"]
+        __col_list = ["c1","c2","c3","c4","c5","c6","c7","c1+c2"]
+        self.__range_to_timestamp(cols=__col_list, tables=__table_list)
 
-        tdSql.execute(
-            f'''insert into t1 values
-            ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
-            ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
-            ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
-            ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
-            ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
-            ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
-            ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
-            ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
-            ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
-            ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
-            ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
-            ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
-            '''
-        )
+    def all_test(self):
 
         tdSql.query("select c1  from ct4")
         data_ct4_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
@@ -82,9 +77,9 @@ class TDTestCase:
 
         tdLog.printNoPrefix("==========step5: cast int to binary, expect changes to str(int) ")
 
-        tdSql.query("select cast(c1 as binary(32)) as b from ct4")
-        for i in range(len(data_ct4_c1)):
-            tdSql.checkData( i, 0, str(data_ct4_c1[i]) )
+        #tdSql.query("select cast(c1 as binary(32)) as b from ct4")
+        #for i in range(len(data_ct4_c1)):
+        #    tdSql.checkData( i, 0, str(data_ct4_c1[i]) )
         tdSql.query("select cast(c1 as binary(32)) as b from t1")
         for i in range(len(data_t1_c1)):
             tdSql.checkData( i, 0, str(data_t1_c1[i]) )
@@ -240,7 +235,7 @@ class TDTestCase:
                 tdSql.checkData( i, 0, date_data)
 
 
-        tdLog.printNoPrefix("==========step16: cast smallint to bigint, expect no changes")
+        tdLog.printNoPrefix("==========step16: cast tinyint to bigint, expect no changes")
         tdSql.query("select c4  from ct4")
         data_ct4_c4 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
         tdSql.query("select c4  from t1")
@@ -254,7 +249,7 @@ class TDTestCase:
             tdSql.checkData( i, 0, data_t1_c4[i])
 
 
-        tdLog.printNoPrefix("==========step17: cast smallint to binary, expect changes to str(int) ")
+        tdLog.printNoPrefix("==========step17: cast tinyint to binary, expect changes to str(int) ")
 
         tdSql.query("select cast(c4 as binary(32)) as b from ct4")
         for i in range(len(data_ct4_c4)):
@@ -263,7 +258,7 @@ class TDTestCase:
         for i in range(len(data_t1_c4)):
             tdSql.checkData( i, 0, str(data_t1_c4[i]) )
 
-        tdLog.printNoPrefix("==========step18: cast smallint to nchar, expect changes to str(int) ")
+        tdLog.printNoPrefix("==========step18: cast tinyint to nchar, expect changes to str(int) ")
 
         tdSql.query("select cast(c4 as nchar(32)) as b from ct4")
         for i in range(len(data_ct4_c4)):
@@ -272,7 +267,7 @@ class TDTestCase:
         for i in range(len(data_t1_c4)):
             tdSql.checkData( i, 0, str(data_t1_c4[i]) )
 
-        tdLog.printNoPrefix("==========step19: cast smallint to timestamp, expect changes to timestamp ")
+        tdLog.printNoPrefix("==========step19: cast tinyint to timestamp, expect changes to timestamp ")
 
         tdSql.query("select cast(c4 as timestamp) as b from ct4")
         for i in range(len(data_ct4_c4)):
@@ -624,7 +619,67 @@ class TDTestCase:
         tdSql.error("select cast(c8 as timestamp ) as b from ct4")
         tdSql.error("select cast(c9 as timestamp ) as b from ct4")
         tdSql.error("select cast(c9 as binary(64) ) as b from ct4")
+        pass
+
+    def run(self):
+        tdSql.prepare()
+
+        tdLog.printNoPrefix("==========step1:create table")
+        tdSql.execute(
+            '''create table stb1
+            (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+            tags (t1 int)
+            '''
+        )
+        tdSql.execute(
+            '''
+            create table t1
+            (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+            '''
+        )
+        for i in range(4):
+            tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+        tdLog.printNoPrefix("==========step2:insert data")
+        for i in range(9):
+            tdSql.execute(
+                f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
+            )
+            tdSql.execute(
+                f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
+            )
+        tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
+        tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
+
+        tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
+        tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
+        tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL  ) ")
+
+        tdSql.execute(
+            f'''insert into t1 values
+            ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
+            ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
+            ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
+            ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
+            ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
+            ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
+            ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
+            ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
+            ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
+            ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            '''
+        )
+
+        self.all_test()
+
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+
+        tdSql.execute("use db")
 
+        self.all_test()
 
     def stop(self):
         tdSql.close()

tests/system-test/2-query/sum.py

+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+
+
+INT_COL     = "c1"
+BINT_COL    = "c2"
+SINT_COL    = "c3"
+TINT_COL    = "c4"
+FLOAT_COL   = "c5"
+DOUBLE_COL  = "c6"
+BOOL_COL    = "c7"
+
+BINARY_COL  = "c8"
+NCHAR_COL   = "c9"
+TS_COL      = "c10"
+
+NUM_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ]
+UN_NUM_COL = [BOOL_COL, BINARY_COL, NCHAR_COL, ]
+TS_TYPE_COL = [TS_COL]
+
+class TDTestCase:
+
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+
+    def __sum_condition(self):
+        sum_condition = []
+        for num_col in NUM_COL:
+            sum_condition.extend(
+                (
+                    num_col,
+                    f"ceil( {num_col} )",
+                )
+            )
+            sum_condition.extend( f"{num_col} + {num_col_2}" for num_col_2 in NUM_COL )
+            sum_condition.extend( f"{num_col} + {un_num_col} " for un_num_col in UN_NUM_COL )
+
+        sum_condition.append(1)
+
+        return sum_condition
+
+    def __where_condition(self, col):
+        return f" where abs( {col} ) < 1000000 "
+
+    def __group_condition(self, col, having = ""):
+        return f" group by {col} having {having}" if having else f" group by {col} "
+
+    def __sum_current_check(self, tbname):
+        sum_condition = self.__sum_condition()
+        for condition in sum_condition:
+            where_condition = self.__where_condition(condition)
+            group_condition = self.__group_condition(condition, having=f"{condition} is not null " )
+
+            tdSql.query(f"select {condition} from {tbname} {where_condition} ")
+            datas = [tdSql.getData(i,0) for i in range(tdSql.queryRows)]
+            sum_data = sum(filter(None, datas))
+            tdSql.query(f"select sum( {condition} ) from {tbname} {where_condition} ")
+            tdSql.checkData(0, 0, sum_data)
+
+            tdSql.query(f"select {condition} from {tbname} {where_condition} {group_condition} ")
+
+    def __sum_err_check(self,tbanme):
+        sqls = []
+
+        for un_num_col in UN_NUM_COL:
+            sqls.extend(
+                (
+                    f"select sum( {un_num_col} ) from {tbanme} ",
+                    f"select sum(ceil( {un_num_col} )) from {tbanme} ",
+                )
+            )
+            sqls.extend( f"select sum( {un_num_col} + {un_num_col_2} ) from {tbanme} " for un_num_col_2 in UN_NUM_COL )
+
+        sqls.extend( f"select sum( {num_col} + {ts_col} ) from {tbanme} " for num_col in NUM_COL for ts_col in TS_TYPE_COL)
+        sqls.extend(
+            (
+                f"select sum() from {tbanme} ",
+                f"select sum(*) from {tbanme} ",
+                f"select sum(ccccccc) from {tbanme} ",
+                f"select sum('test') from {tbanme} ",
+            )
+        )
+
+        return sqls
+
+    def __test_current(self):
+        tdLog.printNoPrefix("==========current sql condition check , must return query ok==========")
+        tbname = ["ct1", "ct2", "ct4", "t1"]
+        for tb in tbname:
+            self.__sum_current_check(tb)
+            tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========")
+
+    def __test_error(self):
+        tdLog.printNoPrefix("==========err sql condition check , must return error==========")
+        tbname = ["ct1", "ct2", "ct4", "t1"]
+
+        for tb in tbname:
+            for errsql in self.__sum_err_check(tb):
+                tdSql.error(sql=errsql)
+            tdLog.printNoPrefix(f"==========err sql condition check in {tb} over==========")
+
+
+    def all_test(self):
+        self.__test_current()
+        self.__test_error()
+
+
+    def __create_tb(self):
+        tdSql.prepare()
+
+        tdLog.printNoPrefix("==========step1:create table")
+        create_stb_sql  =  f'''create table stb1(
+                ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
+                 {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
+                 {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
+            ) tags (t1 int)
+            '''
+        create_ntb_sql = f'''create table t1(
+                ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
+                 {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool,
+                 {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp
+            )
+            '''
+        tdSql.execute(create_stb_sql)
+        tdSql.execute(create_ntb_sql)
+
+        for i in range(4):
+            tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+    def __insert_data(self, rows):
+        for i in range(9):
+            tdSql.execute(
+                f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
+            )
+            tdSql.execute(
+                f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
+            )
+            tdSql.execute(
+                f"insert into ct2 values ( now()-{i*90}d, {-1*i}, {-11111*i}, {-111*i}, {-11*i}, {-1.11*i}, {-11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
+            )
+        tdSql.execute(
+            '''insert into ct1 values
+            ( now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )
+            ( now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )
+            '''
+        )
+
+        tdSql.execute(
+            f'''insert into ct4 values
+            ( now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( now()+{rows * 9}d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            (
+                now()+{rows * 9-10}d, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127,
+                { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nachar_limit-1", now()-1d
+                )
+            (
+                now()+{rows * 9-20}d, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126,
+                { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nachar_limit-2", now()-2d
+                )
+            '''
+        )
+
+        tdSql.execute(
+            f'''insert into ct2 values
+            ( now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( now()+{rows * 9}d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            (
+                now()+{rows * 9-10}d, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126,
+                { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nachar_limit-1", now()-1d
+                )
+            (
+                now()+{rows * 9-20}d, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127,
+                { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nachar_limit-2", now()-2d
+                )
+            '''
+        )
+
+        for i in range(rows):
+            insert_data = f'''insert into t1 values
+                ( now()-{i}h, {i}, {i}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2},
+                "binary_{i}", "nchar_{i}", now()-{i}s )
+                '''
+            tdSql.execute(insert_data)
+        tdSql.execute(
+            f'''insert into t1 values
+            ( now() + 3h, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( now()-{ ( rows // 2 ) * 60 + 30 }m, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( now()-{rows}h, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( now() + 2h, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127,
+                { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 },
+                "binary_limit-1", "nachar_limit-1", now()-1d
+                )
+            (
+                now() + 1h , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126,
+                { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 },
+                "binary_limit-2", "nachar_limit-2", now()-2d
+                )
+            '''
+        )
+
+
+    def run(self):
+        tdSql.prepare()
+
+        tdLog.printNoPrefix("==========step1:create table")
+        self.__create_tb()
+
+        tdLog.printNoPrefix("==========step2:insert data")
+        self.__insert_data(100)
+
+        tdLog.printNoPrefix("==========step3:all check")
+        self.all_test()
+
+        # tdDnodes.stop(1)
+        # tdDnodes.start(1)
+
+        # tdSql.execute("use db")
+
+        # tdLog.printNoPrefix("==========step4:after wal, all check again ")
+        # self.all_test()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/fulltest.sh

@@ -2,6 +2,6 @@
 set -e
 
 #python3 ./test.py -f 2-query/between.py
-python3 ./test.py -f 2-query/distinct.py
+#python3 ./test.py -f 2-query/distinct.py
 python3 ./test.py -f 2-query/varchar.py
-python3 ./test.py -f 2-query/cast.py
+#python3 ./test.py -f 2-query/cast.py

tests/test/c/tmqSim.c

@@ -125,13 +125,13 @@ void saveConfigToLogFile() {
   for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) {	  
     taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId);
 	taosFprintfFile(g_fp, "  Topics: ");
-    for (int i = 0 ; i < g_stConfInfo.stThreads[i].numOfTopic; i++) {
-    taosFprintfFile(g_fp, "%s, ",  g_stConfInfo.stThreads[i].topics[i]);
+    for (int j = 0 ; j < g_stConfInfo.stThreads[i].numOfTopic; j++) {
+    taosFprintfFile(g_fp, "%s, ",  g_stConfInfo.stThreads[i].topics[j]);
     }
     taosFprintfFile(g_fp, "\n");  
     taosFprintfFile(g_fp, "  Key: ");
-    for (int i = 0 ; i < g_stConfInfo.stThreads[i].numOfKey; i++) {
-      taosFprintfFile(g_fp, "%s:%s, ",  g_stConfInfo.stThreads[i].key[i], g_stConfInfo.stThreads[i].value[i]);
+    for (int k = 0 ; k < g_stConfInfo.stThreads[i].numOfKey; k++) {
+      taosFprintfFile(g_fp, "%s:%s, ",  g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]);
     }
     taosFprintfFile(g_fp, "\n");
   }

tools/shell/inc/shell.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_SHELL_H_
-#define _TD_SHELL_H_
-
-#include "os.h"
-
-#include "taos.h"
-#include "taosdef.h"
-
-#define MAX_USERNAME_SIZE      64
-#define MAX_DBNAME_SIZE        64
-#define MAX_IP_SIZE            20
-#define MAX_HISTORY_SIZE       1000
-#define MAX_COMMAND_SIZE       1048586
-#define HISTORY_FILE           ".taos_history"
-
-#define DEFAULT_RES_SHOW_NUM   100
-
-typedef struct SShellHistory {
-  char* hist[MAX_HISTORY_SIZE];
-  int   hstart;
-  int   hend;
-} SShellHistory;
-
-typedef struct SShellArguments {
-  char* host;
-  char* password;
-  char* user;
-  char* auth;
-  char* database;
-  char* timezone;
-  bool  is_raw_time;
-  bool  is_use_passwd;
-  bool  dump_config;
-  char  file[TSDB_FILENAME_LEN];
-  char  dir[TSDB_FILENAME_LEN];
-  int   threadNum;
-  int   check;
-  bool  status;
-  bool  verbose;
-  char* commands;
-  int   abort;
-  int   port;
-  int   pktLen;
-  int   pktNum;
-  char* pktType;
-  char* netTestRole;
-} SShellArguments;
-
-/**************** Function declarations ****************/
-extern void shellParseArgument(int argc, char* argv[], SShellArguments* arguments);
-extern TAOS* shellInit(SShellArguments* args);
-extern void* shellLoopQuery(void* arg);
-extern void taos_error(TAOS_RES* tres, int64_t st);
-extern int regex_match(const char* s, const char* reg, int cflags);
-int32_t shellReadCommand(TAOS* con, char command[]);
-int32_t shellRunCommand(TAOS* con, char* command);
-void shellRunCommandOnServer(TAOS* con, char command[]);
-void read_history();
-void write_history();
-void source_file(TAOS* con, char* fptr);
-void source_dir(TAOS* con, SShellArguments* args);
-void get_history_path(char* history);
-void shellCheck(TAOS* con, SShellArguments* args);
-void cleanup_handler(void* arg);
-void exitShell();
-int shellDumpResult(TAOS_RES* con, char* fname, int* error_no, bool printMode);
-void shellGetGrantInfo(void *con);
-int isCommentLine(char *line);
-
-/**************** Global variable declarations ****************/
-extern char           PROMPT_HEADER[];
-extern char           CONTINUE_PROMPT[];
-extern int            prompt_size;
-extern SShellHistory  history;
-extern SShellArguments args;
-extern int64_t         result;
-
-#endif

tools/shell/inc/shellCommand.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_SHELL_COMMAND_H_
-#define _TD_SHELL_COMMAND_H_
-
-#include "shell.h"
-
-#define LEFT  1
-#define RIGHT 2
-#define UP    3
-#define DOWN  4
-
-typedef struct Command Command;
-struct Command {
-  char *   buffer;
-  char *   command;
-  unsigned commandSize;
-  unsigned bufferSize;
-  unsigned cursorOffset;
-  unsigned screenOffset;
-  unsigned endOffset;
-};
-
-extern void backspaceChar(Command *cmd);
-extern void clearLineBefore(Command *cmd);
-extern void clearLineAfter(Command *cmd);
-extern void deleteChar(Command *cmd);
-extern void moveCursorLeft(Command *cmd);
-extern void moveCursorRight(Command *cmd);
-extern void positionCursorHome(Command *cmd);
-extern void positionCursorEnd(Command *cmd);
-extern void showOnScreen(Command *cmd);
-extern void updateBuffer(Command *cmd);
-extern int isReadyGo(Command *cmd);
-extern void resetCommand(Command *cmd, const char s[]);
-
-int countPrefixOnes(unsigned char c);
-void clearScreen(int ecmd_pos, int cursor_pos);
-void printChar(char c, int times);
-void positionCursor(int step, int direction);
-
-#endif

tools/shell/inc/shellInt.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_SHELL_INT_H_
+#define _TD_SHELL_INT_H_
+
+#include "os.h"
+#include "taos.h"
+#include "taosdef.h"
+#include "taoserror.h"
+#include "tconfig.h"
+#include "tglobal.h"
+#include "trpc.h"
+#include "ttypes.h"
+#include "tutil.h"
+
+#define SHELL_MAX_HISTORY_SIZE                 1000
+#define SHELL_MAX_COMMAND_SIZE                 1048586
+#define SHELL_HISTORY_FILE                     ".taos_history"
+#define SHELL_DEFAULT_RES_SHOW_NUM             100
+#define SHELL_DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30
+#define SHELL_MAX_PKG_LEN                      2 * 1024 * 1024
+#define SHELL_MIN_PKG_LEN                      1
+#define SHELL_DEF_PKG_LEN                      1024
+#define SHELL_MAX_PKG_NUM                      1 * 1024 * 1024
+#define SHELL_MIN_PKG_NUM                      1
+#define SHELL_DEF_PKG_NUM                      100
+
+typedef struct {
+  char*   hist[SHELL_MAX_HISTORY_SIZE];
+  char    file[TSDB_FILENAME_LEN];
+  int32_t hstart;
+  int32_t hend;
+} SShellHistory;
+
+typedef struct {
+  const char* host;
+  const char* user;
+  const char* auth;
+  const char* database;
+  const char* cfgdir;
+  const char* commands;
+  const char* netrole;
+  char        file[PATH_MAX];
+  char        password[TSDB_USET_PASSWORD_LEN];
+  bool        is_gen_auth;
+  bool        is_raw_time;
+  bool        is_version;
+  bool        is_dump_config;
+  bool        is_check;
+  bool        is_startup;
+  bool        is_help;
+  uint16_t    port;
+  int32_t     pktLen;
+  int32_t     pktNum;
+  int32_t     displayWidth;
+  int32_t     abort;
+} SShellArgs;
+
+typedef struct {
+  const char* clientVersion;
+  const char* promptHeader;
+  const char* promptContinue;
+  const char* osname;
+  int32_t     promptSize;
+  char        programVersion[32];
+} SShellOsDetails;
+
+typedef struct {
+  SShellArgs      args;
+  SShellHistory   history;
+  SShellOsDetails info;
+  TAOS*           conn;
+  TdThread        pid;
+  tsem_t          cancelSem;
+  int64_t         result;
+} SShellObj;
+
+// shellArguments.c
+int32_t shellParseArgs(int32_t argc, char* argv[]);
+
+// shellCommand.c
+int32_t shellReadCommand(char* command);
+
+// shellEngine.c
+int32_t shellExecute();
+
+// shellUtil.c
+int32_t shellCheckIntSize();
+void    shellPrintVersion();
+void    shellPrintHelp();
+void    shellGenerateAuth();
+void    shellDumpConfig();
+void    shellCheckServerStatus();
+bool    shellRegexMatch(const char* s, const char* reg, int32_t cflags);
+void    shellExit();
+
+// shellNettest.c
+void shellTestNetWork();
+
+// shellMain.c
+extern SShellObj shell;
+extern void      taos_init();
+
+#endif /*_TD_SHELL_INT_H_*/

tools/shell/inc/syncMsg.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_SYNC_MSG_H
-#define TDENGINE_SYNC_MSG_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "tsync.h"
-
-typedef enum {
-  TAOS_SMSG_START         = 0,
-  TAOS_SMSG_SYNC_DATA     = 1,
-  TAOS_SMSG_SYNC_DATA_RSP = 2,
-  TAOS_SMSG_SYNC_FWD      = 3,
-  TAOS_SMSG_SYNC_FWD_RSP  = 4,
-  TAOS_SMSG_SYNC_REQ      = 5,
-  TAOS_SMSG_SYNC_REQ_RSP  = 6,
-  TAOS_SMSG_SYNC_MUST     = 7,
-  TAOS_SMSG_SYNC_MUST_RSP = 8,
-  TAOS_SMSG_STATUS        = 9,
-  TAOS_SMSG_STATUS_RSP    = 10,
-  TAOS_SMSG_SETUP         = 11,
-  TAOS_SMSG_SETUP_RSP     = 12,
-  TAOS_SMSG_SYNC_FILE     = 13,
-  TAOS_SMSG_SYNC_FILE_RSP = 14,
-  TAOS_SMSG_TEST          = 15,
-  TAOS_SMSG_END           = 16
-} ESyncMsgType;
-
-typedef enum {
-  SYNC_STATUS_BROADCAST,
-  SYNC_STATUS_BROADCAST_RSP,
-  SYNC_STATUS_SETUP_CONN,
-  SYNC_STATUS_SETUP_CONN_RSP,
-  SYNC_STATUS_EXCHANGE_DATA,
-  SYNC_STATUS_EXCHANGE_DATA_RSP,
-  SYNC_STATUS_CHECK_ROLE,
-  SYNC_STATUS_CHECK_ROLE_RSP
-} ESyncStatusType;
-
-#pragma pack(push, 1)
-
-typedef struct {
-  int8_t   type;       // msg type
-  int8_t   protocol;   // protocol version
-  uint16_t signature;  // fixed value
-  int32_t  code;       //
-  int32_t  cId;        // cluster Id
-  int32_t  vgId;       // vg ID
-  int32_t  len;        // content length, does not include head
-  uint32_t cksum;
-} SSyncHead;
-
-typedef struct {
-  SSyncHead head;
-  uint16_t  port;
-  uint16_t  tranId;
-  int32_t   sourceId;  // only for arbitrator
-  char      fqdn[TSDB_FQDN_LEN];
-} SSyncMsg;
-
-typedef struct {
-  SSyncHead head;
-  int8_t    sync;
-  int8_t    reserved;
-  uint16_t  tranId;
-  int8_t    reserverd[4];
-} SSyncRsp;
-
-typedef struct {
-  int8_t    role;
-  uint64_t  version;
-} SPeerStatus;
-
-typedef struct {
-  SSyncHead   head;
-  int8_t      role;
-  int8_t      ack;
-  int8_t      type;
-  int8_t      reserved[3];
-  uint16_t    tranId;
-  uint64_t    version;
-  SPeerStatus peersStatus[TAOS_SYNC_MAX_REPLICA];
-} SPeersStatus;
-
-typedef struct {
-  SSyncHead head;
-  uint64_t  fversion;
-} SFileVersion;
-
-typedef struct {
-  SSyncHead head;
-  int8_t    ack;
-} SFileAck;
-
-typedef struct {
-  SSyncHead head;
-  uint64_t  version;
-  int32_t   code;
-} SFwdRsp;
-
-#pragma pack(pop)
-
-#define SYNC_PROTOCOL_VERSION 1
-#define SYNC_SIGNATURE ((uint16_t)(0xCDEF))
-
-extern char *statusType[];
-
-uint16_t syncGenTranId();
-int32_t  syncCheckHead(SSyncHead *pHead);
-
-void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len);
-void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code);
-void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId);
-void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId);
-void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId);
-void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId);
-void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId);
-
-void syncBuildFileAck(SFileAck *pMsg, int32_t vgId);
-void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_VNODEPEER_H

tools/shell/inc/tsync.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_SYNC_H
-#define TDENGINE_SYNC_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define TAOS_SYNC_MAX_REPLICA 5
-#define TAOS_SYNC_MAX_INDEX   0x7FFFFFFF
-
-typedef enum {
-  TAOS_SYNC_ROLE_OFFLINE  = 0,
-  TAOS_SYNC_ROLE_UNSYNCED = 1,
-  TAOS_SYNC_ROLE_SYNCING  = 2,
-  TAOS_SYNC_ROLE_SLAVE    = 3,
-  TAOS_SYNC_ROLE_MASTER   = 4
-} ESyncRole;
-
-typedef enum {
-  TAOS_SYNC_STATUS_INIT  = 0,
-  TAOS_SYNC_STATUS_START = 1,
-  TAOS_SYNC_STATUS_FILE  = 2,
-  TAOS_SYNC_STATUS_CACHE = 3
-} ESyncStatus;
-
-typedef struct {
-  uint32_t  nodeId;    // node ID assigned by TDengine
-  uint16_t  nodePort;  // node sync Port
-  char      nodeFqdn[TSDB_FQDN_LEN]; // node FQDN  
-} SNodeInfo;
-
-typedef struct {
-  int8_t     quorum;    // number of confirms required, >=1 
-  int8_t     replica;   // number of replications, >=1
-  SNodeInfo  nodeInfo[TAOS_SYNC_MAX_REPLICA];
-} SSyncCfg;
-
-typedef struct {
-  int32_t  selfIndex;
-  uint32_t nodeId[TAOS_SYNC_MAX_REPLICA];
-  int32_t  role[TAOS_SYNC_MAX_REPLICA];
-} SNodesRole;
-
-// get the wal file from index or after
-// return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
-typedef int32_t  (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); 
- 
-// when a forward pkt is received, call this to handle data
-typedef int32_t  (*FWriteToCache)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg);
-
-// when forward is confirmed by peer, master call this API to notify app
-typedef void     (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code);
-
-// when role is changed, call this to notify app
-typedef void     (*FNotifyRole)(int32_t vgId, int8_t role);
-
-// if a number of retrieving data failed, call this to start flow control 
-typedef void     (*FNotifyFlowCtrl)(int32_t vgId, int32_t level);
-
-// when data file is synced successfully, notity app
-typedef void     (*FStartSyncFile)(int32_t vgId);
-typedef void     (*FStopSyncFile)(int32_t vgId, uint64_t fversion);
-
-// get file version
-typedef int32_t  (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver);
-
-typedef int32_t  (*FSendFile)(void *tsdb, SOCKET socketFd);
-typedef int32_t  (*FRecvFile)(void *tsdb, SOCKET socketFd);
-
-typedef struct {
-  int32_t  vgId;       // vgroup ID
-  uint64_t version;    // initial version
-  SSyncCfg syncCfg;    // configuration from mgmt
-  char     path[TSDB_FILENAME_LEN];  // path to the file
-  void *   pTsdb;
-  FGetWalInfo       getWalInfoFp;
-  FWriteToCache     writeToCacheFp;
-  FConfirmForward   confirmForward;
-  FNotifyRole       notifyRoleFp;
-  FNotifyFlowCtrl   notifyFlowCtrlFp;
-  FStartSyncFile    startSyncFileFp;
-  FStopSyncFile     stopSyncFileFp;
-  FGetVersion       getVersionFp;
-  FSendFile         sendFileFp;
-  FRecvFile         recvFileFp;
-} SSyncInfo;
-
-typedef void *tsync_h;
-
-int32_t syncInit();
-void    syncCleanUp();
-
-int64_t syncStart(const SSyncInfo *);
-void    syncStop(int64_t rid);
-int32_t syncReconfig(int64_t rid, const SSyncCfg *);
-int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype, bool force);
-void    syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force);
-void    syncRecover(int64_t rid);  // recover from other nodes:
-int32_t syncGetNodesRole(int64_t rid, SNodesRole *);
-
-extern char *syncRole[];
-
-//global configurable parameters
-extern int32_t  sDebugFlag;
-extern char     tsArbitrator[];
-extern uint16_t tsSyncPort;
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  // TDENGINE_SYNC_H

tools/shell/src/backup/shellCheck.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _GNU_SOURCE
-#define _XOPEN_SOURCE
-#define _DEFAULT_SOURCE
-
-#include "os.h"
-#include "shell.h"
-#include "shellCommand.h"
-#include "tglobal.h"
-#include "tutil.h"
-
-#define SHELL_SQL_LEN 1024
-static int32_t tbNum = 0;
-static int32_t tbMallocNum = 0;
-static char ** tbNames = NULL;
-static int32_t checkedNum = 0;
-static int32_t errorNum = 0;
-
-typedef struct {
-  TdThread threadID;
-  int       threadIndex;
-  int       totalThreads;
-  void *    taos;
-  char *    db;
-} ShellThreadObj;
-
-static int32_t shellUseDb(TAOS *con, char *db) {
-  if (db == NULL) {
-    fprintf(stdout, "no dbname input\n");
-    return -1;
-  }
-
-  char sql[SHELL_SQL_LEN] = {0};
-  snprintf(sql, SHELL_SQL_LEN, "use %s", db);
-
-  TAOS_RES *pSql = taos_query(con, sql);
-  int32_t   code = taos_errno(pSql);
-  if (code != 0) {
-    fprintf(stdout, "failed to execute sql:%s since %s", sql, taos_errstr(pSql));
-  }
-
-  taos_free_result(pSql);
-  return code;
-}
-
-static int32_t shellShowTables(TAOS *con, char *db) {
-  char sql[SHELL_SQL_LEN] = {0};
-  snprintf(sql, SHELL_SQL_LEN, "show %s.tables", db);
-
-  TAOS_RES *pSql = taos_query(con, sql);
-  int32_t   code = taos_errno(pSql);
-
-  if (code != 0) {
-    fprintf(stdout, "failed to execute sql:%s since %s\n", sql, taos_errstr(pSql));
-  } else {
-    TAOS_ROW row;
-    while ((row = taos_fetch_row(pSql))) {
-      int32_t tbIndex = tbNum++;
-      if (tbMallocNum < tbNum) {
-        tbMallocNum = (tbMallocNum * 2 + 1);
-        char** tbNames1 = taosMemoryRealloc(tbNames, tbMallocNum * sizeof(char *));
-        if (tbNames1 == NULL) {
-          fprintf(stdout, "failed to malloc tablenames, num:%d\n", tbMallocNum);
-          code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-          break;
-        }
-        tbNames = tbNames1;
-      }
-
-      tbNames[tbIndex] = taosMemoryMalloc(TSDB_TABLE_NAME_LEN);
-      strncpy(tbNames[tbIndex], (const char *)row[0], TSDB_TABLE_NAME_LEN);
-      if (tbIndex % 100000 == 0 && tbIndex != 0) {
-        fprintf(stdout, "%d tablenames fetched\n", tbIndex);
-      }
-    }
-  }
-
-  taos_free_result(pSql);
-
-  fprintf(stdout, "total %d tablenames fetched, over\n", tbNum);
-  return code;
-}
-
-static void shellFreeTbnames() {
-  for (int32_t i = 0; i < tbNum; ++i) {
-    taosMemoryFree(tbNames[i]);
-  }
-  taosMemoryFree(tbNames);
-}
-
-static void *shellCheckThreadFp(void *arg) {
-  ShellThreadObj *pThread = (ShellThreadObj *)arg;
-
-  setThreadName("shellCheckThrd");
-
-  int32_t interval = tbNum / pThread->totalThreads + 1;
-  int32_t start = pThread->threadIndex * interval;
-  int32_t end = (pThread->threadIndex + 1) * interval;
-
-  if (end > tbNum) end = tbNum + 1;
-
-  char file[32] = {0};
-  snprintf(file, 32, "tb%d.txt", pThread->threadIndex);
-
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (!fp) {
-    fprintf(stdout, "failed to open %s, reason:%s", file, strerror(errno));
-    return NULL;
-  }
-
-  char sql[SHELL_SQL_LEN];
-  for (int32_t t = start; t < end; ++t) {
-    char *tbname = tbNames[t];
-    if (tbname == NULL) break;
-
-    snprintf(sql, SHELL_SQL_LEN, "select * from %s limit 1", tbname);
-
-    TAOS_RES *pSql = taos_query(pThread->taos, sql);
-    int32_t   code = taos_errno(pSql);
-    if (code != 0) {
-      int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname);
-      taosWriteFile(pFile, sql, len);
-      atomic_add_fetch_32(&errorNum, 1);
-    }
-
-    int32_t cnum = atomic_add_fetch_32(&checkedNum, 1);
-    if (cnum % 5000 == 0 && cnum != 0) {
-      fprintf(stdout, "%d tables checked\n", cnum);
-    }
-
-    taos_free_result(pSql);
-  }
-
-  taosFsync(pFile);
-  taosCloseFile(&pFile);
-
-  return NULL;
-}
-
-static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) {
-  TdThreadAttr  thattr;
-  ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj));
-  for (int t = 0; t < _args->threadNum; ++t) {
-    ShellThreadObj *pThread = threadObj + t;
-    pThread->threadIndex = t;
-    pThread->totalThreads = _args->threadNum;
-    pThread->taos = con;
-    pThread->db = _args->database;
-
-    taosThreadAttrInit(&thattr);
-    taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);
-
-    if (taosThreadCreate(&(pThread->threadID), &thattr, shellCheckThreadFp, (void *)pThread) != 0) {
-      fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex);
-      exit(0);
-    }
-  }
-
-  for (int t = 0; t < _args->threadNum; ++t) {
-    taosThreadJoin(threadObj[t].threadID, NULL);
-  }
-
-  for (int t = 0; t < _args->threadNum; ++t) {
-    taos_close(threadObj[t].taos);
-  }
-  taosMemoryFree(threadObj);
-}
-
-void shellCheck(TAOS *con, SShellArguments *_args) {
-  int64_t start = taosGetTimestampMs();
-
-  if (shellUseDb(con, _args->database) != 0) {
-    shellFreeTbnames();
-    return;
-  }
-
-  if (shellShowTables(con, _args->database) != 0) {
-    shellFreeTbnames();
-    return;
-  }
-
-  fprintf(stdout, "total %d tables will be checked by %d threads\n", tbNum, _args->threadNum);
-  shellRunCheckThreads(con, _args);
-
-  int64_t end = taosGetTimestampMs();
-  fprintf(stdout, "total %d tables checked, failed:%d, time spent %.2f seconds\n", checkedNum, errorNum,
-          (end - start) / 1000.0);
-}

tools/shell/src/backup/shellDarwin.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define __USE_XOPEN
-
-#include "os.h"
-
-#include "shell.h"
-#include "shellCommand.h"
-#include "tbase64.h"
-
-#include "tscLog.h"
-
-#define OPT_ABORT 1 /* �Cabort */
-
-int indicator = 1;
-struct termios oldtio;
-
-void insertChar(Command *cmd, char *c, int size);
-
-
-void printHelp() {
-  char indent[10] = "        ";
-  printf("taos shell is used to test the TDengine database\n");
-
-  printf("%s%s\n", indent, "-h");
-  printf("%s%s%s\n", indent, indent, "TDengine server IP address to connect. The default host is localhost.");
-  printf("%s%s\n", indent, "-p");
-  printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server.");
-  printf("%s%s\n", indent, "-P");
-  printf("%s%s%s\n", indent, indent, "The TCP/IP port number to use for the connection");
-  printf("%s%s\n", indent, "-u");
-  printf("%s%s%s\n", indent, indent, "The user name to use when connecting to the server.");
-  printf("%s%s\n", indent, "-c");
-  printf("%s%s%s\n", indent, indent, "Configuration directory.");
-  printf("%s%s\n", indent, "-s");
-  printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell.");
-  printf("%s%s\n", indent, "-r");
-  printf("%s%s%s\n", indent, indent, "Output time as unsigned long..");
-  printf("%s%s\n", indent, "-f");
-  printf("%s%s%s\n", indent, indent, "Script to run without enter the shell.");
-  printf("%s%s\n", indent, "-d");
-  printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server.");
-  printf("%s%s\n", indent, "-t");
-  printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local.");
-  printf("%s%s\n", indent, "-D");
-  printf("%s%s%s\n", indent, indent, "Use multi-thread to import all SQL files in the directory separately.");
-  printf("%s%s\n", indent, "-T");
-  printf("%s%s%s\n", indent, indent, "Number of threads when using multi-thread to import data.");
-
-  exit(EXIT_SUCCESS);
-}
-
-char      DARWINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n"
-                             "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
-char g_password[SHELL_MAX_PASSWORD_LEN];
-
-void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
-  wordexp_t full_path;
-  for (int i = 1; i < argc; i++) {
-    // for host
-    if (strcmp(argv[i], "-h") == 0) {
-      if (i < argc - 1) {
-        arguments->host = argv[++i];
-      } else {
-        fprintf(stderr, "option -h requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-      // for password
-    else if ((strncmp(argv[i], "-p", 2) == 0)
-            || (strncmp(argv[i], "--password", 10) == 0)) {
-        strcpy(tsOsName, "Darwin");
-        printf(DARWINCLIENT_VERSION, tsOsName, taos_get_client_info());
-        if ((strlen(argv[i]) == 2)
-                  || (strncmp(argv[i], "--password", 10) == 0)) {
-            printf("Enter password: ");
-            taosSetConsoleEcho(false);
-            if (scanf("%s", g_password) > 1) {
-                fprintf(stderr, "password read error\n");
-            }
-            taosSetConsoleEcho(true);
-            getchar();
-        } else {
-            tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN);
-        }
-        arguments->password = g_password;
-        arguments->is_use_passwd = true;
-        strcpy(argv[i], "");
-        argc -= 1;
-    }
-    // for management port
-    else if (strcmp(argv[i], "-P") == 0) {
-      if (i < argc - 1) {
-        arguments->port = atoi(argv[++i]);
-      } else {
-        fprintf(stderr, "option -P requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-      // for user
-    else if (strcmp(argv[i], "-u") == 0) {
-      if (i < argc - 1) {
-        arguments->user = argv[++i];
-      } else {
-        fprintf(stderr, "option -u requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    } else if (strcmp(argv[i], "-c") == 0) {
-      if (i < argc - 1) {
-        if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) {
-          fprintf(stderr, "config file path: %s overflow max len %d\n", argv[i], TSDB_FILENAME_LEN - 1);
-          exit(EXIT_FAILURE);
-        }
-        strcpy(configDir, argv[i]);
-      } else {
-        fprintf(stderr, "Option -c requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    } else if (strcmp(argv[i], "-s") == 0) {
-      if (i < argc - 1) {
-        arguments->commands = argv[++i];
-      } else {
-        fprintf(stderr, "option -s requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    } else if (strcmp(argv[i], "-r") == 0) {
-      arguments->is_raw_time = true;
-    }
-      // For temperory batch commands to run TODO
-    else if (strcmp(argv[i], "-f") == 0) {
-      if (i < argc - 1) {
-        strcpy(arguments->file, argv[++i]);
-      } else {
-        fprintf(stderr, "option -f requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-      // for default database
-    else if (strcmp(argv[i], "-d") == 0) {
-      if (i < argc - 1) {
-        arguments->database = argv[++i];
-      } else {
-        fprintf(stderr, "option -d requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-      // For time zone
-    else if (strcmp(argv[i], "-t") == 0) {
-      if (i < argc - 1) {
-        arguments->timezone = argv[++i];
-      } else {
-        fprintf(stderr, "option -t requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-      // For import directory
-    else if (strcmp(argv[i], "-D") == 0) {
-      if (i < argc - 1) {
-        if (wordexp(argv[++i], &full_path, 0) != 0) {
-          fprintf(stderr, "Invalid path %s\n", argv[i]);
-          exit(EXIT_FAILURE);
-        }
-        strcpy(arguments->dir, full_path.we_wordv[0]);
-        wordfree(&full_path);
-      } else {
-        fprintf(stderr, "option -D requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-      // For time zone
-    else if (strcmp(argv[i], "-T") == 0) {
-      if (i < argc - 1) {
-        arguments->threadNum = atoi(argv[++i]);
-      } else {
-        fprintf(stderr, "option -T requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-      // For temperory command TODO
-    else if (strcmp(argv[i], "--help") == 0) {
-      printHelp();
-      exit(EXIT_FAILURE);
-    } else {
-      fprintf(stderr, "wrong options\n");
-      printHelp();
-      exit(EXIT_FAILURE);
-    }
-  }
-}
-
-int32_t shellReadCommand(TAOS *con, char *command) {
-  unsigned hist_counter = history.hend;
-  char utf8_array[10] = "\0";
-  Command cmd;
-  memset(&cmd, 0, sizeof(cmd));
-  cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
-  cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
-  showOnScreen(&cmd);
-
-  // Read input.
-  char c;
-  while (1) {
-    c = getchar();
-
-    if (c < 0) {  // For UTF-8
-      int count = countPrefixOnes(c);
-      utf8_array[0] = c;
-      for (int k = 1; k < count; k++) {
-        c = getchar();
-        utf8_array[k] = c;
-      }
-      insertChar(&cmd, utf8_array, count);
-    } else if (c < '\033') {
-      // Ctrl keys.  TODO: Implement ctrl combinations
-      switch (c) {
-        case 1:  // ctrl A
-          positionCursorHome(&cmd);
-          break;
-        case 3:
-          printf("\n");
-          resetCommand(&cmd, "");
-          kill(0, SIGINT);
-          break;
-        case 4:  // EOF or Ctrl+D
-          printf("\n");
-          taos_close(con);
-          // write the history
-          write_history();
-          exitShell();
-          break;
-        case 5:  // ctrl E
-          positionCursorEnd(&cmd);
-          break;
-        case 8:
-          backspaceChar(&cmd);
-          break;
-        case '\n':
-        case '\r':
-          printf("\n");
-          if (isReadyGo(&cmd)) {
-            sprintf(command, "%s%s", cmd.buffer, cmd.command);
-            taosMemoryFreeClear(cmd.buffer);
-            taosMemoryFreeClear(cmd.command);
-            return 0;
-          } else {
-            updateBuffer(&cmd);
-          }
-          break;
-        case 11:  // Ctrl + K;
-          clearLineAfter(&cmd);
-          break;
-        case 12:  // Ctrl + L;
-          system("clear");
-          showOnScreen(&cmd);
-          break;
-        case 21:  // Ctrl + U
-          clearLineBefore(&cmd);
-          break;
-      }
-    } else if (c == '\033') {
-      c = getchar();
-      switch (c) {
-        case '[':
-          c = getchar();
-          switch (c) {
-            case 'A':  // Up arrow
-              if (hist_counter != history.hstart) {
-                hist_counter = (hist_counter + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE;
-                resetCommand(&cmd, (history.hist[hist_counter] == NULL) ? "" : history.hist[hist_counter]);
-              }
-              break;
-            case 'B':  // Down arrow
-              if (hist_counter != history.hend) {
-                int next_hist = (hist_counter + 1) % MAX_HISTORY_SIZE;
-
-                if (next_hist != history.hend) {
-                  resetCommand(&cmd, (history.hist[next_hist] == NULL) ? "" : history.hist[next_hist]);
-                } else {
-                  resetCommand(&cmd, "");
-                }
-                hist_counter = next_hist;
-              }
-              break;
-            case 'C':  // Right arrow
-              moveCursorRight(&cmd);
-              break;
-            case 'D':  // Left arrow
-              moveCursorLeft(&cmd);
-              break;
-            case '1':
-              if ((c = getchar()) == '~') {
-                // Home key
-                positionCursorHome(&cmd);
-              }
-              break;
-            case '2':
-              if ((c = getchar()) == '~') {
-                // Insert key
-              }
-              break;
-            case '3':
-              if ((c = getchar()) == '~') {
-                // Delete key
-                deleteChar(&cmd);
-              }
-              break;
-            case '4':
-              if ((c = getchar()) == '~') {
-                // End key
-                positionCursorEnd(&cmd);
-              }
-              break;
-            case '5':
-              if ((c = getchar()) == '~') {
-                // Page up key
-              }
-              break;
-            case '6':
-              if ((c = getchar()) == '~') {
-                // Page down key
-              }
-              break;
-            case 72:
-              // Home key
-              positionCursorHome(&cmd);
-              break;
-            case 70:
-              // End key
-              positionCursorEnd(&cmd);
-              break;
-          }
-          break;
-      }
-    } else if (c == 0x7f) {
-      // press delete key
-      backspaceChar(&cmd);
-    } else {
-      insertChar(&cmd, &c, 1);
-    }
-  }
-
-  return 0;
-}
-
-void *shellLoopQuery(void *arg) {
-  if (indicator) {
-    getOldTerminalMode();
-    indicator = 0;
-  }
-
-  TAOS *con = (TAOS *)arg;
-
-  setThreadName("shellLoopQuery");
-
-  taosThreadCleanupPush(cleanup_handler, NULL);
-
-    char *command = taosMemoryMalloc(MAX_COMMAND_SIZE);
-    if (command == NULL){
-      tscError("failed to malloc command");
-      return NULL;
-    }
-
-    int32_t err = 0;
-
-    do {
-      // Read command from shell.
-      memset(command, 0, MAX_COMMAND_SIZE);
-      setTerminalMode();
-      err = shellReadCommand(con, command);
-      if (err) {
-        break;
-      }
-      resetTerminalMode();
-    } while (shellRunCommand(con, command) == 0);
-
-  taosMemoryFreeClear(command);
-  exitShell();
-
-  taosThreadCleanupPop(1);
-
-  return NULL;
-}
-
-void get_history_path(char *history) { sprintf(history, "%s/%s", getpwuid(getuid())->pw_dir, HISTORY_FILE); }
-
-void clearScreen(int ecmd_pos, int cursor_pos) {
-  struct winsize w;
-  if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
-    //fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n");
-    w.ws_col = 120;
-    w.ws_row = 30;
-  }
-
-  int cursor_x = cursor_pos / w.ws_col;
-  int cursor_y = cursor_pos % w.ws_col;
-  int command_x = ecmd_pos / w.ws_col;
-  positionCursor(cursor_y, LEFT);
-  positionCursor(command_x - cursor_x, DOWN);
-  fprintf(stdout, "\033[2K");
-  for (int i = 0; i < command_x; i++) {
-    positionCursor(1, UP);
-    fprintf(stdout, "\033[2K");
-  }
-  fflush(stdout);
-}
-
-void showOnScreen(Command *cmd) {
-  struct winsize w;
-  if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) {
-    //fprintf(stderr, "No stream device\n");
-    w.ws_col = 120;
-    w.ws_row = 30;
-  }
-
-  TdWchar wc;
-  int size = 0;
-
-  // Print out the command.
-  char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE);
-  memset(total_string, '\0', MAX_COMMAND_SIZE);
-  if (strcmp(cmd->buffer, "") == 0) {
-    sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command);
-  } else {
-    sprintf(total_string, "%s%s", CONTINUE_PROMPT, cmd->command);
-  }
-
-  int remain_column = w.ws_col;
-  /* size = cmd->commandSize + prompt_size; */
-  for (char *str = total_string; size < cmd->commandSize + prompt_size;) {
-    int ret = taosMbToWchar(&wc, str, MB_CUR_MAX);
-    if (ret < 0) break;
-    size += ret;
-    /* assert(size >= 0); */
-    int width = taosWcharWidth(wc);
-    if (remain_column > width) {
-      printf("%lc", wc);
-      remain_column -= width;
-    } else {
-      if (remain_column == width) {
-        printf("%lc\n\r", wc);
-        remain_column = w.ws_col;
-      } else {
-        printf("\n\r%lc", wc);
-        remain_column = w.ws_col - width;
-      }
-    }
-
-    str = total_string + size;
-  }
-
-  taosMemoryFree(total_string);
-  /* for (int i = 0; i < size; i++){ */
-  /*     char c = total_string[i]; */
-  /*     if (k % w.ws_col == 0) { */
-  /*         printf("%c\n\r", c); */
-  /*     } */
-  /*     else { */
-  /*         printf("%c", c); */
-  /*     } */
-  /*     k += 1; */
-  /* } */
-
-  // Position the cursor
-  int cursor_pos = cmd->screenOffset + prompt_size;
-  int ecmd_pos = cmd->endOffset + prompt_size;
-
-  int cursor_x = cursor_pos / w.ws_col;
-  int cursor_y = cursor_pos % w.ws_col;
-  // int cursor_y = cursor % w.ws_col;
-  int command_x = ecmd_pos / w.ws_col;
-  int command_y = ecmd_pos % w.ws_col;
-  // int command_y = (command.size() + prompt_size) % w.ws_col;
-  positionCursor(command_y, LEFT);
-  positionCursor(command_x, UP);
-  positionCursor(cursor_x, DOWN);
-  positionCursor(cursor_y, RIGHT);
-  fflush(stdout);
-}
-
-void cleanup_handler(void *arg) { resetTerminalMode(); }
-
-void exitShell() {
-  resetTerminalMode();
-  exit(EXIT_SUCCESS);
-}

tools/shell/src/backup/shellImport.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _GNU_SOURCE
-#define _XOPEN_SOURCE
-#define _DEFAULT_SOURCE
-
-#include "os.h"
-#include "shell.h"
-#include "shellCommand.h"
-#include "tglobal.h"
-#include "tutil.h"
-
-static char **shellSQLFiles = NULL;
-static int32_t shellSQLFileNum = 0;
-static char shellTablesSQLFile[TSDB_FILENAME_LEN] = {0};
-
-typedef struct {
-  TdThread threadID;
-  int       threadIndex;
-  int       totalThreads;
-  void     *taos;
-} ShellThreadObj;
-
-static int shellGetFilesNum(const char *directoryName, const char *prefix)
-{
-  char cmd[1024] = { 0 };
-  sprintf(cmd, "ls %s/*.%s | wc -l ", directoryName, prefix);
-
-  char buf[1024] = { 0 };
-  if (taosSystem(cmd, buf, sizeof(buf)) < 0) {
-    fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
-    exit(0);
-  }
-
-  int fileNum = 0;
-  if (sscanf(buf, "%d", &fileNum) != 1) {
-    fprintf(stderr, "ERROR: failed to execute:%s, parse result error\n", cmd);
-    exit(0);
-  }
-
-  if (fileNum <= 0) {
-    fprintf(stderr, "ERROR: directory:%s is empry\n", directoryName);
-    exit(0);
-  }
-
-  return fileNum;
-}
-
-static void shellParseDirectory(const char *directoryName, const char *prefix, char **fileArray, int totalFiles)
-{
-  char cmd[1024] = { 0 };
-  sprintf(cmd, "ls %s/*.%s | sort", directoryName, prefix);
-
-  char buf[1024] = { 0 };
-  if (taosSystem(cmd, buf, sizeof(buf)) < 0) {
-    fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
-    exit(0);
-  }
-
-  int fileNum = 0;
-  while (sscanf(buf, "%128s", fileArray[fileNum++])) {
-    if (strcmp(fileArray[fileNum-1], shellTablesSQLFile) == 0) {
-      fileNum--;
-    }
-    if (fileNum >= totalFiles) {
-      break;
-    }
-  }
-
-  if (fileNum != totalFiles) {
-    fprintf(stderr, "ERROR: directory:%s changed while read\n", directoryName);
-    exit(0);
-  }
-}
-
-static void shellCheckTablesSQLFile(const char *directoryName)
-{
-  sprintf(shellTablesSQLFile, "%s/tables.sql", directoryName);
-
-  if (taosFStatFile(shellTablesSQLFile, NULL, NULL) < 0) {
-    shellTablesSQLFile[0] = 0;
-  }
-}
-
-static void shellMallocSQLFiles()
-{
-  shellSQLFiles = (char**)taosMemoryCalloc(shellSQLFileNum, sizeof(char*));
-  for (int i = 0; i < shellSQLFileNum; i++) {
-    shellSQLFiles[i] = taosMemoryCalloc(1, TSDB_FILENAME_LEN);
-  }
-}
-
-static void shellGetDirectoryFileList(char *inputDir)
-{
-  if (!taosDirExist(inputDir)) {
-    fprintf(stderr, "ERROR: %s not exist\n", inputDir);
-    exit(0);
-  }
-
-  if (taosIsDir(inputDir)) {
-    shellCheckTablesSQLFile(inputDir);
-    shellSQLFileNum = shellGetFilesNum(inputDir, "sql");
-    int totalSQLFileNum = shellSQLFileNum;
-    if (shellTablesSQLFile[0] != 0) {
-      shellSQLFileNum--;
-    }
-    shellMallocSQLFiles();
-    shellParseDirectory(inputDir, "sql", shellSQLFiles, shellSQLFileNum);
-    fprintf(stdout, "\nstart to dispose %d files in %s\n", totalSQLFileNum, inputDir);
-  }
-  else {
-    fprintf(stderr, "ERROR: %s is not a directory\n", inputDir);
-    exit(0);
-  }
-}
-
-static void shellSourceFile(TAOS *con, char *fptr) {
-  wordexp_t full_path;
-  int       read_len = 0;
-  char *    cmd = taosMemoryMalloc(tsMaxSQLStringLen);
-  size_t    cmd_len = 0;
-  char *    line = NULL;
-
-  if (wordexp(fptr, &full_path, 0) != 0) {
-    fprintf(stderr, "ERROR: illegal file name\n");
-    taosMemoryFree(cmd);
-    return;
-  }
-
-  char *fname = full_path.we_wordv[0];
-  if (fname == NULL) {
-    fprintf(stderr, "ERROR: invalid filename\n");
-    taosMemoryFree(cmd);
-    return;
-  }
-
-  /*
-  if (access(fname, F_OK) != 0) {
-    fprintf(stderr, "ERROR: file %s is not exist\n", fptr);
-    
-    wordfree(&full_path);
-    taosMemoryFree(cmd);
-    return;
-  }
-  
-  if (access(fname, R_OK) != 0) {
-    fprintf(stderr, "ERROR: file %s is not readable\n", fptr);
-    
-    wordfree(&full_path);
-    taosMemoryFree(cmd);
-    return;
-  }
-  */
-
-  // FILE *f = fopen(fname, "r");
-  TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ | TD_FILE_STREAM);
-  if (pFile == NULL) {
-    fprintf(stderr, "ERROR: failed to open file %s\n", fname);
-    wordfree(&full_path);
-    taosMemoryFree(cmd);
-    return;
-  }
-
-  fprintf(stdout, "begin import file:%s\n", fname);
-
-  int lineNo = 0;
-  while ((read_len = taosGetLineFile(pFile, &line)) != -1) {
-    ++lineNo;
-    if (read_len >= tsMaxSQLStringLen) continue;
-    line[--read_len] = '\0';
-
-    if (read_len == 0 || isCommentLine(line)) {  // line starts with #
-      continue;
-    }
-
-    if (line[read_len - 1] == '\\') {
-      line[read_len - 1] = ' ';
-      memcpy(cmd + cmd_len, line, read_len);
-      cmd_len += read_len;
-      continue;
-    }
-
-    memcpy(cmd + cmd_len, line, read_len);
-    
-    TAOS_RES* pSql = taos_query(con, cmd);
-    int32_t code = taos_errno(pSql);
-    
-    if (code != 0) {
-      fprintf(stderr, "DB error: %s: %s (%d)\n", taos_errstr(pSql), fname, lineNo);
-    }
-    
-    /* free local resouce: allocated memory/metric-meta refcnt */
-    taos_free_result(pSql);
-
-    memset(cmd, 0, MAX_COMMAND_SIZE);
-    cmd_len = 0;
-  }
-
-  taosMemoryFree(cmd);
-  if(line != NULL) taosMemoryFree(line);
-  wordfree(&full_path);
-  taosCloseFile(&pFile);
-}
-
-void* shellImportThreadFp(void *arg)
-{
-  ShellThreadObj *pThread = (ShellThreadObj*)arg;
-  setThreadName("shellImportThrd");
-
-  for (int f = 0; f < shellSQLFileNum; ++f) {
-    if (f % pThread->totalThreads == pThread->threadIndex) {
-      char *SQLFileName = shellSQLFiles[f];
-      shellSourceFile(pThread->taos, SQLFileName);
-    }
-  }
-
-  return NULL;
-}
-
-static void shellRunImportThreads(SShellArguments* _args)
-{
-  TdThreadAttr thattr;
-  ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj));
-  for (int t = 0; t < _args->threadNum; ++t) {
-    ShellThreadObj *pThread = threadObj + t;
-    pThread->threadIndex = t;
-    pThread->totalThreads = _args->threadNum;
-    pThread->taos = taos_connect(_args->host, _args->user, _args->password, _args->database, tsDnodeShellPort);
-    if (pThread->taos == NULL) {
-      fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, "null taos"/*taos_errstr(pThread->taos)*/);
-      exit(0);
-    }
-
-    taosThreadAttrInit(&thattr);
-    taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);
-
-    if (taosThreadCreate(&(pThread->threadID), &thattr, shellImportThreadFp, (void*)pThread) != 0) {
-      fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex);
-      exit(0);
-    }
-  }
-
-  for (int t = 0; t < _args->threadNum; ++t) {
-    taosThreadJoin(threadObj[t].threadID, NULL);
-  }
-
-  for (int t = 0; t < _args->threadNum; ++t) {
-    taos_close(threadObj[t].taos);
-  }
-  taosMemoryFree(threadObj);
-}
-
-void source_dir(TAOS* con, SShellArguments* _args) {
-  shellGetDirectoryFileList(_args->dir);
-  int64_t start = taosGetTimestampMs();
-
-  if (shellTablesSQLFile[0] != 0) {
-    shellSourceFile(con, shellTablesSQLFile);
-    int64_t end = taosGetTimestampMs();
-    fprintf(stdout, "import %s finished, time spent %.2f seconds\n", shellTablesSQLFile, (end - start) / 1000.0);
-  }
-
-  shellRunImportThreads(_args);
-  int64_t end = taosGetTimestampMs();
-  fprintf(stdout, "import %s finished, time spent %.2f seconds\n", _args->dir, (end - start) / 1000.0);
-}

tools/shell/src/backup/shellWindows.c

-/*******************************************************************
-*           Copyright (c) 2017 by TAOS Technologies, Inc.
-*                     All rights reserved.
-*
-*  This file is proprietary and confidential to TAOS Technologies.
-*  No part of this file may be reproduced, stored, transmitted,
-*  disclosed or used in any form or by any means other than as
-*  expressly provided by the written permission from Jianhui Tao
-*
-* ****************************************************************/
-
-#include <assert.h>
-#include <regex.h>
-#include <stdio.h>
-#include "../../../../include/client/taos.h"
-#include "os.h"
-#include "shell.h"
-#include "shellCommand.h"
-
-extern char configDir[];
-
-char      WINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n"
-                             "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
-
-void printVersion() {
-  printf("version: %s\n", version);
-}
-
-void printHelp() {
-  char indent[10] = "        ";
-  printf("taos shell is used to test the TDengine database\n");
-
-  printf("%s%s\n", indent, "-h");
-  printf("%s%s%s\n", indent, indent, "TDengine server FQDN to connect. The default host is localhost.");
-  printf("%s%s\n", indent, "-p");
-  printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server.");
-  printf("%s%s\n", indent, "-P");
-  printf("%s%s%s\n", indent, indent, "The TCP/IP port number to use for the connection");
-  printf("%s%s\n", indent, "-u");
-  printf("%s%s%s\n", indent, indent, "The user name to use when connecting to the server.");
-  printf("%s%s\n", indent, "-A");
-  printf("%s%s%s\n", indent, indent, "The user auth to use when connecting to the server.");
-  printf("%s%s\n", indent, "-c");
-  printf("%s%s%s\n", indent, indent, "Configuration directory.");
-  printf("%s%s\n", indent, "-C");
-  printf("%s%s%s\n", indent, indent, "Dump configuration.");
-  printf("%s%s\n", indent, "-s");
-  printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell.");
-  printf("%s%s\n", indent, "-r");
-  printf("%s%s%s\n", indent, indent, "Output time as unsigned long..");
-  printf("%s%s\n", indent, "-f");
-  printf("%s%s%s\n", indent, indent, "Script to run without enter the shell.");
-  printf("%s%s\n", indent, "-d");
-  printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server.");
-  printf("%s%s\n", indent, "-t");
-  printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local.");
-  printf("%s%s\n", indent, "-n");
-  printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speed|fqdn.");
-  printf("%s%s\n", indent, "-l");
-  printf("%s%s%s\n", indent, indent, "Packet length used for net test, default is 1000 bytes.");
-  printf("%s%s\n", indent, "-N");
-  printf("%s%s%s\n", indent, indent, "Packet numbers used for net test, default is 100.");
-  printf("%s%s\n", indent, "-S");
-  printf("%s%s%s\n", indent, indent, "Packet type used for net test, default is TCP.");
-  printf("%s%s\n", indent, "-V");
-  printf("%s%s%s\n", indent, indent, "Print program version.");
-
-  exit(EXIT_SUCCESS);
-}
-
-char g_password[SHELL_MAX_PASSWORD_LEN];
-
-void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) {
-  for (int i = 1; i < argc; i++) {
-    // for host
-    if (strcmp(argv[i], "-h") == 0) {
-      if (i < argc - 1) {
-        arguments->host = argv[++i];
-      } else {
-        fprintf(stderr, "option -h requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-    // for password
-    else if ((strncmp(argv[i], "-p", 2) == 0)
-            || (strncmp(argv[i], "--password", 10) == 0)) {
-        arguments->is_use_passwd = true;
-        strcpy(tsOsName, "Windows");
-        printf(WINCLIENT_VERSION, tsOsName, taos_get_client_info());
-        if ((strlen(argv[i]) == 2)
-                  || (strncmp(argv[i], "--password", 10) == 0)) {
-            printf("Enter password: ");
-            taosSetConsoleEcho(false);
-            if (scanf("%s", g_password) > 1) {
-                fprintf(stderr, "password read error!\n");
-            }
-            taosSetConsoleEcho(true);
-            getchar();
-        } else {
-            tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN);
-        }
-        arguments->password = g_password;
-        strcpy(argv[i], "");
-        argc -= 1;
-    }
-    // for management port
-    else if (strcmp(argv[i], "-P") == 0) {
-      if (i < argc - 1) {
-        arguments->port = atoi(argv[++i]);
-      } else {
-        fprintf(stderr, "option -P requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-    // for user
-    else if (strcmp(argv[i], "-u") == 0) {
-      if (i < argc - 1) {
-        arguments->user = argv[++i];
-      } else {
-        fprintf(stderr, "option -u requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    } else if (strcmp(argv[i], "-A") == 0) {
-      if (i < argc - 1) {
-        arguments->auth = argv[++i];
-      } else {
-        fprintf(stderr, "option -A requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    } else if (strcmp(argv[i], "-c") == 0) {
-      if (i < argc - 1) {
-        char *tmp = argv[++i];
-        if (strlen(tmp) >= TSDB_FILENAME_LEN) {
-          fprintf(stderr, "config file path: %s overflow max len %d\n", tmp, TSDB_FILENAME_LEN - 1);
-          exit(EXIT_FAILURE);
-        }
-        strcpy(configDir, tmp);
-      } else {
-        fprintf(stderr, "Option -c requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    } else if (strcmp(argv[i], "-C") == 0) {
-      arguments->dump_config = true;
-    } else if (strcmp(argv[i], "-s") == 0) {
-      if (i < argc - 1) {
-        arguments->commands = argv[++i];
-      } else {
-        fprintf(stderr, "option -s requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    } else if (strcmp(argv[i], "-r") == 0) {
-      arguments->is_raw_time = true;
-    }
-    // For temperory batch commands to run TODO
-    else if (strcmp(argv[i], "-f") == 0) {
-      if (i < argc - 1) {
-        strcpy(arguments->file, argv[++i]);
-      } else {
-        fprintf(stderr, "option -f requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-    // for default database
-    else if (strcmp(argv[i], "-d") == 0) {
-      if (i < argc - 1) {
-        arguments->database = argv[++i];
-      } else {
-        fprintf(stderr, "option -d requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-    // For time zone
-    else if (strcmp(argv[i], "-t") == 0) {
-      if (i < argc - 1) {
-        arguments->timezone = argv[++i];
-      } else {
-        fprintf(stderr, "option -t requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-    else if (strcmp(argv[i], "-n") == 0) {
-      if (i < argc - 1) {
-        arguments->netTestRole = argv[++i];
-      } else {
-        fprintf(stderr, "option -n requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-    else if (strcmp(argv[i], "-l") == 0) {
-      if (i < argc - 1) {
-        arguments->pktLen = atoi(argv[++i]);
-      } else {
-        fprintf(stderr, "option -l requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-    else if (strcmp(argv[i], "-N") == 0) {
-      if (i < argc - 1) {
-        arguments->pktNum = atoi(argv[++i]);
-      } else {
-        fprintf(stderr, "option -N requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-    else if (strcmp(argv[i], "-S") == 0) {
-      if (i < argc - 1) {
-        arguments->pktType = argv[++i];
-      } else {
-        fprintf(stderr, "option -S requires an argument\n");
-        exit(EXIT_FAILURE);
-      }
-    }
-    else if (strcmp(argv[i], "-V") == 0) {
-      printVersion();
-      exit(EXIT_SUCCESS);
-    }
-    // For temperory command TODO
-    else if (strcmp(argv[i], "--help") == 0) {
-      printHelp();
-      exit(EXIT_SUCCESS);
-    } else {
-      fprintf(stderr, "wrong options\n");
-      printHelp();
-      exit(EXIT_FAILURE);
-    }
-  }
-}
-
-void shellPrintContinuePrompt() { printf("%s", CONTINUE_PROMPT); }
-
-void shellPrintPrompt() { printf("%s", PROMPT_HEADER); }
-
-void updateBuffer(Command *cmd) {
-  if (regex_match(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " ");
-  strcat(cmd->buffer, cmd->command);
-
-  memset(cmd->command, 0, MAX_COMMAND_SIZE);
-  cmd->cursorOffset = 0;
-}
-
-int isReadyGo(Command *cmd) {
-  char *total = taosMemoryMalloc(MAX_COMMAND_SIZE);
-  memset(total, 0, MAX_COMMAND_SIZE);
-  sprintf(total, "%s%s", cmd->buffer, cmd->command);
-
-  char *reg_str =
-      "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^"
-      "\\s*clear\\s*$)";
-  if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) {
-    taosMemoryFree(total);
-    return 1;
-  }
-
-  taosMemoryFree(total);
-  return 0;
-}
-
-void insertChar(Command *cmd, char c) {
-  // TODO: Check if the length enough.
-  if (cmd->cursorOffset >= MAX_COMMAND_SIZE) {
-    fprintf(stdout, "sql is larger than %d bytes", MAX_COMMAND_SIZE);
-    return;
-  }
-
-  cmd->command[cmd->cursorOffset++] = c;
-}
-
-int32_t shellReadCommand(TAOS *con, char command[]) {
-  Command cmd;
-  memset(&cmd, 0, sizeof(cmd));
-  cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
-  cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE);
-
-  // Read input.
-  char c;
-  while (1) {
-    c = getchar();
-
-    switch (c) {
-      case '\n':
-      case '\r':
-        if (isReadyGo(&cmd)) {
-          sprintf(command, "%s%s", cmd.buffer, cmd.command);
-          taosMemoryFree(cmd.buffer);
-          cmd.buffer = NULL;
-          taosMemoryFree(cmd.command);
-          cmd.command = NULL;
-          return 0;
-        } else {
-          shellPrintContinuePrompt();
-          updateBuffer(&cmd);
-        }
-        break;
-      default:
-        insertChar(&cmd, c);
-    }
-  }
-
-  return 0;
-}
-
-void *shellLoopQuery(void *arg) {
-  TAOS *con = (TAOS *)arg;
-  char *command = taosMemoryMalloc(MAX_COMMAND_SIZE);
-  if (command == NULL) return NULL;
-
-  int32_t err = 0;
-
-  do {
-    memset(command, 0, MAX_COMMAND_SIZE);
-    shellPrintPrompt();
-
-    // Read command from shell.
-    err = shellReadCommand(con, command);
-    if (err) {
-      break;
-    }
-  } while (shellRunCommand(con, command) == 0);
-
-  return NULL;
-}
-
-void get_history_path(char *history) { sprintf(history, "C:/TDengine/%s", HISTORY_FILE); }
-
-void exitShell() { exit(EXIT_SUCCESS); }

tools/shell/src/shellArguments.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "shellInt.h"
+
+#define SHELL_HOST     "The auth string to use when connecting to the server."
+#define SHELL_PORT     "The TCP/IP port number to use for the connection."
+#define SHELL_USER     "The user name to use when connecting to the server."
+#define SHELL_PASSWORD "The password to use when connecting to the server."
+#define SHELL_AUTH     "The auth string to use when connecting to the server."
+#define SHELL_GEN_AUTH "Generate auth string from password."
+#define SHELL_CFG_DIR  "Configuration directory."
+#define SHELL_DMP_CFG  "Dump configuration."
+#define SHELL_CMD      "Commands to run without enter the shell."
+#define SHELL_RAW_TIME "Output time as uint64_t."
+#define SHELL_FILE     "Script to run without enter the shell."
+#define SHELL_DB       "Database to use when connecting to the server."
+#define SHELL_CHECK    "Check the service status."
+#define SHELL_STARTUP  "Check the details of the service status."
+#define SHELL_WIDTH    "Set the default binary display width, default is 30."
+#define SHELL_NET_ROLE "Net role when network connectivity test, options: client|server."
+#define SHELL_PKG_LEN  "Packet length used for net test, default is 1024 bytes."
+#define SHELL_PKT_NUM  "Packet numbers used for net test, default is 100."
+#define SHELL_VERSION  "Print program version."
+#define SHELL_EMAIL    "<support@taosdata.com>"
+
+void shellPrintHelp() {
+  char indent[] = "  ";
+  printf("Usage: taos [OPTION...] \n\n");
+  printf("%s%s%s%s\n", indent, "-a,", indent, SHELL_AUTH);
+  printf("%s%s%s%s\n", indent, "-A,", indent, SHELL_GEN_AUTH);
+  printf("%s%s%s%s\n", indent, "-c,", indent, SHELL_CFG_DIR);
+  printf("%s%s%s%s\n", indent, "-C,", indent, SHELL_DMP_CFG);
+  printf("%s%s%s%s\n", indent, "-d,", indent, SHELL_DB);
+  printf("%s%s%s%s\n", indent, "-f,", indent, SHELL_FILE);
+  printf("%s%s%s%s\n", indent, "-h,", indent, SHELL_HOST);
+  printf("%s%s%s%s\n", indent, "-k,", indent, SHELL_CHECK);
+  printf("%s%s%s%s\n", indent, "-l,", indent, SHELL_PKG_LEN);
+  printf("%s%s%s%s\n", indent, "-n,", indent, SHELL_NET_ROLE);
+  printf("%s%s%s%s\n", indent, "-N,", indent, SHELL_PKT_NUM);
+  printf("%s%s%s%s\n", indent, "-p,", indent, SHELL_PASSWORD);
+  printf("%s%s%s%s\n", indent, "-P,", indent, SHELL_PORT);
+  printf("%s%s%s%s\n", indent, "-r,", indent, SHELL_RAW_TIME);
+  printf("%s%s%s%s\n", indent, "-s,", indent, SHELL_CMD);
+  printf("%s%s%s%s\n", indent, "-t,", indent, SHELL_STARTUP);
+  printf("%s%s%s%s\n", indent, "-u,", indent, SHELL_USER);
+  printf("%s%s%s%s\n", indent, "-w,", indent, SHELL_WIDTH);
+  printf("%s%s%s%s\n", indent, "-V,", indent, SHELL_VERSION);
+  printf("\n\nReport bugs to %s.\n", SHELL_EMAIL);
+}
+
+static int32_t shellParseSingleOpt(int32_t key, char *arg) {
+  SShellArgs *pArgs = &shell.args;
+
+  switch (key) {
+    case 'h':
+      pArgs->host = arg;
+      break;
+    case 'P':
+      pArgs->port = atoi(arg);
+      break;
+    case 'u':
+      pArgs->user = arg;
+      break;
+    case 'p':
+      break;
+    case 'a':
+      pArgs->auth = arg;
+      break;
+    case 'A':
+      pArgs->is_gen_auth = true;
+      break;
+    case 'c':
+      pArgs->cfgdir = arg;
+      break;
+    case 'C':
+      pArgs->is_dump_config = true;
+      break;
+    case 's':
+      pArgs->commands = arg;
+      break;
+    case 'r':
+      pArgs->is_raw_time = true;
+      break;
+    case 'f':
+      tstrncpy(pArgs->file, arg, sizeof(pArgs->file));
+      break;
+    case 'd':
+      pArgs->database = arg;
+      break;
+    case 'k':
+      pArgs->is_check = true;
+      break;
+    case 't':
+      pArgs->is_startup = true;
+      break;
+    case 'w':
+      pArgs->displayWidth = atoi(arg);
+      break;
+    case 'n':
+      pArgs->netrole = arg;
+      break;
+    case 'l':
+      pArgs->pktLen = atoi(arg);
+      break;
+    case 'N':
+      pArgs->pktNum = atoi(arg);
+      break;
+    case 'V':
+      pArgs->is_version = true;
+      break;
+    case '?':
+      pArgs->is_help = true;
+      break;
+    case 1:
+      pArgs->abort = 1;
+      break;
+    default:
+      return ARGP_ERR_UNKNOWN;
+  }
+  return 0;
+}
+
+int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) {
+  SShellArgs *pArgs = &shell.args;
+
+  for (int i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0) {
+      shellParseSingleOpt('?', NULL);
+      return 0;
+    }
+
+    char   *key = argv[i];
+    int32_t keyLen = strlen(key);
+    if (keyLen != 2) {
+      fprintf(stderr, "invalid option %s\n", key);
+      return -1;
+    }
+    if (key[0] != '-') {
+      fprintf(stderr, "invalid option %s\n", key);
+      return -1;
+    }
+
+    if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' || key[1] == 'a' || key[1] == 'c' || key[1] == 's' ||
+        key[1] == 'f' || key[1] == 'd' || key[1] == 'w' || key[1] == 'n' || key[1] == 'l' || key[1] == 'N') {
+      if (i + 1 >= argc) {
+        fprintf(stderr, "option %s requires an argument\n", key);
+        return -1;
+      }
+      char *val = argv[i + 1];
+      if (val[0] == '-') {
+        fprintf(stderr, "option %s requires an argument\n", key);
+        return -1;
+      }
+      shellParseSingleOpt(key[1], val);
+      i++;
+    } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'c' || key[1] == 'r' || key[1] == 'k' || key[1] == 't' ||
+               key[1] == 'V') {
+      shellParseSingleOpt(key[1], NULL);
+    } else {
+      fprintf(stderr, "invalid option %s\n", key);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+#ifdef LINUX
+#include <argp.h>
+#include <termio.h>
+
+const char *argp_program_version = version;
+const char *argp_program_bug_address = SHELL_EMAIL;
+
+static struct argp_option shellOptions[] = {
+    {"host", 'h', "HOST", 0, SHELL_HOST},
+    {"port", 'P', "PORT", 0, SHELL_PORT},
+    {"user", 'u', "USER", 0, SHELL_USER},
+    {0, 'p', 0, 0, SHELL_PASSWORD},
+    {"auth", 'a', "AUTH", 0, SHELL_AUTH},
+    {"generate-auth", 'A', 0, 0, SHELL_GEN_AUTH},
+    {"config-dir", 'c', "DIR", 0, SHELL_CFG_DIR},
+    {"dump-config", 'C', 0, 0, SHELL_DMP_CFG},
+    {"commands", 's', "COMMANDS", 0, SHELL_CMD},
+    {"raw-time", 'r', 0, 0, SHELL_RAW_TIME},
+    {"file", 'f', "FILE", 0, SHELL_FILE},
+    {"database", 'd', "DATABASE", 0, SHELL_DB},
+    {"check", 'k', 0, 0, SHELL_CHECK},
+    {"startup", 't', 0, 0, SHELL_STARTUP},
+    {"display-width", 'w', "WIDTH", 0, SHELL_WIDTH},
+    {"netrole", 'n', "NETROLE", 0, SHELL_NET_ROLE},
+    {"pktlen", 'l', "PKTLEN", 0, SHELL_PKG_LEN},
+    {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM},
+    {0},
+};
+
+static error_t shellParseOpt(int32_t key, char *arg, struct argp_state *state) { return shellParseSingleOpt(key, arg); }
+
+static struct argp shellArgp = {shellOptions, shellParseOpt, "", ""};
+
+static void shellParseArgsUseArgp(int argc, char *argv[]) {
+  argp_program_version = shell.info.programVersion;
+  argp_parse(&shellArgp, argc, argv, 0, 0, &shell.args);
+}
+
+#endif
+
+static void shellInitArgs(int argc, char *argv[]) {
+  for (int i = 1; i < argc; i++) {
+    if (strncmp(argv[i], "-p", 2) == 0) {
+      printf(shell.info.clientVersion, tsOsName, taos_get_client_info());
+      if (strlen(argv[i]) == 2) {
+        printf("Enter password: ");
+        taosSetConsoleEcho(false);
+        if (scanf("%20s", shell.args.password) > 1) {
+          fprintf(stderr, "password reading error\n");
+        }
+        taosSetConsoleEcho(true);
+        if (EOF == getchar()) {
+          fprintf(stderr, "getchar() return EOF\n");
+        }
+      } else {
+        tstrncpy(shell.args.password, (char *)(argv[i] + 2), sizeof(shell.args.password));
+        strcpy(argv[i], "-p");
+      }
+    }
+  }
+  if (strlen(shell.args.password) == 0) {
+    tstrncpy(shell.args.password, TSDB_DEFAULT_PASS, sizeof(shell.args.password));
+  }
+
+  SShellArgs *pArgs = &shell.args;
+  pArgs->user = TSDB_DEFAULT_USER;
+  pArgs->pktLen = SHELL_DEF_PKG_LEN;
+  pArgs->pktNum = SHELL_DEF_PKG_NUM;
+  pArgs->displayWidth = SHELL_DEFAULT_MAX_BINARY_DISPLAY_WIDTH;
+}
+
+static int32_t shellCheckArgs() {
+  SShellArgs *pArgs = &shell.args;
+  if (pArgs->host != NULL && (strlen(pArgs->host) <= 0 || strlen(pArgs->host) > TSDB_FQDN_LEN)) {
+    printf("Invalid host:%s\n", pArgs->host);
+    return -1;
+  }
+
+  if (pArgs->user != NULL && (strlen(pArgs->user) <= 0 || strlen(pArgs->user) > TSDB_USER_LEN)) {
+    printf("Invalid user:%s\n", pArgs->user);
+    return -1;
+  }
+
+  if (pArgs->auth != NULL && (strlen(pArgs->auth) <= 0 || strlen(pArgs->auth) > TSDB_PASSWORD_LEN)) {
+    printf("Invalid auth:%s\n", pArgs->auth);
+    return -1;
+  }
+
+  if (pArgs->database != NULL && (strlen(pArgs->database) <= 0 || strlen(pArgs->database) > TSDB_DB_NAME_LEN)) {
+    printf("Invalid database:%s\n", pArgs->database);
+    return -1;
+  }
+
+  if (pArgs->file[0] != 0) {
+    char fullname[PATH_MAX] = {0};
+    if (taosExpandDir(pArgs->file, fullname, PATH_MAX) == 0) {
+      tstrncpy(pArgs->file, fullname, PATH_MAX);
+    }
+  }
+
+  if (pArgs->cfgdir != NULL) {
+    if (strlen(pArgs->cfgdir) <= 0 || strlen(pArgs->cfgdir) >= PATH_MAX) {
+      printf("Invalid cfgdir:%s\n", pArgs->cfgdir);
+      return -1;
+    } else {
+      if (taosExpandDir(pArgs->cfgdir, configDir, PATH_MAX) != 0) {
+        tstrncpy(configDir, pArgs->cfgdir, PATH_MAX);
+      }
+    }
+  }
+
+  if (pArgs->commands != NULL && (strlen(pArgs->commands) <= 0)) {
+    printf("Invalid commands:%s\n", pArgs->commands);
+    return -1;
+  }
+
+  if (pArgs->netrole != NULL && !(strcmp(pArgs->netrole, "client") == 0 || strcmp(pArgs->netrole, "server") == 0)) {
+    printf("Invalid netrole:%s\n", pArgs->netrole);
+    return -1;
+  }
+
+  if (pArgs->password != NULL && (strlen(pArgs->password) <= 0)) {
+    printf("Invalid password\n");
+    return -1;
+  }
+
+  if (pArgs->pktLen < SHELL_MIN_PKG_LEN || pArgs->pktLen > SHELL_MAX_PKG_LEN) {
+    printf("Invalid pktLen:%d, range:[%d, %d]\n", pArgs->pktLen, SHELL_MIN_PKG_LEN, SHELL_MAX_PKG_LEN);
+    return -1;
+  }
+
+  if (pArgs->pktNum < SHELL_MIN_PKG_NUM || pArgs->pktNum > SHELL_MAX_PKG_NUM) {
+    printf("Invalid pktNum:%d, range:[%d, %d]\n", pArgs->pktNum, SHELL_MIN_PKG_NUM, SHELL_MAX_PKG_NUM);
+    return -1;
+  }
+
+  if (pArgs->displayWidth <= 0 || pArgs->displayWidth > 10 * 1024) {
+    printf("Invalid displayWidth:%d, range:[1, 10 * 1024]\n", pArgs->displayWidth);
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t shellParseArgs(int32_t argc, char *argv[]) {
+  shellInitArgs(argc, argv);
+  shell.info.clientVersion =
+      "Welcome to the TDengine shell from %s, Client Version:%s\n"
+      "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n";
+  shell.info.promptHeader = "taos> ";
+  shell.info.promptContinue = "   -> ";
+  shell.info.promptSize = 6;
+  snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "version: %s", version);
+
+#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
+  shell.info.osname = "Windows";
+  snprintf(shell.history.file, TSDB_FILENAME_LEN, "C:/TDengine/%s", SHELL_HISTORY_FILE);
+  if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1;
+#elif defined(_TD_DARWIN_64)
+  shell.info.osname = "Darwin";
+  snprintf(shell.history.file, TSDB_FILENAME_LEN, "%s/%s", getpwuid(getuid())->pw_dir, SHELL_HISTORY_FILE);
+  if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1;
+#else
+  shell.info.osname = "Linux";
+  snprintf(shell.history.file, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), SHELL_HISTORY_FILE);
+  shellParseArgsUseArgp(argc, argv);
+  // if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1;
+  if (shell.args.abort) {
+    return -1;
+  }
+#endif
+
+  return shellCheckArgs();
+}

tools/shell/src/shellNettest.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _GNU_SOURCE
+#include "shellInt.h"
+
+static void shellWorkAsClient() {
+  SShellArgs *pArgs = &shell.args;
+  SRpcInit    rpcInit = {0};
+  SEpSet      epSet = {.inUse = 0, .numOfEps = 1};
+  SRpcMsg     rpcRsp = {0};
+  void       *clientRpc = NULL;
+  char        pass[TSDB_PASSWORD_LEN + 1] = {0};
+
+  taosEncryptPass_c((uint8_t *)("_pwd"), strlen("_pwd"), pass);
+  rpcInit.label = "CHK";
+  rpcInit.numOfThreads = 1;
+  rpcInit.sessions = 16;
+  rpcInit.connType = TAOS_CONN_CLIENT;
+  rpcInit.idleTime = tsShellActivityTimer * 1000;
+  rpcInit.user = "_dnd";
+  rpcInit.ckey = "_key";
+  rpcInit.spi = 1;
+  rpcInit.secret = pass;
+
+  clientRpc = rpcOpen(&rpcInit);
+  if (clientRpc == NULL) {
+    printf("failed to init net test client since %s\n", terrstr());
+    goto _OVER;
+  }
+
+  if (pArgs->host == NULL) {
+    pArgs->host = tsFirst;
+  }
+  char fqdn[TSDB_FQDN_LEN] = {0};
+  tstrncpy(fqdn, pArgs->host, TSDB_FQDN_LEN);
+  strtok(fqdn, ":");
+
+  if (pArgs->port == 0) {
+    pArgs->port = tsServerPort;
+  }
+
+  printf("network test client is initialized, the server is %s:%u\n", fqdn, pArgs->port);
+
+  tstrncpy(epSet.eps[0].fqdn, fqdn, TSDB_FQDN_LEN);
+  epSet.eps[0].port = (uint16_t)pArgs->port;
+
+  int32_t  totalSucc = 0;
+  uint64_t startTime = taosGetTimestampUs();
+
+  for (int32_t i = 0; i < pArgs->pktNum; ++i) {
+    SRpcMsg rpcMsg = {.ahandle = (void *)0x9525, .msgType = TDMT_DND_NET_TEST};
+    rpcMsg.pCont = rpcMallocCont(pArgs->pktLen);
+    rpcMsg.contLen = pArgs->pktLen;
+
+    printf("request is sent, size:%d\n", rpcMsg.contLen);
+    rpcSendRecv(clientRpc, &epSet, &rpcMsg, &rpcRsp);
+    if (rpcRsp.code == 0 &&rpcRsp.contLen == rpcMsg.contLen) {
+      printf("response is received, size:%d\n", rpcMsg.contLen);
+      if (rpcRsp.code == 0) totalSucc++;
+    } else {
+      printf("response not received since %s\n", tstrerror(rpcRsp.code));
+    }
+
+    rpcFreeCont(rpcRsp.pCont);
+    rpcRsp.pCont = NULL;
+  }
+
+  uint64_t endTime = taosGetTimestampUs();
+  uint64_t elT = endTime - startTime;
+
+  printf("\ntotal succ:%5d/%d\tcost:%8.2lf ms\tspeed:%8.2lf MB/s\n", totalSucc, pArgs->pktNum, elT / 1000.0,
+         pArgs->pktLen / (elT / 1000000.0) / 1024.0 / 1024.0 * totalSucc);
+
+_OVER:
+  if (clientRpc != NULL) {
+    rpcClose(clientRpc);
+  }
+  if (rpcRsp.pCont != NULL) {
+    rpcFreeCont(rpcRsp.pCont);
+  }
+}
+
+static void shellProcessMsg(void *p, SRpcMsg *pRpc, SEpSet *pEpSet) {
+  printf("request is received, size:%d\n", pRpc->contLen);
+  fflush(stdout);
+  SRpcMsg rsp = {.handle = pRpc->handle, .refId = pRpc->refId, .ahandle = pRpc->ahandle, .code = 0};
+  rsp.pCont = rpcMallocCont(shell.args.pktLen);
+  rsp.contLen = shell.args.pktLen;
+  rpcSendResponse(&rsp);
+}
+
+void shellNettestHandler(int32_t signum, void *sigInfo, void *context) { shellExit(); }
+
+static void shellWorkAsServer() {
+  SShellArgs *pArgs = &shell.args;
+
+  if (pArgs->port == 0) {
+    pArgs->port = tsServerPort;
+  }
+
+  SRpcInit rpcInit = {0};
+  rpcInit.localPort = pArgs->port;
+  rpcInit.label = "CHK";
+  rpcInit.numOfThreads = tsNumOfRpcThreads;
+  rpcInit.cfp = (RpcCfp)shellProcessMsg;
+  rpcInit.sessions = 10;
+  rpcInit.connType = TAOS_CONN_SERVER;
+  rpcInit.idleTime = tsShellActivityTimer * 1000;
+
+  void *serverRpc = rpcOpen(&rpcInit);
+  if (serverRpc == NULL) {
+    printf("failed to init net test server since %s", terrstr());
+  } else {
+    printf("network test server is initialized, port:%u\n", pArgs->port);
+    taosSetSignal(SIGTERM, shellNettestHandler);
+    while (1) taosMsleep(10);
+  }
+}
+
+void shellTestNetWork() {
+  if (strcmp(shell.args.netrole, "client") == 0) {
+    shellWorkAsClient();
+  }
+
+  if (strcmp(shell.args.netrole, "server") == 0) {
+    shellWorkAsServer();
+  }
+}

tools/shell/src/shellUtil.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _BSD_SOURCE
+#define _GNU_SOURCE
+#define _XOPEN_SOURCE
+#define _DEFAULT_SOURCE
+
+#include "shellInt.h"
+
+bool shellRegexMatch(const char *s, const char *reg, int32_t cflags) {
+  regex_t regex = {0};
+  char    msgbuf[100] = {0};
+
+  /* Compile regular expression */
+  if (regcomp(&regex, reg, cflags) != 0) {
+    fprintf(stderr, "Fail to compile regex");
+    shellExit();
+  }
+
+  /* Execute regular expression */
+  int32_t reti = regexec(&regex, s, 0, NULL, 0);
+  if (!reti) {
+    regfree(&regex);
+    return true;
+  } else if (reti == REG_NOMATCH) {
+    regfree(&regex);
+    return false;
+  } else {
+    regerror(reti, &regex, msgbuf, sizeof(msgbuf));
+    fprintf(stderr, "Regex match failed: %s\n", msgbuf);
+    regfree(&regex);
+    shellExit();
+  }
+
+  return false;
+}
+
+int32_t shellCheckIntSize() {
+  if (sizeof(int8_t) != 1) {
+    printf("taos int8 size is %d(!= 1)", (int)sizeof(int8_t));
+    return -1;
+  }
+  if (sizeof(int16_t) != 2) {
+    printf("taos int16 size is %d(!= 2)", (int)sizeof(int16_t));
+    return -1;
+  }
+  if (sizeof(int32_t) != 4) {
+    printf("taos int32 size is %d(!= 4)", (int)sizeof(int32_t));
+    return -1;
+  }
+  if (sizeof(int64_t) != 8) {
+    printf("taos int64 size is %d(!= 8)", (int)sizeof(int64_t));
+    return -1;
+  }
+  return 0;
+}
+
+void shellPrintVersion() { printf("version: %s\n", version); }
+
+void shellGenerateAuth() {
+  char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
+  taosEncryptPass_c((uint8_t *)shell.args.password, strlen(shell.args.password), secretEncrypt);
+  printf("%s\n", secretEncrypt);
+  fflush(stdout);
+}
+
+void shellDumpConfig() {
+  SConfig *pCfg = taosGetCfg();
+  if (pCfg == NULL) {
+    printf("TDengine read global config failed!\n");
+  } else {
+    cfgDumpCfg(pCfg, 1, true);
+  }
+  fflush(stdout);
+}
+
+void shellCheckServerStatus() {
+  TSDB_SERVER_STATUS code;
+
+  do {
+    char details[1024] = {0};
+    code = taos_check_server_status(shell.args.host, shell.args.port, details, 1024);
+    switch (code) {
+      case TSDB_SRV_STATUS_UNAVAILABLE:
+        printf("0: unavailable\n");
+        break;
+      case TSDB_SRV_STATUS_NETWORK_OK:
+        printf("1: network ok\n");
+        break;
+      case TSDB_SRV_STATUS_SERVICE_OK:
+        printf("2: service ok\n");
+        break;
+      case TSDB_SRV_STATUS_SERVICE_DEGRADED:
+        printf("3: service degraded\n");
+        break;
+      case TSDB_SRV_STATUS_EXTING:
+        printf("4: exiting\n");
+        break;
+    }
+    if (strlen(details) != 0) {
+      printf("%s\n\n", details);
+    }
+    fflush(stdout);
+    if (code == TSDB_SRV_STATUS_NETWORK_OK && shell.args.is_startup) {
+      taosMsleep(1000);
+    } else {
+      break;
+    }
+  } while (1);
+}
+
+void shellExit() {
+  if (shell.conn != NULL) {
+    taos_close(shell.conn);
+    shell.conn = NULL;
+  }
+  taos_cleanup();
+  exit(EXIT_FAILURE);
+}
\ No newline at end of file