Merge branch 'develop' into xiaoping/add_test_case

Jenkinsfile

@@ -5,7 +5,7 @@ node {
     git url: 'https://github.com/taosdata/TDengine.git'
 }
 
-
+def kipstage=0
 def abortPreviousBuilds() {
   def currentJobName = env.JOB_NAME
   def currentBuildNumber = env.BUILD_NUMBER.toInteger()
@@ -31,7 +31,6 @@ def abort_previous(){
   if (buildNumber > 1) milestone(buildNumber - 1)
   milestone(buildNumber)
 }
-def kipstage=0
 def pre_test(){
     catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                 sh '''
@@ -46,7 +45,7 @@ def pre_test(){
     git pull
     git fetch origin +refs/pull/${CHANGE_ID}/merge
     git checkout -qf FETCH_HEAD
-    git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD develop)|grep -v -E '.*md|.*src/connector|Jenkinsfile' || exit 0
+    git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD develop)|grep -v -E '.*md|//src//connector|Jenkinsfile' || exit 0
     cd ${WK}
     git reset --hard HEAD~10
     git checkout develop
@@ -64,6 +63,7 @@ def pre_test(){
     '''
     return 1
 }
+
 pipeline {
   agent none
   
@@ -75,6 +75,9 @@ pipeline {
   stages {
       stage('pre_build'){
           agent{label 'master'}
+          when {
+              changeRequest()
+          }
           steps {
           sh'''
           cd ${WORKSPACE}
@@ -90,11 +93,12 @@ pipeline {
       }
     
       stage('Parallel test stage') {
+        
         //only build pr
         when {
               changeRequest()
-              expression {
-                    skipstage == 0
+               expression {
+                    skipstage != 1
               }
           }
       parallel {
@@ -258,8 +262,7 @@ pipeline {
     }
   }
   }
-  post {      
-      
+  post {  
         success {
             emailext (
                 subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' SUCCESS",
@@ -283,7 +286,7 @@ pipeline {
                                     <li>构建结果：<span style="color:green"> Successful </span></li>
                                     <li>构建编号：${BUILD_NUMBER}</li>
                                     <li>触发用户：${env.CHANGE_AUTHOR}</li>
-                                    <li>提交信息：${CHANGE_TITLE}</li>
+                                    <li>提交信息：${env.CHANGE_TITLE}</li>
                                     <li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
                                     <li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
                                     
@@ -318,10 +321,10 @@ pipeline {
                                 <ul>
                                 <div style="font-size:18px">
                                     <li>构建名称>>分支：${env.BRANCH_NAME}</li>
-                                    <li>构建结果：<span style="color:green"> Successful </span></li>
+                                    <li>构建结果：<span style="color:red"> Failure </span></li>
                                     <li>构建编号：${BUILD_NUMBER}</li>
                                     <li>触发用户：${env.CHANGE_AUTHOR}</li>
-                                    <li>提交信息：${CHANGE_TITLE}</li>
+                                    <li>提交信息：${env.CHANGE_TITLE}</li>
                                     <li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
                                     <li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
                                     

cmake/version.inc

@@ -13,28 +13,40 @@ ELSE ()
   SET(TD_VER_COMPATIBLE "2.0.0.0")
 ENDIF ()
 
+find_program(HAVE_GIT NAMES git)
+
 IF (DEFINED GITINFO)
   SET(TD_VER_GIT ${GITINFO})
+ELSEIF (HAVE_GIT)
+  execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${TD_COMMUNITY_DIR} OUTPUT_VARIABLE GIT_COMMITID)
+  message(STATUS "git log result:${GIT_COMMITID}")
+  IF (GIT_COMMITID)
+    string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID})
+    SET(TD_VER_GIT ${GIT_COMMITID})
+  ELSE ()
+    message(STATUS "not a git repository")
+    SET(TD_VER_GIT "no git commit id")
+  ENDIF ()
 ELSE ()
-  execute_process(
-	  COMMAND git log -1 --format=%H
-	  WORKING_DIRECTORY ${TD_COMMUNITY_DIR} 
-	  OUTPUT_VARIABLE GIT_COMMITID
-  )
-  string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID})
-  SET(TD_VER_GIT ${GIT_COMMITID})
+  message(STATUS "no git cmd")
+  SET(TD_VER_GIT "no git commit id")
 ENDIF ()
 
 IF (DEFINED GITINFOI)
   SET(TD_VER_GIT_INTERNAL ${GITINFOI})
+ELSEIF (HAVE_GIT)
+  execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID)
+  message(STATUS "git log result:${GIT_COMMITID}")
+  IF (GIT_COMMITID)
+    string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID})
+    SET(TD_VER_GIT_INTERNAL ${GIT_COMMITID})
+  ELSE ()
+    message(STATUS "not a git repository")
+    SET(TD_VER_GIT "no git commit id")
+  ENDIF ()
 ELSE ()
-  execute_process(
-	  COMMAND git log -1 --format=%H
-	  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} 
-	  OUTPUT_VARIABLE GIT_COMMITID
-  )
-  string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID})
-  SET(TD_VER_GIT_INTERNAL ${GIT_COMMITID})
+  message(STATUS "no git cmd")
+  SET(TD_VER_GIT_INTERNAL "no git commit id")
 ENDIF ()
 
 IF (DEFINED VERDATE)

src/client/src/tscLocal.c

@@ -892,7 +892,12 @@ int tscProcessLocalCmd(SSqlObj *pSql) {
   SSqlRes *pRes = &pSql->res;
 
   if (pCmd->command == TSDB_SQL_CFG_LOCAL) {
-    pRes->code = (uint8_t)taosCfgDynamicOptions(pCmd->payload);
+    if (taosCfgDynamicOptions(pCmd->payload)) {
+       pRes->code = TSDB_CODE_SUCCESS;
+    } else {
+       pRes->code = TSDB_CODE_COM_INVALID_CFG_MSG; 
+    } 
+    pRes->numOfRows = 0;
   } else if (pCmd->command == TSDB_SQL_DESCRIBE_TABLE) {
     pRes->code = (uint8_t)tscProcessDescribeTable(pSql);
   } else if (pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) {

src/client/src/tscSQLParser.c

@@ -129,6 +129,7 @@ static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo);
 static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo);
 static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index);
 static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid);
+static bool    validateDebugFlag(int32_t flag);
 
 int16_t getNewResColId(SQueryInfo* pQueryInfo) {
   return pQueryInfo->resColumnId--;
@@ -173,6 +174,16 @@ static uint8_t convertOptr(SStrToken *pToken) {
   }
 }
 
+static bool validateDebugFlag(int32_t v) {
+  const static int validFlag[] = {131, 135, 143};
+  
+  for (int i = 0; i < tListLen(validFlag); i++) {
+    if (v == validFlag[i]) {
+        return true;
+    } 
+  }
+  return false;
+}
 /*
  * Used during parsing query sql. Since the query sql usually small in length, error position
  * is not needed in the final error message.
@@ -565,16 +576,16 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       }
 
       int32_t numOfToken = (int32_t) taosArrayGetSize(pMiscInfo->a);
-      SStrToken* t = taosArrayGet(pMiscInfo->a, 0);
-      SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1);
+      assert(numOfToken >= 1 && numOfToken <= 2);
 
+      SStrToken* t = taosArrayGet(pMiscInfo->a, 0);
       strncpy(pCmd->payload, t->z, t->n);
       if (numOfToken == 2) {
+        SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1);
         pCmd->payload[t->n] = ' ';  // add sep
         strncpy(&pCmd->payload[t->n + 1], t1->z, t1->n);
-      }
-
-      break;
+      } 
+      return TSDB_CODE_SUCCESS; 
     }
 
     case TSDB_SQL_CREATE_TABLE: {
@@ -5357,13 +5368,15 @@ int32_t validateLocalConfig(SMiscInfo* pOptions) {
   SDNodeDynConfOption LOCAL_DYNAMIC_CFG_OPTIONS[6] = {{"resetLog", 8},    {"rpcDebugFlag", 12}, {"tmrDebugFlag", 12},
                                                       {"cDebugFlag", 10}, {"uDebugFlag", 10},   {"debugFlag", 9}};
 
+
   SStrToken* pOptionToken = taosArrayGet(pOptions->a, 0);
 
   if (numOfToken == 1) {
     // reset log does not need value
     for (int32_t i = 0; i < 1; ++i) {
       SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i];
-      if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) {
+      if ((pOption->len == pOptionToken->n) && 
+              (strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0)) {
         return TSDB_CODE_SUCCESS;
       }
     }
@@ -5371,15 +5384,14 @@ int32_t validateLocalConfig(SMiscInfo* pOptions) {
     SStrToken* pValToken = taosArrayGet(pOptions->a, 1);
 
     int32_t val = strtol(pValToken->z, NULL, 10);
-    if (val < 131 || val > 199) {
-      // options value is out of valid range
+    if (!validateDebugFlag(val)) {
       return TSDB_CODE_TSC_INVALID_SQL;
     }
 
     for (int32_t i = 1; i < tListLen(LOCAL_DYNAMIC_CFG_OPTIONS); ++i) {
       SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i];
-      if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) {
-        // options is valid
+      if ((pOption->len == pOptionToken->n) 
+              && (strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0)) {
         return TSDB_CODE_SUCCESS;
       }
     }

src/common/inc/tdataformat.h

@@ -134,6 +134,22 @@ typedef uint64_t TKEY;
 #define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key)))
 #define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1))
 
+#define MIN_TS_KEY ((TSKEY)0x8000000000000001)
+#define MAX_TS_KEY ((TSKEY)0x3fffffffffffffff)
+
+#define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key))
+
+static FORCE_INLINE TKEY keyToTkey(TSKEY key) {
+  TSKEY lkey = key;
+  if (key > MAX_TS_KEY) {
+    lkey = MAX_TS_KEY;
+  } else if (key < MIN_TS_KEY) {
+    lkey = MIN_TS_KEY;
+  }
+
+  return tdGetTKEY(lkey);
+}
+
 static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) {
   TSKEY key1 = tdGetKey(*(TKEY *)tkey1);
   TSKEY key2 = tdGetKey(*(TKEY *)tkey2);

src/common/src/tglobal.c

@@ -273,7 +273,7 @@ bool taosCfgDynamicOptions(char *msg) {
   int32_t   vint = 0;
 
   paGetToken(msg, &option, &olen);
-  if (olen == 0) return TSDB_CODE_COM_INVALID_CFG_MSG;
+  if (olen == 0) return false;;
 
   paGetToken(option + olen + 1, &value, &vlen);
   if (vlen == 0)
@@ -316,11 +316,9 @@ bool taosCfgDynamicOptions(char *msg) {
       }
       return true;
     }
-
     if (strncasecmp(cfg->option, "debugFlag", olen) == 0) {
-      taosSetAllDebugFlag();
+       taosSetAllDebugFlag(); 
     }
-    
     return true;
   }
 

src/inc/taoserror.h

@@ -192,7 +192,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING,           0, 0x0386, "Database n
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY,         0, 0x0387, "Database unsynced")
 
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_DAYS,   0, 0x0390, "Invalid database option: days out of range")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP,   0, 0x0391, "Invalid database option: keep >= keep2 >= keep1 >= days")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP,   0, 0x0391, "Invalid database option: keep >= keep1 >= keep0 >= days")
 
 // dnode
 TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED,        0, 0x0400, "Message not processed")

src/inc/tfs.h

@@ -64,7 +64,7 @@ typedef struct {
 #define tfsclose(fd) close(fd)
 #define tfsremove(pf) remove(TFILE_NAME(pf))
 #define tfscopy(sf, df) taosCopy(TFILE_NAME(sf), TFILE_NAME(df))
-#define tfsrename(sf, df) rename(TFILE_NAME(sf), TFILE_NAME(df))
+#define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df))
 
 void  tfsInitFile(TFILE *pf, int level, int id, const char *bname);
 bool  tfsIsSameFile(const TFILE *pf1, const TFILE *pf2);

src/kit/taosdemo/taosdemo.c

@@ -67,6 +67,7 @@ typedef struct DemoArguments {
   char *   sqlFile;
   bool     use_metric;
   bool     insert_only;
+  bool     answer_yes;
   char *   output_file;
   int      mode;
   char *   datatype[MAX_NUM_DATATYPE + 1];
@@ -114,6 +115,7 @@ typedef struct DemoArguments {
     {0, 'c', "config_directory",         0, "Configuration directory. Default is '/etc/taos/'.",                                                                14},
     #endif
     {0, 'x', 0,                          0, "Insert only flag.",                                                                                                13},
+    {0, 'y', 0,                          0, "Default input yes for prompt",                                                                                                13},
     {0, 'O', "order",                    0, "Insert mode--0: In order, 1: Out of order. Default is in order.",                                                  14},
     {0, 'R', "rate",                     0, "Out of order data's rate--if order=1 Default 10, min: 0, max: 50.",                                                14},
     {0, 'D', "delete table",             0, "Delete data methods——0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database",             14},
@@ -210,6 +212,9 @@ typedef struct DemoArguments {
       case 'x':
         arguments->insert_only = true;
         break;
+      case 'y':
+        arguments->answer_yes = true;
+        break;
       case 'c':
         if (wordexp(arg, &full_path, 0) != 0) {
           fprintf(stderr, "Invalid path %s\n", arg);
@@ -328,6 +333,8 @@ typedef struct DemoArguments {
     #endif
     printf("%s%s\n", indent, "-x");
     printf("%s%s%s\n", indent, indent, "flag, Insert only flag.");
+    printf("%s%s\n", indent, "-y");
+    printf("%s%s%s\n", indent, indent, "flag, Anser Yes for prompt.");
     printf("%s%s\n", indent, "-O");
     printf("%s%s%s\n", indent, indent, "order, Insert mode--0: In order, 1: Out of order. Default is in order.");
     printf("%s%s\n", indent, "-R");
@@ -409,6 +416,8 @@ typedef struct DemoArguments {
         arguments->use_metric = true;
       } else if (strcmp(argv[i], "-x") == 0) {
         arguments->insert_only = true;
+      } else if (strcmp(argv[i], "-y") == 0) {
+        arguments->answer_yes = true;
       } else if (strcmp(argv[i], "-c") == 0) {
         strcpy(configDir, argv[++i]);
       } else if (strcmp(argv[i], "-O") == 0) {
@@ -549,6 +558,7 @@ int main(int argc, char *argv[]) {
                                 NULL,
                                 false,           // use_metric
                                 false,           // insert_only
+                                false,           // answer_yes
                                 "./output.txt",  // output_file
                                 0,               // mode
                                 {
@@ -584,6 +594,7 @@ int main(int argc, char *argv[]) {
   arguments.num_of_RPR = 1000;
   arguments.use_metric = true;
   arguments.insert_only = false;
+  arguments.answer_yes = false;
   // end change
 
   parse_args(argc, argv, &arguments);
@@ -606,6 +617,7 @@ int main(int argc, char *argv[]) {
   int nrecords_per_request = arguments.num_of_RPR;
   bool use_metric = arguments.use_metric;
   bool insert_only = arguments.insert_only;
+  bool answer_yes = arguments.answer_yes;
   char **data_type = arguments.datatype;
   int count_data_type = 0;
   char dataString[STRING_LEN];
@@ -666,9 +678,12 @@ int main(int argc, char *argv[]) {
   printf("# Delete method:                     %d\n", method_of_delete);
   printf("# Test time:                         %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1,
           tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
-  printf("###################################################################\n\n");
-  printf("Press enter key to continue");
-  (void)getchar();
+
+  if (!answer_yes) {
+    printf("###################################################################\n\n");
+    printf("Press enter key to continue");
+    (void)getchar();
+  }
 
   fprintf(fp, "###################################################################\n");
   fprintf(fp, "# Server IP:                         %s:%hu\n", ip_addr == NULL ? "localhost" : ip_addr, port);

src/kit/taosdemox/taosdemox.c

@@ -1380,7 +1380,7 @@ static void printfDbInfoForQueryToFile(char* filename, SDbInfo* dbInfos, int ind
   fprintf(fp, "replica: %d\n", dbInfos->replica);
   fprintf(fp, "quorum: %d\n", dbInfos->quorum);
   fprintf(fp, "days: %d\n", dbInfos->days);    
-  fprintf(fp, "keep1,keep2,keep(D): %s\n", dbInfos->keeplist);      
+  fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist);      
   fprintf(fp, "cache(MB): %d\n", dbInfos->cache);
   fprintf(fp, "blocks: %d\n", dbInfos->blocks);
   fprintf(fp, "minrows: %d\n", dbInfos->minrows);

src/mnode/inc/mnodeDnode.h

@@ -67,6 +67,7 @@ void    mnodeCleanupDnodes();
 int32_t mnodeGetDnodesNum();
 int32_t mnodeGetOnlinDnodesCpuCoreNum();
 int32_t mnodeGetOnlineDnodesNum();
+void    mnodeGetOnlineAndTotalDnodesNum(int32_t *onlineNum, int32_t *totalNum);
 void *  mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode);
 void    mnodeCancelGetNextDnode(void *pIter);
 void    mnodeIncDnodeRef(SDnodeObj *pDnode);

src/mnode/src/mnodeDb.c

@@ -568,7 +568,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn
 
   pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "keep1,keep2,keep(D)");
+  strcpy(pSchema[cols].name, "keep0,keep1,keep(D)");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 

src/mnode/src/mnodeDnode.c

@@ -263,6 +263,28 @@ int32_t mnodeGetOnlineDnodesNum() {
   return onlineDnodes;
 }
 
+void mnodeGetOnlineAndTotalDnodesNum(int32_t *onlineNum, int32_t *totalNum) {
+  SDnodeObj *pDnode = NULL;
+  void *     pIter = NULL;
+  int32_t    onlineDnodes = 0, totalDnodes = 0;
+
+  while (1) {
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+    if (pDnode->status != TAOS_DN_STATUS_OFFLINE) ++onlineDnodes;
+    ++totalDnodes;
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  if (onlineNum) {
+    *onlineNum = onlineDnodes;
+  }
+
+  if (totalNum) {
+    *totalNum = totalDnodes;
+  }
+}
+
 void *mnodeGetDnode(int32_t dnodeId) {
   return sdbGetRow(tsDnodeSdb, &dnodeId);
 }

src/mnode/src/mnodeShow.c

@@ -280,8 +280,11 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) {
     }
   }
 
-  pRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum());
-  pRsp->totalDnodes = htonl(mnodeGetDnodesNum());
+  int32_t    onlineDnodes = 0, totalDnodes = 0;
+  mnodeGetOnlineAndTotalDnodesNum(&onlineDnodes, &totalDnodes);
+
+  pRsp->onlineDnodes = htonl(onlineDnodes);
+  pRsp->totalDnodes = htonl(totalDnodes);
   mnodeGetMnodeEpSetForShell(&pRsp->epSet, false);
 
   pMsg->rpcRsp.rsp = pRsp;

src/os/inc/osDir.h

@@ -25,8 +25,8 @@ extern "C" {
 // TAOS_OS_FUNC_DIR
 void taosRemoveDir(char *rootDir);
 int  taosMkDir(const char *pathname, mode_t mode); 
-void taosRename(char* oldName, char *newName);
 void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays);
+int32_t taosRename(char* oldName, char *newName);
 int32_t taosCompressFile(char *srcFileName, char *destFileName);
 
 #ifdef __cplusplus

src/os/inc/osWindows.h

@@ -69,6 +69,8 @@ extern "C" {
 #define TAOS_OS_FUNC_FILE_GETTMPFILEPATH
 #define TAOS_OS_FUNC_FILE_FTRUNCATE 
 
+#define TAOS_OS_FUNC_DIR
+
 #define TAOS_OS_FUNC_MATH
   #define SWAP(a, b, c)      \
     do {                     \

src/os/src/darwin/darwinEnv.c

@@ -29,17 +29,15 @@ static const char* expand_like_shell(const char *path) {
 
 void osInit() {
   if (configDir[0] == 0) {
-    strcpy(configDir, expand_like_shell("~/TDengine/cfg"));
+    strcpy(configDir, expand_like_shell("/usr/local/etc/taos"));
   }
+  strcpy(tsDataDir,   expand_like_shell("/usr/local/var/lib/taos"));
+  strcpy(tsLogDir,    expand_like_shell("/usr/local/var/log/taos"));
+  strcpy(tsScriptDir, expand_like_shell("/usr/local/etc/taos"));
 
   strcpy(tsVnodeDir, "");
   strcpy(tsDnodeDir, "");
   strcpy(tsMnodeDir, "");
-
-  strcpy(tsDataDir,   expand_like_shell("~/TDengine/data"));
-  strcpy(tsLogDir,    expand_like_shell("~/TDengine/log"));
-  strcpy(tsScriptDir, expand_like_shell("~/TDengine/cfg"));
-
   strcpy(tsOsName, "Darwin");
 }
 

src/os/src/detail/osDir.c

@@ -51,19 +51,22 @@ int taosMkDir(const char *path, mode_t mode) {
   return code;
 }
 
-void taosRename(char* oldName, char *newName) {
-  // if newName in not empty, rename return fail. 
-  // the newName must be empty or does not exist
-#ifdef WINDOWS
-  remove(newName);
-#endif
-  if (rename(oldName, newName)) {
+
+#ifndef TAOS_OS_FUNC_DIR
+
+int32_t taosRename(char* oldName, char *newName) {
+  int32_t code = rename(oldName, newName);
+  if (code < 0) {
     uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
   } else {
-    uInfo("successfully to rename file %s to %s", oldName, newName);
+    uTrace("successfully to rename file %s to %s", oldName, newName);
   }
+
+  return code;
 }
 
+#endif
+
 void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) {
   DIR *dir = opendir(rootDir);
   if (dir == NULL) return;

src/os/src/windows/wFile.c

@@ -160,7 +160,6 @@ int32_t taosFtruncate(int32_t fd, int64_t l_size) {
   return 0;
 }
 
-
 int fsync(int filedes) { 
   if (filedes < 0) {
     errno = EBADF;
@@ -172,3 +171,14 @@ int fsync(int filedes) {
   
   return FlushFileBuffers(h);
 }
+
+int32_t taosRename(char* oldName, char *newName) {
+  int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED);
+  if (code < 0) {
+    uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
+  } else {
+    uTrace("successfully to rename file %s to %s", oldName, newName);
+  }
+
+  return code;
+}
\ No newline at end of file

src/query/src/qExecutor.c

@@ -4571,7 +4571,7 @@ static void generateBlockDistResult(STableBlockDist *pTableBlockDist) {
   if (pTableBlockDist == NULL) {
      return;
   }
-  int64_t min = INT64_MAX, max = INT64_MIN, avg = 0;    
+  int64_t min = 0, max = 0, avg = 0;    
   SArray* blockInfos= pTableBlockDist->dataBlockInfos;  
   int64_t totalRows = 0, totalBlocks = taosArrayGetSize(blockInfos); 
   for (size_t i = 0; i < taosArrayGetSize(blockInfos); i++) {

src/sync/src/syncMain.c

@@ -741,11 +741,14 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
       if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue;
       if (masterIndex < 0) {
         masterIndex = index;
+        sDebug("vgId:%d, peer:%s is master, index:%d", pNode->vgId, pTemp->id, index);
       } else {  // multiple masters, it shall not happen
         if (masterIndex == pNode->selfIndex) {
           sError("%s, peer is master, work as slave instead", pTemp->id);
           nodeRole = TAOS_SYNC_ROLE_SLAVE;
           (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
+        } else {
+          sError("vgId:%d, peer:%s is master too, masterIndex:%d index:%d", pNode->vgId, pTemp->id, masterIndex, index);
         }
       }
     }
@@ -833,7 +836,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new
   }
 
   if (oldPeerRole != newPeerRole || nodeRole != oldSelfRole) {
-    sDebug("vgId:%d, roles changed, broadcast status", pNode->vgId);
+    sDebug("vgId:%d, roles changed, broadcast status, replica:%d", pNode->vgId, pNode->replica);
     syncBroadcastStatus(pNode);
   }
 
@@ -860,8 +863,12 @@ static void syncRestartPeer(SSyncPeer *pPeer) {
 void syncRestartConnection(SSyncPeer *pPeer) {
   if (pPeer->ip == 0) return;
 
+  if (syncAcquirePeer(pPeer->rid) == NULL) return;
+  
   syncRestartPeer(pPeer);
   syncCheckRole(pPeer, NULL, TAOS_SYNC_ROLE_OFFLINE);
+
+  syncReleasePeer(pPeer);
 }
 
 static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) {

src/sync/src/syncRestore.c

@@ -131,6 +131,11 @@ static int32_t syncProcessBufferedFwd(SSyncPeer *pPeer) {
   SRecvBuffer *pRecv = pNode->pRecv;
   int32_t      forwards = 0;
 
+  if (pRecv == NULL) {
+    sError("%s, recv buffer is null, restart connect", pPeer->id);
+    return -1;
+  }
+
   sDebug("%s, number of buffered forwards:%d", pPeer->id, pRecv->forwards);
 
   char *offset = pRecv->buffer;
@@ -179,6 +184,7 @@ int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) {
 
 static void syncCloseRecvBuffer(SSyncNode *pNode) {
   if (pNode->pRecv) {
+    sDebug("vgId:%d, recv buffer:%p is freed", pNode->vgId, pNode->pRecv);
     tfree(pNode->pRecv->buffer);
   }
 
@@ -203,6 +209,7 @@ static int32_t syncOpenRecvBuffer(SSyncNode *pNode) {
 
   pNode->pRecv = pRecv;
 
+  sDebug("vgId:%d, recv buffer:%p is created", pNode->vgId, pNode->pRecv);
   return 0;
 }
 

src/tsdb/src/tsdbFS.c

@@ -395,7 +395,7 @@ static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) {
   }
 
   (void)close(fd);
-  (void)rename(tfname, cfname);
+  (void)taosRename(tfname, cfname);
   taosTZfree(pBuf);
 
   return 0;

src/tsdb/src/tsdbRead.c

@@ -317,7 +317,7 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC
   pQueryHandle->window      = pCond->twindow;
   pQueryHandle->pTsdb       = tsdb;
   pQueryHandle->type        = TSDB_QUERY_TYPE_ALL;
-  pQueryHandle->cur.fid     = -1;
+  pQueryHandle->cur.fid     = INT32_MIN;
   pQueryHandle->cur.win     = TSWINDOW_INITIALIZER;
   pQueryHandle->checkFiles  = true;
   pQueryHandle->activeIndex = 0;   // current active table index
@@ -478,16 +478,18 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
   if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) {
     pMem = pMemT->tData[pCheckInfo->tableId.tid];
     if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid
+      TKEY tLastKey = keyToTkey(pCheckInfo->lastKey);
       pCheckInfo->iter =
-          tSkipListCreateIterFromVal(pMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
+          tSkipListCreateIterFromVal(pMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
     }
   }
 
   if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) {
     pIMem = pIMemT->tData[pCheckInfo->tableId.tid];
     if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid
+      TKEY tLastKey = keyToTkey(pCheckInfo->lastKey);
       pCheckInfo->iiter =
-          tSkipListCreateIterFromVal(pIMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
+          tSkipListCreateIterFromVal(pIMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
     }
   }
 
@@ -652,7 +654,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
   STsdbCfg *pCfg = &pHandle->pTsdb->config;
   size_t size = taosArrayGetSize(pHandle->pTableCheckInfo);
   assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1);
-  pHandle->cur.fid = -1;
+  pHandle->cur.fid = INT32_MIN;
 
   STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
 
@@ -1117,7 +1119,12 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity
 
     if (pColInfo->info.colId == src->colId) {
 
-      if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) {
+      if (pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
+        for (int32_t n = 0; n < num; n++) {
+          TKEY tkey = *(TKEY *)((char*)src->pData + bytes * start + n * sizeof(TKEY));
+          *(TSKEY *)(pData + n * sizeof(TSKEY)) = tdGetKey(tkey);
+        }
+      } else if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) {
         memmove(pData, (char*)src->pData + bytes * start, bytes * num);
       } else {  // handle the var-string
         char* dst = pData;
@@ -1225,7 +1232,6 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
           break;
         case TSDB_DATA_TYPE_BIGINT:
         case TSDB_DATA_TYPE_UBIGINT:
-        case TSDB_DATA_TYPE_TIMESTAMP:  
           *(uint64_t *)pData = *(uint64_t *)value;
           break;
         case TSDB_DATA_TYPE_FLOAT:
@@ -1234,6 +1240,9 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
         case TSDB_DATA_TYPE_DOUBLE:
           SET_DOUBLE_PTR(pData, value);
           break;
+        case TSDB_DATA_TYPE_TIMESTAMP:  
+          *(TSKEY *)pData = tdGetKey(*(TKEY *)value);
+          break;
         default:
           memcpy(pData, value, pColInfo->info.bytes);
       }
@@ -1893,7 +1902,7 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist
       assert(pQueryHandle->pFileGroup == NULL);
     }
 
-    cur->fid = -1;  // denote that there are no data in file anymore
+    cur->fid = INT32_MIN;  // denote that there are no data in file anymore
     *exists = false;
     return code;
   }
@@ -2496,7 +2505,7 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p
   STable* pTable = NULL;
 
   // there are data in file
-  if (pHandle->cur.fid >= 0) {
+  if (pHandle->cur.fid != INT32_MIN) {
     STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot];
     pTable = pBlockInfo->pTableCheckInfo->pTableObj;
   } else {
@@ -2580,7 +2589,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
    */
   STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle;
 
-  if (pHandle->cur.fid < 0) {
+  if (pHandle->cur.fid == INT32_MIN) {
     return pHandle->pColumns;
   } else {
     STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot];

src/wal/src/walWrite.c

@@ -39,7 +39,7 @@ int32_t walRenew(void *handle) {
 
   if (tfValid(pWal->tfd)) {
     tfClose(pWal->tfd);
-    wDebug("vgId:%d, file:%s, it is closed", pWal->vgId, pWal->name);
+    wDebug("vgId:%d, file:%s, it is closed while renew", pWal->vgId, pWal->name);
   }
 
   if (pWal->keep == TAOS_WAL_KEEP) {
@@ -56,7 +56,7 @@ int32_t walRenew(void *handle) {
     code = TAOS_SYSTEM_ERROR(errno);
     wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno));
   } else {
-    wDebug("vgId:%d, file:%s, it is created", pWal->vgId, pWal->name);
+    wDebug("vgId:%d, file:%s, it is created and open while renew", pWal->vgId, pWal->name);
   }
 
   pthread_mutex_unlock(&pWal->mutex);
@@ -95,11 +95,15 @@ void walRemoveAllOldFiles(void *handle) {
   int64_t fileId = -1;
 
   pthread_mutex_lock(&pWal->mutex);
+  
+  tfClose(pWal->tfd);
+  wDebug("vgId:%d, file:%s, it is closed before remove all wals", pWal->vgId, pWal->name);
+
   while (walGetNextFile(pWal, &fileId) >= 0) {
     snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId);
 
     if (remove(pWal->name) < 0) {
-      wError("vgId:%d, wal:%p file:%s, failed to remove", pWal->vgId, pWal, pWal->name);
+      wError("vgId:%d, wal:%p file:%s, failed to remove since %s", pWal->vgId, pWal, pWal->name, strerror(errno));
     } else {
       wInfo("vgId:%d, wal:%p file:%s, it is removed", pWal->vgId, pWal, pWal->name);
     }
@@ -192,7 +196,7 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) {
       wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno));
       return TAOS_SYSTEM_ERROR(errno);
     }
-    wDebug("vgId:%d, file:%s open success", pWal->vgId, pWal->name);
+    wDebug("vgId:%d, file:%s, it is created and open while restore", pWal->vgId, pWal->name);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -265,6 +269,8 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch
     wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno));
     tfree(buffer);
     return TAOS_SYSTEM_ERROR(errno);
+  } else {
+    wDebug("vgId:%d, file:%s, open for restore", pWal->vgId, name);
   }
 
   int32_t   code = TSDB_CODE_SUCCESS;
@@ -332,6 +338,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch
   tfClose(tfd);
   tfree(buffer);
 
+  wDebug("vgId:%d, file:%s, it is closed after restore", pWal->vgId, name);
   return code;
 }
 

tests/examples/JDBC/JDBCDemo/README-jdbc-windows.md

@@ -137,7 +137,7 @@ Welcome to the TDengine shell from Linux, Client Version:2.0.1.1
 Copyright (c) 2017 by TAOS Data, Inc. All rights reserved.
 
 taos> show databases;
-              name              |      created_time       |   ntables   |   vgroups   | replica | quorum |  days  |    keep1,keep2,keep(D)     |  cache(MB)  |   blocks    |   minrows   |   maxrows   | wallevel |    fsync    | comp | precision |    status    |
+              name              |      created_time       |   ntables   |   vgroups   | replica | quorum |  days  |    keep0,keep1,keep(D)     |  cache(MB)  |   blocks    |   minrows   |   maxrows   | wallevel |    fsync    | comp | precision |    status    |
 ===================================================================================================================================================================================================================================================================
  test                           | 2020-08-19 18:43:50.731 |           1 |           1 |       1 |      1 |      2 | 3650,3650,3650             |          16 |           6 |         100 |        4096 |        1 |        3000 |    2 | ms        | ready        |
  log                            | 2020-08-19 18:40:28.064 |           4 |           1 |       1 |      1 |     10 | 30,30,30                   |           1 |           3 |         100 |        4096 |        1 |        3000 |    2 | us        | ready        |

tests/pytest/insert/before_1970.py

@@ -41,7 +41,7 @@ class TDTestCase:
         #TODO : should add more testcases
         tdSql.execute("insert into test values('1930-12-12 01:19:20.345', 1);")
         tdSql.execute("insert into test values('1969-12-30 23:59:59.999', 2);")
-        tdSql.execute("insert into test values(-3600, 3);")
+        tdSql.execute("insert into test values(-3600001, 3);")
         tdSql.execute("insert into test values('2020-10-20 14:02:53.770', 4);")
         print("==============insert data")
 
@@ -61,11 +61,11 @@ class TDTestCase:
         print("==============step4")
         tdSql.execute("use demo;")
         tdSql.query("select * from test;")
-        # print(tdSql.queryResult)
+        print(tdSql.queryResult)
         tdSql.checkRows(4)
         tdSql.checkData(0,0,'1930-12-12 01:19:20.345000')
         tdSql.checkData(1,0,'1969-12-30 23:59:59.999000')
-        tdSql.checkData(2,0,'1970-01-01 07:00:00.000000')
+        tdSql.checkData(2,0,'1970-01-01 06:59:59.999000')
         tdSql.checkData(3,0,'2020-10-20 14:02:53.770000')
         print("==============check data")
 

tests/script/general/http/restful_full.sim

@@ -88,7 +88,7 @@ print ===============  step2 - no db
 #11
 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04'  -d 'show databases' 127.0.0.1:7111/rest/sql
 print 11-> $system_content
-if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"data":[],"rows":0}@ then
+if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep0,keep1,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"data":[],"rows":0}@ then
   return -1
 endi
 

tests/script/general/parser/col_arithmetic_operation.sim

@@ -110,7 +110,7 @@ system sh/exec.sh -n dnode1 -s start
 
 print ================== server restart completed
 sql connect
-sleep 500c
+sleep 5000
 
 run general/parser/col_arithmetic_query.sim
 

tests/script/general/parser/col_arithmetic_query.sim

@@ -41,10 +41,10 @@ if $data00 != 0.000000000 then
   return -1
 endi
 
-if $data01 != -nan then
-  print expect -nan, actual: $data01
-  return -1
-endi
+#if $data01 != -nan then
+#  print expect -nan, actual: $data01
+#  return -1
+#endi
 
 if $data10 != 0.666666667 then
   return -1
@@ -185,9 +185,9 @@ if $data00 != 0.000000000 then
   return -1
 endi
 
-if $data01 != -nan then
-  return -1
-endi
+#if $data01 != -nan then
+#  return -1
+#endi
 
 if $data02 != 0.000000000 then
   return -1

tests/script/sh/deploy.bat

@@ -65,24 +65,24 @@ echo serverPort             %NODE%             >> %TAOS_CFG%
 echo dataDir                %DATA_DIR%         >> %TAOS_CFG%
 echo logDir                 %LOG_DIR%          >> %TAOS_CFG%
 echo debugFlag              0                  >> %TAOS_CFG%
-echo mDebugFlag             143                >> %TAOS_CFG%
-echo sdbDebugFlag           143                >> %TAOS_CFG%
-echo dDebugFlag             143                >> %TAOS_CFG%
-echo vDebugFlag             143                >> %TAOS_CFG%
-echo tsdbDebugFlag          143                >> %TAOS_CFG%
-echo cDebugFlag             143                >> %TAOS_CFG%
-echo jnidebugFlag           143                >> %TAOS_CFG%
-echo odbcdebugFlag          143                >> %TAOS_CFG%
-echo httpDebugFlag          143                >> %TAOS_CFG%
-echo monDebugFlag           143                >> %TAOS_CFG%
-echo mqttDebugFlag          143                >> %TAOS_CFG%
-echo qdebugFlag             143                >> %TAOS_CFG%
-echo rpcDebugFlag           143                >> %TAOS_CFG%
+echo mDebugFlag             135                >> %TAOS_CFG%
+echo sdbDebugFlag           135                >> %TAOS_CFG%
+echo dDebugFlag             135                >> %TAOS_CFG%
+echo vDebugFlag             135                >> %TAOS_CFG%
+echo tsdbDebugFlag          135                >> %TAOS_CFG%
+echo cDebugFlag             135                >> %TAOS_CFG%
+echo jnidebugFlag           135                >> %TAOS_CFG%
+echo odbcdebugFlag          135                >> %TAOS_CFG%
+echo httpDebugFlag          135                >> %TAOS_CFG%
+echo monDebugFlag           135                >> %TAOS_CFG%
+echo mqttDebugFlag          135                >> %TAOS_CFG%
+echo qdebugFlag             135                >> %TAOS_CFG%
+echo rpcDebugFlag           135                >> %TAOS_CFG%
 echo tmrDebugFlag           131                >> %TAOS_CFG%
-echo udebugFlag             143                >> %TAOS_CFG%
-echo sdebugFlag             143                >> %TAOS_CFG%
-echo wdebugFlag             143                >> %TAOS_CFG%
-echo cqdebugFlag            143                >> %TAOS_CFG%
+echo udebugFlag             135                >> %TAOS_CFG%
+echo sdebugFlag             135                >> %TAOS_CFG%
+echo wdebugFlag             135                >> %TAOS_CFG%
+echo cqdebugFlag            135                >> %TAOS_CFG%
 echo monitor                0                  >> %TAOS_CFG%
 echo monitorInterval        1                  >> %TAOS_CFG%
 echo http                   0                  >> %TAOS_CFG%