Merge branch 'develop' of https://github.com/taosdata/TDengine into develop

.travis.yml

@@ -13,7 +13,7 @@ branches:
 
 matrix:
   - os: linux
-    dist: bionic
+    dist: focal
     language: c
 
     git:
@@ -28,8 +28,6 @@ matrix:
           - build-essential
           - cmake
           - net-tools
-          - python-pip
-          - python-setuptools
           - python3-pip
           - python3-setuptools
           - valgrind
@@ -54,13 +52,19 @@ matrix:
           cd ${TRAVIS_BUILD_DIR}/debug
           make install > /dev/null || travis_terminate $?
 
-          pip install numpy
-          pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
           pip3 install numpy
           pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
 
           cd ${TRAVIS_BUILD_DIR}/tests
           ./test-all.sh smoke || travis_terminate $?
+          sleep 1
+
+          cd ${TRAVIS_BUILD_DIR}/tests/pytest
+          pkill -TERM -x taosd
+          fuser -k -n tcp 6030
+          sleep 1
+          ./crash_gen.sh -a -p -t 4 -s 25|| travis_terminate $?
+          sleep 1
 
           cd ${TRAVIS_BUILD_DIR}/tests/pytest
           ./valgrind-test.sh 2>&1 > mem-error-out.log
@@ -160,7 +164,7 @@ matrix:
 
     script:
       - cmake .. > /dev/null
-      - make > /dev/null
+      - make
 
   - os: linux
     dist: bionic

src/client/src/tscLocalMerge.c

@@ -274,6 +274,10 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
   pReducer->numOfBuffer = idx;
 
   SCompareParam *param = malloc(sizeof(SCompareParam));
+  if (param == NULL) {
+    tfree(pReducer);
+    return;
+  }
   param->pLocalData = pReducer->pLocalDataSrc;
   param->pDesc = pReducer->pDesc;
   param->num = pReducer->pLocalDataSrc[0]->pMemBuffer->numOfElemsPerPage;
@@ -284,6 +288,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
 
   pRes->code = tLoserTreeCreate(&pReducer->pLoserTree, pReducer->numOfBuffer, param, treeComparator);
   if (pReducer->pLoserTree == NULL || pRes->code != 0) {
+    tfree(param);
     tfree(pReducer);
     return;
   }
@@ -332,6 +337,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
     tfree(pReducer->pResultBuf);
     tfree(pReducer->pFinalRes);
     tfree(pReducer->prevRowOfInput);
+    tfree(pReducer->pLoserTree);
+    tfree(param);
     tfree(pReducer);
     pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return;

src/client/src/tscStream.c

@@ -255,6 +255,9 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
     // release the metric/meter meta information reference, so data in cache can be updated
 
     taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false);
+    tscFreeSqlResult(pSql);
+    tfree(pSql->pSubs);
+    pSql->numOfSubs = 0;
     tfree(pTableMetaInfo->vgroupList);
     tscSetNextLaunchTimer(pStream, pSql);
   }

src/client/src/tscSubquery.c

@@ -1447,9 +1447,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
 static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) {
   tscDebug("%p start to free subquery result", pSql);
   
-  if (pSql->res.code == TSDB_CODE_SUCCESS) {
-    taos_free_result(pSql);
-  }
+  taos_free_result(pSql);
   
   tfree(trsupport->localBuffer);
   
@@ -1780,6 +1778,7 @@ static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsu
     pSql->pSubs[trsupport->subqueryIndex] = pNew;
   }
   
+  printf("------------alloc:%p\n", pNew);
   return pNew;
 }
 

src/client/src/tscUtil.c

@@ -1822,7 +1822,6 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
     STableMeta*  pPrevTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta);
     
     SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList;
-    pPrevInfo->vgroupList = NULL;
     pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList);
   }
 

src/common/src/tglobal.c

@@ -129,7 +129,7 @@ int32_t tsMnodeEqualVnodeNum = 4;
 int32_t  tsEnableHttpModule = 1;
 int32_t  tsRestRowLimit = 10240;
 uint16_t tsHttpPort = 6020;  // only tcp, range tcp[6020]
-int32_t  tsHttpCacheSessions = 100;
+int32_t  tsHttpCacheSessions = 1000;
 int32_t  tsHttpSessionExpire = 36000;
 int32_t  tsHttpMaxThreads = 2;
 int32_t  tsHttpEnableCompress = 0;

src/connector/jdbc/buildTDengine.sh

-#!/bin/bash
-ulimit -c unlimited
-
-function buildTDengine {    
-	cd /root/TDengine
-
-	git remote update
-	REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
-	LOCAL_COMMIT=`git rev-parse --short @`
-
-	echo " LOCAL: $LOCAL_COMMIT"
-	echo "REMOTE: $REMOTE_COMMIT"
-	if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
-		echo "repo up-to-date"
-	else
-		echo "repo need to pull"        
-		git pull
-
-		LOCAL_COMMIT=`git rev-parse --short @`        
-		cd /root/TDengine/debug        
-		rm -rf /root/TDengine/debug/*        
-		cmake ..
-		make > /dev/null
-		make install
-	fi
-}
-
-function restartTaosd {
-	systemctl stop taosd
-	pkill -KILL -x taosd
-	sleep 10
-	
-    	logDir=`grep 'logDir' /etc/taos/taos.cfg|awk 'END{print $2}'`
-    	dataDir=`grep 'dataDir' /etc/taos/taos.cfg|awk '{print $2}'`    
-    
-	rm -rf $logDir/*
-	rm -rf $dataDir/*
-	
-	taosd 2>&1 > /dev/null &
-	sleep 10
-}
-
-buildTDengine
-restartTaosd

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java

+package com.taosdata.jdbc.utils;
+
+import java.io.File;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+public class TDNode {
+    
+    private int index;
+    private int running;
+    private int deployed;
+    private boolean testCluster;    
+    private String path;    
+    private String cfgDir;
+    private String dataDir;
+    private String logDir;
+    private String cfgPath;
+
+    public TDNode(int index) {
+        this.index = index;
+        running = 0;
+        deployed = 0;
+        testCluster = false;            
+    }
+
+    public void setPath(String path) {
+        this.path = path;
+    }
+
+    public void setTestCluster(boolean testCluster) {
+        this.testCluster = testCluster;
+    }
+
+    public void searchTaosd(File dir, ArrayList<String> taosdPath) {        
+        File[] fileList = dir.listFiles();
+        
+        if(fileList == null || fileList.length == 0) {
+            return;
+        }
+        
+        for(File file : fileList) {
+            if(file.isFile()) {
+                if(file.getName().equals("taosd")) {                        
+                    taosdPath.add(file.getAbsolutePath());
+                }
+            } else {
+                searchTaosd(file, taosdPath);
+            } 
+        }        
+    }
+
+    public void start() {
+        String selfPath = System.getProperty("user.dir");
+        String binPath = "";
+        String projDir = selfPath + "/../../../";
+
+        try {
+            ArrayList<String> taosdPath = new ArrayList<>();
+            
+            File dir = new File(projDir);
+            String realProjDir = dir.getCanonicalPath();
+            dir = new File(realProjDir);
+            System.out.println("project Dir: " + projDir);
+            searchTaosd(dir, taosdPath);
+                        
+            if(taosdPath.size() == 0) {
+                System.out.println("The project path doens't exist");
+                return;
+            } else {
+                for(String p : taosdPath) {
+                    if(!p.contains("packaging")) {
+                        binPath = p;
+                    }
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+        if(binPath.isEmpty()) {
+            System.out.println("taosd not found");
+            return;
+        } else {
+            System.out.println("taosd found in " + binPath);
+        }
+
+        if(this.deployed == 0) {
+            System.out.println("dnode" + index + "is not deployed");
+            return;
+        }
+
+        String cmd = "nohup " + binPath + " -c " + cfgDir + " > /dev/null 2>&1 & "; 
+        System.out.println("start taosd cmd: " + cmd);
+        
+        try{
+            Runtime.getRuntime().exec(cmd);  
+            TimeUnit.SECONDS.sleep(5);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        
+        this.running = 1;        
+    }
+
+    public void stop() {
+        String toBeKilled = "taosd";
+
+        if (this.running != 0) {
+            String killCmd = "pkill -kill -x " + toBeKilled;
+            String[] killCmds = {"sh", "-c", killCmd};
+            try {                                
+                Runtime.getRuntime().exec(killCmds).waitFor();
+
+                for(int port = 6030; port < 6041; port ++) {
+                    String fuserCmd = "fuser -k -n tcp " + port;
+                    Runtime.getRuntime().exec(fuserCmd).waitFor();
+                }
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+
+            this.running = 0;
+            System.out.println("dnode:" + this.index + " is stopped by pkill");
+        }
+    }
+
+    public void startIP() {
+        try{
+            String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " up";        
+            Runtime.getRuntime().exec(cmd).waitFor();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void stopIP() {
+        try{
+            String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " down";            
+            Runtime.getRuntime().exec(cmd).waitFor();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void setCfgConfig(String option, String value) {
+        try{
+            String cmd = "echo " + option + " " + value + " >> " + this.cfgPath;
+            String[] cmdLine = {"sh", "-c", cmd};
+            Process ps = Runtime.getRuntime().exec(cmdLine);
+            ps.waitFor();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public String getDnodeRootDir() {
+        String dnodeRootDir = this.path + "/sim/psim/dnode" + this.index;
+        return dnodeRootDir;
+    }
+
+    public String getDnodesRootDir() {
+        String dnodesRootDir = this.path + "/sim/psim" + this.index;
+        return dnodesRootDir;
+    }
+
+    public void deploy() {
+        this.logDir = this.path + "/sim/dnode" + this.index + "/log";
+        this.dataDir = this.path + "/sim/dnode" + this.index + "/data";
+        this.cfgDir = this.path + "/sim/dnode" + this.index + "/cfg";
+        this.cfgPath = this.path + "/sim/dnode" + this.index + "/cfg/taos.cfg";
+
+        try {        
+            String cmd = "rm -rf " + this.logDir;
+            Runtime.getRuntime().exec(cmd).waitFor();
+            
+            cmd = "rm -rf " + this.cfgDir;
+            Runtime.getRuntime().exec(cmd).waitFor();
+
+            cmd = "rm -rf " + this.dataDir;
+            Runtime.getRuntime().exec(cmd).waitFor();
+
+            cmd = "mkdir -p " + this.logDir;
+            Runtime.getRuntime().exec(cmd).waitFor();
+
+            cmd = "mkdir -p " + this.cfgDir;
+            Runtime.getRuntime().exec(cmd).waitFor();
+
+            cmd = "mkdir -p " + this.dataDir;
+            Runtime.getRuntime().exec(cmd).waitFor();
+
+            cmd = "touch " + this.cfgPath;
+            Runtime.getRuntime().exec(cmd).waitFor();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }        
+
+        if(this.testCluster) {
+            startIP();
+            setCfgConfig("masterIp", "192.168.0.1");
+            setCfgConfig("secondIp", "192.168.0.2");
+            setCfgConfig("publicIp", "192.168.0." + this.index);
+            setCfgConfig("internalIp", "192.168.0." + this.index);
+            setCfgConfig("privateIp", "192.168.0." + this.index);
+        }
+        setCfgConfig("dataDir", this.dataDir);
+        setCfgConfig("logDir", this.logDir);
+        setCfgConfig("numOfLogLines", "1000000/00");
+        setCfgConfig("mnodeEqualVnodeNum", "0");
+        setCfgConfig("walLevel", "1");
+        setCfgConfig("statusInterval", "1");
+        setCfgConfig("numOfTotalVnodes", "64");
+        setCfgConfig("numOfMnodes", "3");
+        setCfgConfig("numOfThreadsPerCore", "2.0");
+        setCfgConfig("monitor", "0");
+        setCfgConfig("maxVnodeConnections", "30000");
+        setCfgConfig("maxMgmtConnections", "30000");
+        setCfgConfig("maxMeterConnections", "30000");
+        setCfgConfig("maxShellConns", "30000");
+        setCfgConfig("locale", "en_US.UTF-8");
+        setCfgConfig("charset", "UTF-8");
+        setCfgConfig("asyncLog", "0");
+        setCfgConfig("anyIp", "0");
+        setCfgConfig("dDebugFlag", "135");
+        setCfgConfig("mDebugFlag", "135");
+        setCfgConfig("sdbDebugFlag", "135");
+        setCfgConfig("rpcDebugFlag", "135");
+        setCfgConfig("tmrDebugFlag", "131");
+        setCfgConfig("cDebugFlag", "135");
+        setCfgConfig("httpDebugFlag", "135");
+        setCfgConfig("monitorDebugFlag", "135");
+        setCfgConfig("udebugFlag", "135");
+        setCfgConfig("jnidebugFlag", "135");
+        setCfgConfig("qdebugFlag", "135"); 
+        this.deployed = 1;
+    }
+}
\ No newline at end of file

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java

+package com.taosdata.jdbc.utils;
+
+import java.io.File;
+import java.util.*;
+
+public class TDNodes {
+    private ArrayList<TDNode> tdNodes; 
+    private boolean testCluster;     
+
+    public TDNodes () {
+        tdNodes = new ArrayList<>();
+        for(int i = 1; i < 11; i ++) {            
+            tdNodes.add(new TDNode(i));
+        }               
+    }
+
+    public void setPath(String path) {
+        try {
+            String killCmd = "pkill -kill -x taosd";
+            String[] killCmds = {"sh", "-c", killCmd};
+            Runtime.getRuntime().exec(killCmds).waitFor();        
+
+            String binPath = System.getProperty("user.dir");
+            binPath += "/../../../debug";
+            System.out.println("binPath: " + binPath);
+        
+            File file = new File(path);
+            binPath = file.getCanonicalPath();
+            System.out.println("binPath real path: " + binPath);
+                
+            if(path.isEmpty()){
+                file = new File(path + "/../../");                
+                path = file.getCanonicalPath();
+            }            
+
+            for(int i = 0; i < tdNodes.size(); i++) {
+                tdNodes.get(i).setPath(path);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+    
+    public void setTestCluster(boolean testCluster) {
+        this.testCluster = testCluster;
+    }
+
+    public void check(int index) {
+        if(index < 1 || index > 10) {
+            System.out.println("index: " + index + " should on a scale of [1, 10]");
+            return;
+        }
+    } 
+
+    public void deploy(int index) {
+        try {     
+            File file = new File(System.getProperty("user.dir") + "/../../../");
+            String projectRealPath = file.getCanonicalPath();
+            check(index);        
+            tdNodes.get(index - 1).setTestCluster(this.testCluster);            
+            tdNodes.get(index - 1).setPath(projectRealPath); 
+            tdNodes.get(index - 1).deploy();
+        } catch (Exception e) {
+            e.printStackTrace();
+            System.out.println("deploy Test Exception");
+        }
+    }
+    
+    public void cfg(int index, String option, String value) {
+        check(index);
+        tdNodes.get(index - 1).setCfgConfig(option, value);
+    }
+    
+    public void start(int index) {
+        check(index);
+        tdNodes.get(index - 1).start();
+    }
+
+    public void stop(int index) {
+        check(index);
+        tdNodes.get(index - 1).stop();
+    }
+    
+    public void startIP(int index) {
+        check(index);
+        tdNodes.get(index - 1).startIP();
+    }
+
+    public void stopIP(int index) {
+        check(index);
+        tdNodes.get(index - 1).stopIP();
+    }
+
+}
\ No newline at end of file

src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java

 package com.taosdata.jdbc;
 
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
+import java.io.File;
+import com.taosdata.jdbc.utils.TDNodes;
 
+import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 public class BaseTest {
+
+    private static boolean testCluster = false;
+    private static String deployPath = System.getProperty("user.dir");         
+    private static TDNodes tdNodes = new TDNodes();    
+    
     
     @BeforeClass
     public static void setupEnv() {
         try{
-            String path = System.getProperty("user.dir");
-            String bashPath = path + "/buildTDengine.sh";
+            File file = new File(deployPath + "/../../../");
+            String rootPath = file.getCanonicalPath();
+                       
+            tdNodes.setPath(rootPath);
+            tdNodes.setTestCluster(testCluster);            
 
-            Process ps = Runtime.getRuntime().exec(bashPath);
-            ps.waitFor();
+            tdNodes.deploy(1);
+            tdNodes.start(1);  
 
-            BufferedReader br = new BufferedReader(new InputStreamReader(ps.getInputStream()));
-            while(br.readLine() != null) {
-                System.out.println(br.readLine());
-            }            
         } catch (Exception e) {
             e.printStackTrace();
+            System.out.println("Base Test Exception");
         }
     }
+
+    @AfterClass
+    public static void cleanUpEnv() {
+        tdNodes.stop(1);
+    }
 }
\ No newline at end of file

src/dnode/inc/dnodeMgmt.h

@@ -22,6 +22,8 @@ extern "C" {
 
 int32_t dnodeInitMgmt();
 void    dnodeCleanupMgmt();
+int32_t dnodeInitMgmtTimer();
+void    dnodeCleanupMgmtTimer();
 void    dnodeDispatchToMgmtQueue(SRpcMsg *rpcMsg);
 
 void*   dnodeGetVnode(int32_t vgId);

src/dnode/src/dnodeMain.c

@@ -57,6 +57,7 @@ static const SDnodeComponent tsDnodeComponents[] = {
   {"server",  dnodeInitServer,     dnodeCleanupServer},
   {"mgmt",    dnodeInitMgmt,       dnodeCleanupMgmt},
   {"modules", dnodeInitModules,    dnodeCleanupModules},
+  {"mgmt-tmr",dnodeInitMgmtTimer,  dnodeCleanupMgmtTimer},
   {"shell",   dnodeInitShell,      dnodeCleanupShell}
 };
 

src/dnode/src/dnodeMgmt.c

@@ -147,6 +147,12 @@ int32_t dnodeInitMgmt() {
     return -1;
   }
 
+  dInfo("dnode mgmt is initialized");
+ 
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t dnodeInitMgmtTimer() {
   tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
   if (tsDnodeTmr == NULL) {
     dError("failed to init dnode timer");
@@ -155,13 +161,11 @@ int32_t dnodeInitMgmt() {
   }
 
   taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
-  
-  dInfo("dnode mgmt is initialized");
- 
+  dInfo("dnode mgmt timer is initialized");
   return TSDB_CODE_SUCCESS;
 }
 
-void dnodeCleanupMgmt() {
+void dnodeCleanupMgmtTimer() {
   if (tsStatusTimer != NULL) {
     taosTmrStopA(&tsStatusTimer);
     tsStatusTimer = NULL;
@@ -171,7 +175,10 @@ void dnodeCleanupMgmt() {
     taosTmrCleanUp(tsDnodeTmr);
     tsDnodeTmr = NULL;
   }
+}
 
+void dnodeCleanupMgmt() {
+  dnodeCleanupMgmtTimer();
   dnodeCloseVnodes();
 
   if (tsMgmtQset) taosQsetThreadResume(tsMgmtQset);

src/dnode/src/dnodeSystem.c

@@ -20,6 +20,7 @@
 #include "tglobal.h"
 #include "dnodeInt.h"
 #include "dnodeMain.h"
+#include "tfile.h"
 
 static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context);
 static sem_t exitSem;
@@ -67,6 +68,18 @@ int32_t main(int32_t argc, char *argv[]) {
         taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, NULL, true);
       }
     }
+#endif
+#ifdef TAOS_RANDOM_FILE_FAIL
+    else if (strcmp(argv[i], "--random-file-fail-factor") == 0) {
+      if ( (i+1) < argc ) {
+        int factor = atoi(argv[i+1]);
+        printf("The factor of random failure is %d\n", factor);
+        taosSetRandomFileFailFactor(factor);
+      } else {
+        printf("Please specify a number for random failure factor!");
+        exit(EXIT_FAILURE);
+      }
+    }
 #endif
   }
 

src/mnode/src/mnodeDb.c

@@ -179,9 +179,14 @@ void mnodeDecDbRef(SDbObj *pDb) {
 
 SDbObj *mnodeGetDbByTableId(char *tableId) {
   char db[TSDB_TABLE_ID_LEN], *pos;
-
+ 
+  // tableId format should be :  acct.db.table
   pos = strstr(tableId, TS_PATH_DELIMITER);
+  assert(NULL != pos);
+
   pos = strstr(pos + 1, TS_PATH_DELIMITER);
+  assert(NULL != pos);
+
   memset(db, 0, sizeof(db));
   strncpy(db, tableId, pos - tableId);
 

src/mnode/src/mnodeMain.c

@@ -41,7 +41,7 @@ typedef struct {
   void              (*cleanup)();
 } SMnodeComponent;
 
-void *tsMnodeTmr;
+void *tsMnodeTmr = NULL;
 static bool tsMgmtIsRunning = false;
 
 static const SMnodeComponent tsMnodeComponents[] = {

src/mnode/src/mnodeVgroup.c

@@ -372,7 +372,6 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
           pVgroup->vnodeGid[i].dnodeId);
   }
 
-  mnodeIncVgroupRef(pVgroup);
   pMsg->expected = pVgroup->numOfVnodes;
   mnodeSendCreateVgroupMsg(pVgroup, pMsg);
 
@@ -393,6 +392,9 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg, SDbObj *pDb) {
     return TSDB_CODE_MND_NO_ENOUGH_DNODES;
   }
 
+  pMsg->pVgroup = pVgroup;
+  mnodeIncVgroupRef(pVgroup);
+
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsVgroupSdb,
@@ -402,8 +404,6 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg, SDbObj *pDb) {
     .cb = mnodeCreateVgroupCb
   };
 
-  pMsg->pVgroup = pVgroup;
-
   int32_t code = sdbInsertRow(&oper);
   if (code != TSDB_CODE_SUCCESS) {
     pMsg->pVgroup = NULL;
@@ -814,19 +814,20 @@ static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) {
     mDebug("dnode:%s, vgId:%d, invalid dnode", taosIpStr(pCfg->dnodeId), pCfg->vgId);
     return TSDB_CODE_MND_VGROUP_NOT_EXIST;
   }
-  mnodeDecDnodeRef(pDnode);
 
   SVgObj *pVgroup = mnodeGetVgroup(pCfg->vgId);
   if (pVgroup == NULL) {
     mDebug("dnode:%s, vgId:%d, no vgroup info", taosIpStr(pCfg->dnodeId), pCfg->vgId);
+    mnodeDecDnodeRef(pDnode);
     return TSDB_CODE_MND_VGROUP_NOT_EXIST;
   }
-  mnodeDecVgroupRef(pVgroup);
 
   mDebug("vgId:%d, send create vnode msg to dnode %s for vnode cfg msg", pVgroup->vgId, pDnode->dnodeEp);
   SRpcIpSet ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp);
   mnodeSendCreateVnodeMsg(pVgroup, &ipSet, NULL);
 
+  mnodeDecDnodeRef(pDnode);
+  mnodeDecVgroupRef(pVgroup);
   return TSDB_CODE_SUCCESS;
 }
 

src/plugins/http/inc/httpInt.h

@@ -206,7 +206,7 @@ typedef struct HttpThread {
   pthread_mutex_t threadMutex;
   bool            stop;
   int             pollFd;
-  int             numOfFds;
+  int             numOfContexts;
   int             threadId;
   char            label[HTTP_LABEL_SIZE];
   bool (*processData)(HttpContext *pContext);

src/plugins/http/src/httpContext.c

@@ -44,7 +44,7 @@ static void httpDestroyContext(void *data) {
   HttpThread *pThread = pContext->pThread;
   httpRemoveContextFromEpoll(pContext);
   httpReleaseSession(pContext);
-  atomic_sub_fetch_32(&pThread->numOfFds, 1);
+  atomic_sub_fetch_32(&pThread->numOfContexts, 1);
   
   pContext->pThread = 0;
   pContext->state = HTTP_CONTEXT_STATE_CLOSED;
@@ -171,38 +171,39 @@ bool httpInitContext(HttpContext *pContext) {
 
 void httpCloseContextByApp(HttpContext *pContext) {
   pContext->parsed = false;
-
   bool keepAlive = true;
+
   if (pContext->httpVersion == HTTP_VERSION_10 && pContext->httpKeepAlive != HTTP_KEEPALIVE_ENABLE) {
     keepAlive = false;
   } else if (pContext->httpVersion != HTTP_VERSION_10 && pContext->httpKeepAlive == HTTP_KEEPALIVE_DISABLE) {
     keepAlive = false;
-  } else {}
+  } else {
+  }
 
   if (keepAlive) {
     if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) {
-      httpDebug("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse connect",
-              pContext, pContext->fd, pContext->ipstr);
+      httpDebug("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse context", pContext, pContext->fd,
+                pContext->ipstr);
     } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_CLOSED)) {
       httpRemoveContextFromEpoll(pContext);
-      httpDebug("context:%p, fd:%d, ip:%s, last state:dropping, keepAlive:true, close connect",
-              pContext, pContext->fd, pContext->ipstr);
+      httpDebug("context:%p, fd:%d, ip:%s, last state:dropping, keepAlive:true, close connect", pContext, pContext->fd,
+                pContext->ipstr);
     } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) {
-      httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, reuse connect",
-              pContext, pContext->fd, pContext->ipstr);
+      httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, reuse context", pContext, pContext->fd,
+                pContext->ipstr);
     } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
       httpRemoveContextFromEpoll(pContext);
-      httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, close connect",
-                pContext, pContext->fd, pContext->ipstr);
+      httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, close connect", pContext, pContext->fd,
+                pContext->ipstr);
     } else {
       httpRemoveContextFromEpoll(pContext);
-      httpError("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:true, close connect",
-              pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
+      httpError("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:true, close connect", pContext, pContext->fd,
+                pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
     }
   } else {
     httpRemoveContextFromEpoll(pContext);
-    httpDebug("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close connect",
-              pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
+    httpDebug("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close context", pContext, pContext->fd,
+              pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
   }
 
   httpReleaseContext(pContext);
@@ -214,7 +215,7 @@ void httpCloseContextByServer(HttpContext *pContext) {
   } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_DROPPING)) {
     httpDebug("context:%p, fd:%d, ip:%s, epoll already finished, wait app finished", pContext, pContext->fd, pContext->ipstr);
   } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_CLOSED)) {
-    httpDebug("context:%p, fd:%d, ip:%s, epoll finished, close context", pContext, pContext->fd, pContext->ipstr);
+    httpDebug("context:%p, fd:%d, ip:%s, epoll finished, close connect", pContext, pContext->fd, pContext->ipstr);
   } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
     httpDebug("context:%p, fd:%d, ip:%s, epoll finished, will be closed soon", pContext, pContext->fd, pContext->ipstr);
   } else {

src/plugins/http/src/httpHandle.c

@@ -313,9 +313,9 @@ bool httpParseRequest(HttpContext* pContext) {
     return true;
   }
 
-  httpTraceL("context:%p, fd:%d, ip:%s, thread:%s, numOfFds:%d, read size:%d, raw data:\n%s", pContext, pContext->fd,
-            pContext->ipstr, pContext->pThread->label, pContext->pThread->numOfFds, pContext->parser.bufsize,
-            pContext->parser.buffer);
+  httpTraceL("context:%p, fd:%d, ip:%s, thread:%s, numOfContexts:%d, read size:%d, raw data:\n%s", pContext,
+             pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->pThread->numOfContexts,
+             pContext->parser.bufsize, pContext->parser.buffer);
 
   if (!httpGetHttpMethod(pContext)) {
     return false;

src/plugins/http/src/httpServer.c

@@ -293,7 +293,7 @@ static void *httpAcceptHttpConnection(void *arg) {
 
     totalFds = 1;
     for (int i = 0; i < pServer->numOfThreads; ++i) {
-      totalFds += pServer->pThreads[i].numOfFds;
+      totalFds += pServer->pThreads[i].numOfContexts;
     }
 
     if (totalFds > tsHttpCacheSessions * 100) {
@@ -332,9 +332,9 @@ static void *httpAcceptHttpConnection(void *arg) {
     }
 
     // notify the data process, add into the FdObj list
-    atomic_add_fetch_32(&pThread->numOfFds, 1);
-    httpDebug("context:%p, fd:%d, ip:%s, thread:%s numOfFds:%d totalFds:%d, accept a new connection", pContext, connFd,
-              pContext->ipstr, pThread->label, pThread->numOfFds, totalFds);
+    atomic_add_fetch_32(&pThread->numOfContexts, 1);
+    httpDebug("context:%p, fd:%d, ip:%s, thread:%s numOfContexts:%d totalFds:%d, accept a new connection", pContext,
+              connFd, pContext->ipstr, pThread->label, pThread->numOfContexts, totalFds);
 
     // pick up next thread for next connection
     threadId++;

src/plugins/http/src/httpSql.c

@@ -233,10 +233,11 @@ void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int num
   }
 }
 
-void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int code) {
+void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) {
   HttpContext *pContext = (HttpContext *)param;
   if (pContext == NULL) return;
 
+  int32_t code = taos_errno(result);
   HttpEncodeMethod *encode = pContext->encodeMethod;
 
   if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
@@ -260,8 +261,8 @@ void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int code) {
     return;
   }
 
-  int num_fields = taos_field_count(result);
-  if (num_fields == 0) {
+  bool isUpdate = tscIsUpdateQuery(result);
+  if (isUpdate) {
     // not select or show commands
     int affectRows = taos_affected_rows(result);
 

src/query/src/qExecutor.c

@@ -4475,6 +4475,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       }
 
       pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo);
+      taosArrayDestroy(g1);
+      taosArrayDestroy(tx);
 
       SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle);
       assert(taosArrayGetSize(s) >= 1);
@@ -5857,6 +5859,18 @@ _error:
   return code;
 }
 
+static void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) {
+    if (pFilter == NULL) {
+      return;
+    }
+    for (int32_t i = 0; i < numOfFilters; i++) {
+      if (pFilter[i].filterstr) {
+        free((void*)(pFilter[i].pz));
+      }
+    }
+    free(pFilter);
+}
+
 static void freeQInfo(SQInfo *pQInfo) {
   if (!isValidQInfo(pQInfo)) {
     return;
@@ -5925,7 +5939,15 @@ static void freeQInfo(SQInfo *pQInfo) {
 
   tfree(pQuery->tagColList);
   tfree(pQuery->pFilterInfo);
-  tfree(pQuery->colList);
+
+  if (pQuery->colList != NULL) {
+    for (int32_t i = 0; i < pQuery->numOfCols; i++) {
+      SColumnInfo* column = pQuery->colList + i;
+      freeColumnFilterInfo(column->filters, column->numOfFilters);
+    }
+    tfree(pQuery->colList);
+  }
+
   tfree(pQuery->sdata);
 
   tfree(pQuery);
@@ -6122,6 +6144,11 @@ _over:
   free(pExprMsg);
   taosArrayDestroy(pTableIdList);
 
+  for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) {
+    SColumnInfo* column = pQueryMsg->colList + i;
+    freeColumnFilterInfo(column->filters, column->numOfFilters);
+  }
+
   //pQInfo already freed in initQInfo, but *pQInfo may not pointer to null;
   if (code != TSDB_CODE_SUCCESS) {
     *pQInfo = NULL;

src/tsdb/src/tsdbMemTable.c

@@ -119,7 +119,8 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
 
 int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) {
   if (pMemTable == NULL) return 0;
-  T_REF_INC(pMemTable);
+  int ref = T_REF_INC(pMemTable);
+	tsdbDebug("vgId:%d ref memtable %p ref %d", REPO_ID(pRepo), pMemTable, ref);
   return 0;
 }
 
@@ -127,7 +128,9 @@ int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) {
 int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) {
   if (pMemTable == NULL) return 0;
 
-  if (T_REF_DEC(pMemTable) == 0) {
+	int ref = T_REF_DEC(pMemTable);
+	tsdbDebug("vgId:%d unref memtable %p ref %d", REPO_ID(pRepo), pMemTable, ref);
+  if (ref == 0) {
     STsdbCfg *    pCfg = &pRepo->config;
     STsdbBufPool *pBufPool = pRepo->pPool;
 
@@ -167,6 +170,7 @@ int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) {
   tsdbRefMemTable(pRepo, *pIMem);
 
   if (tsdbUnlockRepo(pRepo) < 0) return -1;
+	tsdbDebug("vgId:%d take memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), *pMem, *pIMem);
 
   return 0;
 }

src/util/inc/tfile.h

@@ -18,6 +18,7 @@
 
 #ifdef TAOS_RANDOM_FILE_FAIL
 
+void taosSetRandomFileFailFactor(int factor);
 ssize_t taos_tread(int fd, void *buf, size_t count);
 ssize_t taos_twrite(int fd, void *buf, size_t count);
 off_t taos_lseek(int fd, off_t offset, int whence);

src/util/src/tfile.c

@@ -26,40 +26,51 @@
 
 #include "os.h"
 
-#define RANDOM_FILE_FAIL_FACTOR 5
+#ifdef TAOS_RANDOM_FILE_FAIL
+
+static int random_file_fail_factor = 20;
+
+void taosSetRandomFileFailFactor(int factor)
+{
+  random_file_fail_factor = factor;
+}
+#endif
 
 ssize_t taos_tread(int fd, void *buf, size_t count)
 {
 #ifdef TAOS_RANDOM_FILE_FAIL
-  if (rand() % RANDOM_FILE_FAIL_FACTOR == 0) {
-    errno = EIO;
-    return -1;
+  if (random_file_fail_factor > 0) {
+    if (rand() % random_file_fail_factor == 0) {
+      errno = EIO;
+      return -1;
+    }
   }
 #endif
-
   return tread(fd, buf, count);
 }
 
 ssize_t taos_twrite(int fd, void *buf, size_t count)
 {
 #ifdef TAOS_RANDOM_FILE_FAIL
-  if (rand() % RANDOM_FILE_FAIL_FACTOR == 0) {
-    errno = EIO;
-    return -1;
+  if (random_file_fail_factor > 0) {
+    if (rand() % random_file_fail_factor == 0) {
+      errno = EIO;
+      return -1;
+    }
   }
 #endif
-
   return twrite(fd, buf, count);
 }
 
 off_t taos_lseek(int fd, off_t offset, int whence)
 {
 #ifdef TAOS_RANDOM_FILE_FAIL
-  if (rand() % RANDOM_FILE_FAIL_FACTOR == 0) {
-    errno = EIO;
-    return -1;
+  if (random_file_fail_factor > 0) {
+    if (rand() % random_file_fail_factor == 0) {
+      errno = EIO;
+      return -1;
+    }
   }
 #endif
-
   return lseek(fd, offset, whence);
 }

src/wal/src/walMain.c

@@ -28,6 +28,7 @@
 #include "taoserror.h"
 #include "twal.h"
 #include "tqueue.h"
+#include "tfile.h"
 
 #define walPrefix "wal"
 
@@ -180,7 +181,7 @@ int walWrite(void *handle, SWalHead *pHead) {
   taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead));
   int contLen = pHead->len + sizeof(SWalHead);
 
-  if(write(pWal->fd, pHead, contLen) != contLen) {
+  if(twrite(pWal->fd, pHead, contLen) != contLen) {
     wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
   } else {
@@ -325,7 +326,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
   wDebug("wal:%s, start to restore", name);
 
   while (1) {
-    int ret = read(fd, pHead, sizeof(SWalHead));
+    int ret = tread(fd, pHead, sizeof(SWalHead));
     if ( ret == 0)  break;  
 
     if (ret != sizeof(SWalHead)) {
@@ -340,7 +341,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
       break;
     } 
 
-    ret = read(fd, pHead->cont, pHead->len);
+    ret = tread(fd, pHead->cont, pHead->len);
     if ( ret != pHead->len) {
       wWarn("wal:%s, failed to read body, skip, len:%d ret:%d", name, pHead->len, ret);
       terrno = TAOS_SYSTEM_ERROR(errno);

tests/pytest/crash_gen.sh

@@ -31,11 +31,22 @@ then
     exit -1
 fi
 
+CURR_DIR=`pwd`
+IN_TDINTERNAL="community"
+if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then
+  TAOS_DIR=$CURR_DIR/../../..
+else
+  TAOS_DIR=$CURR_DIR/../..
+fi
+TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1`
+
+LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6|rev`/lib
+
 # First we need to set up a path for Python to find our own TAOS modules, so that "import" can work.
 export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3
 
 # Then let us set up the library path so that our compiled SO file can be loaded by Python
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/../../build/build/lib
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR
 
 # Now we are all let, and let's see if we can find a crash. Note we pass all params
 python3 ./crash_gen.py $@

tests/pytest/fulltest.sh

@@ -121,7 +121,7 @@ python3 ./test.py -f import_merge/importTORestart.py
 python3 ./test.py -f import_merge/importTPORestart.py
 python3 ./test.py -f import_merge/importTRestart.py
 python3 ./test.py -f import_merge/importInsertThenImport.py
-
+python3 ./test.py -f import_merge/importCSV.py
 # user
 python3 ./test.py -f user/user_create.py
 python3 ./test.py -f user/pass_len.py
@@ -156,3 +156,7 @@ python3 ./test.py -f alter/alter_table_crash.py
 
 # client
 python3 ./test.py -f client/client.py
+
+# Misc
+python3 testCompress.py
+python3 testNoCompress.py

tests/pytest/regressiontest.sh

@@ -121,7 +121,7 @@ python3 ./test.py -f import_merge/importTORestart.py
 python3 ./test.py -f import_merge/importTPORestart.py
 python3 ./test.py -f import_merge/importTRestart.py
 python3 ./test.py -f import_merge/importInsertThenImport.py
-
+python3 ./test.py -f import_merge/importCSV.py
 # user
 python3 ./test.py -f user/user_create.py
 python3 ./test.py -f user/pass_len.py
@@ -150,3 +150,7 @@ python3 ./test.py -f alter/alter_table_crash.py
 
 # client
 python3 ./test.py -f client/client.py
+
+# Misc
+python3 testCompress.py
+python3 testNoCompress.py

tests/pytest/util/dnodes-random-fail.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+import os.path
+import subprocess
+from util.log import *
+
+
+class TDSimClient:
+    def __init__(self):
+        self.testCluster = False
+
+        self.cfgDict = {
+            "numOfLogLines": "100000000",
+            "numOfThreadsPerCore": "2.0",
+            "locale": "en_US.UTF-8",
+            "charset": "UTF-8",
+            "asyncLog": "0",
+            "anyIp": "0",
+            "sdbDebugFlag": "135",
+            "rpcDebugFlag": "135",
+            "tmrDebugFlag": "131",
+            "cDebugFlag": "135",
+            "udebugFlag": "135",
+            "jnidebugFlag": "135",
+            "qdebugFlag": "135",
+            }
+
+    def init(self, path):
+        self.__init__()
+        self.path = path
+
+    def getLogDir(self):
+        self.logDir = "%s/sim/psim/log" % (self.path)
+        return self.logDir
+
+    def getCfgDir(self):
+        self.cfgDir = "%s/sim/psim/cfg" % (self.path)
+        return self.cfgDir
+
+    def setTestCluster(self, value):
+        self.testCluster = value
+
+    def addExtraCfg(self, option, value):
+        self.cfgDict.update({option: value})
+
+    def cfg(self, option, value):
+        cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath)
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+    def deploy(self):
+        self.logDir = "%s/sim/psim/log" % (self.path)
+        self.cfgDir = "%s/sim/psim/cfg" % (self.path)
+        self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path)
+
+        cmd = "rm -rf " + self.logDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+    
+        cmd = "mkdir -p " + self.logDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "rm -rf " + self.cfgDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "mkdir -p " + self.cfgDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "touch " + self.cfgPath
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        if self.testCluster:
+            self.cfg("masterIp", "192.168.0.1")
+            self.cfg("secondIp", "192.168.0.2")
+        self.cfg("logDir", self.logDir)
+
+        for key, value in self.cfgDict.items():
+            self.cfg(key, value)
+
+        tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath))
+
+
+class TDDnode:
+    def __init__(self, index):
+        self.index = index
+        self.running = 0
+        self.deployed = 0
+        self.testCluster = False
+        self.valgrind = 0
+
+    def init(self, path):
+        self.path = path
+
+    def setTestCluster(self, value):
+        self.testCluster = value
+
+    def setValgrind(self, value):
+        self.valgrind = value
+
+    def getDataSize(self):
+        totalSize = 0
+
+        if (self.deployed == 1):
+            for dirpath, dirnames, filenames in os.walk(self.dataDir):
+                for f in filenames:
+                    fp = os.path.join(dirpath, f)
+
+                    if not os.path.islink(fp):
+                        totalSize = totalSize + os.path.getsize(fp)
+
+        return totalSize
+
+    def deploy(self):
+        self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index)
+        self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index)
+        self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index)
+        self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % (
+            self.path, self.index)
+
+        cmd = "rm -rf " + self.dataDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "rm -rf " + self.logDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "rm -rf " + self.cfgDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "mkdir -p " + self.dataDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "mkdir -p " + self.logDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "mkdir -p " + self.cfgDir
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        cmd = "touch " + self.cfgPath
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+        if self.testCluster:
+            self.startIP()
+
+        if self.testCluster:
+            self.cfg("masterIp", "192.168.0.1")
+            self.cfg("secondIp", "192.168.0.2")
+            self.cfg("publicIp", "192.168.0.%d" % (self.index))
+            self.cfg("internalIp", "192.168.0.%d" % (self.index))
+            self.cfg("privateIp", "192.168.0.%d" % (self.index))
+        self.cfg("dataDir", self.dataDir)
+        self.cfg("logDir", self.logDir)
+        self.cfg("numOfLogLines", "100000000")
+        self.cfg("mnodeEqualVnodeNum", "0")
+        self.cfg("walLevel", "1")
+        self.cfg("statusInterval", "1")
+        self.cfg("numOfTotalVnodes", "64")
+        self.cfg("numOfMnodes", "3")
+        self.cfg("numOfThreadsPerCore", "2.0")
+        self.cfg("monitor", "0")
+        self.cfg("maxVnodeConnections", "30000")
+        self.cfg("maxMgmtConnections", "30000")
+        self.cfg("maxMeterConnections", "30000")
+        self.cfg("maxShellConns", "30000")
+        self.cfg("locale", "en_US.UTF-8")
+        self.cfg("charset", "UTF-8")
+        self.cfg("asyncLog", "0")
+        self.cfg("anyIp", "0")
+        self.cfg("dDebugFlag", "135")
+        self.cfg("mDebugFlag", "135")
+        self.cfg("sdbDebugFlag", "135")
+        self.cfg("rpcDebugFlag", "135")
+        self.cfg("tmrDebugFlag", "131")
+        self.cfg("cDebugFlag", "135")
+        self.cfg("httpDebugFlag", "135")
+        self.cfg("monitorDebugFlag", "135")
+        self.cfg("udebugFlag", "135")
+        self.cfg("jnidebugFlag", "135")
+        self.cfg("qdebugFlag", "135")
+        self.deployed = 1
+        tdLog.debug(
+            "dnode:%d is deployed and configured by %s" %
+            (self.index, self.cfgPath))
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
+
+    def start(self):
+        buildPath = self.getBuildPath()
+
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+
+        binPath = buildPath + "/build/bin/taosd"
+
+        if self.deployed == 0:
+            tdLog.exit("dnode:%d is not deployed" % (self.index))
+
+        if self.valgrind == 0:
+            cmd = "nohup %s -c %s > /dev/null 2>&1 & " % (
+                binPath, self.cfgDir)
+        else:
+            valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes"
+
+            cmd = "nohup %s %s -c %s --random-file-fail-factor 5 2>&1 & " % (
+                valgrindCmdline, binPath, self.cfgDir)
+
+            print(cmd)
+
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+        self.running = 1
+        tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
+
+        tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index))
+        time.sleep(5)
+
+    def stop(self):
+        if self.valgrind == 0:
+            toBeKilled = "taosd"
+        else:
+            toBeKilled = "valgrind.bin"
+
+        if self.running != 0:
+            psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
+            processID = subprocess.check_output(
+                psCmd, shell=True).decode("utf-8")
+
+            while(processID):
+                killCmd = "kill -INT %s > /dev/null 2>&1" % processID
+                os.system(killCmd)
+                time.sleep(1)
+                processID = subprocess.check_output(
+                    psCmd, shell=True).decode("utf-8")
+            for port in range(6030, 6041):
+                fuserCmd = "fuser -k -n tcp %d" % port
+                os.system(fuserCmd)
+            if self.valgrind:
+                time.sleep(2)
+
+            self.running = 0
+            tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index))
+
+    def forcestop(self):
+        if self.valgrind == 0:
+            toBeKilled = "taosd"
+        else:
+            toBeKilled = "valgrind.bin"
+
+        if self.running != 0:
+            psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
+            processID = subprocess.check_output(
+                psCmd, shell=True).decode("utf-8")
+
+            while(processID):
+                killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
+                os.system(killCmd)
+                time.sleep(1)
+                processID = subprocess.check_output(
+                    psCmd, shell=True).decode("utf-8")
+            for port in range(6030, 6041):
+                fuserCmd = "fuser -k -n tcp %d" % port
+                os.system(fuserCmd)
+            if self.valgrind:
+                time.sleep(2)
+
+            self.running = 0
+            tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index))
+
+    def startIP(self):
+        cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index)
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+    def stopIP(self):
+        cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % (
+            self.index, self.index)
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+    def cfg(self, option, value):
+        cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath)
+        if os.system(cmd) != 0:
+            tdLog.exit(cmd)
+
+    def getDnodeRootDir(self, index):
+        dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index)
+        return dnodeRootDir
+
+    def getDnodesRootDir(self):
+        dnodesRootDir = "%s/sim/psim" % (self.path)
+        return dnodesRootDir
+
+
+class TDDnodes:
+    def __init__(self):
+        self.dnodes = []
+        self.dnodes.append(TDDnode(1))
+        self.dnodes.append(TDDnode(2))
+        self.dnodes.append(TDDnode(3))
+        self.dnodes.append(TDDnode(4))
+        self.dnodes.append(TDDnode(5))
+        self.dnodes.append(TDDnode(6))
+        self.dnodes.append(TDDnode(7))
+        self.dnodes.append(TDDnode(8))
+        self.dnodes.append(TDDnode(9))
+        self.dnodes.append(TDDnode(10))
+        self.simDeployed = False
+
+    def init(self, path):
+        psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
+        processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
+        while(processID):
+            killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(
+                psCmd, shell=True).decode("utf-8")
+
+        psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
+        processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
+        while(processID):
+            killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(
+                psCmd, shell=True).decode("utf-8")
+
+        binPath = os.path.dirname(os.path.realpath(__file__))
+        binPath = binPath + "/../../../debug/"
+        tdLog.debug("binPath %s" % (binPath))
+        binPath = os.path.realpath(binPath)
+        tdLog.debug("binPath real path %s" % (binPath))
+
+        # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath)
+        # tdLog.debug(cmd)
+        # os.system(cmd)
+
+        # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath)
+        # if os.system(cmd) != 0 :
+        #  tdLog.exit(cmd)
+        # tdLog.debug("execute %s" % (cmd))
+
+        # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath)
+        # if os.system(cmd) != 0 :
+        # tdLog.exit(cmd)
+        # tdLog.debug("execute %s" % (cmd))
+
+        if path == "":
+            # self.path = os.path.expanduser('~')
+            self.path = os.path.abspath(binPath + "../../")
+        else:
+            self.path = os.path.realpath(path)
+
+        for i in range(len(self.dnodes)):
+            self.dnodes[i].init(self.path)
+
+        self.sim = TDSimClient()
+        self.sim.init(self.path)
+
+    def setTestCluster(self, value):
+        self.testCluster = value
+
+    def setValgrind(self, value):
+        self.valgrind = value
+
+    def deploy(self, index):
+        self.sim.setTestCluster(self.testCluster)
+
+        if (self.simDeployed == False):
+            self.sim.deploy()
+            self.simDeployed = True
+
+        self.check(index)
+        self.dnodes[index - 1].setTestCluster(self.testCluster)
+        self.dnodes[index - 1].setValgrind(self.valgrind)
+        self.dnodes[index - 1].deploy()
+
+    def cfg(self, index, option, value):
+        self.check(index)
+        self.dnodes[index - 1].cfg(option, value)
+
+    def start(self, index):
+        self.check(index)
+        self.dnodes[index - 1].start()
+
+    def stop(self, index):
+        self.check(index)
+        self.dnodes[index - 1].stop()
+
+    def getDataSize(self, index):
+        self.check(index)
+        return self.dnodes[index - 1].getDataSize()
+
+    def forcestop(self, index):
+        self.check(index)
+        self.dnodes[index - 1].forcestop()
+
+    def startIP(self, index):
+        self.check(index)
+
+        if self.testCluster:
+            self.dnodes[index - 1].startIP()
+
+    def stopIP(self, index):
+        self.check(index)
+
+        if self.dnodes[index - 1].testCluster:
+            self.dnodes[index - 1].stopIP()
+
+    def check(self, index):
+        if index < 1 or index > 10:
+            tdLog.exit("index:%d should on a scale of [1, 10]" % (index))
+
+    def stopAll(self):
+        tdLog.info("stop all dnodes")
+        for i in range(len(self.dnodes)):
+            self.dnodes[i].stop()
+
+        psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'"
+        processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
+        if processID:
+            cmd = "sudo systemctl stop taosd"
+            os.system(cmd)
+        # if os.system(cmd) != 0 :
+        # tdLog.exit(cmd)
+        psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"
+        processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
+        while(processID):
+            killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(
+                psCmd, shell=True).decode("utf-8")
+
+        psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
+        processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
+        while(processID):
+            killCmd = "kill -KILL %s > /dev/null 2>&1" % processID
+            os.system(killCmd)
+            time.sleep(1)
+            processID = subprocess.check_output(
+                psCmd, shell=True).decode("utf-8")
+
+        # if os.system(cmd) != 0 :
+        # tdLog.exit(cmd)
+
+    def getDnodesRootDir(self):
+        dnodesRootDir = "%s/sim" % (self.path)
+        return dnodesRootDir
+
+    def getSimCfgPath(self):
+        return self.sim.getCfgDir()
+
+    def getSimLogPath(self):
+        return self.sim.getLogDir()
+
+    def addSimExtraCfg(self, option, value):
+        self.sim.addExtraCfg(option, value)
+
+
+tdDnodes = TDDnodes()

tests/script/jenkins/basic.txt

@@ -117,8 +117,6 @@ cd ../../../debug; make
 ./test.sh -f general/parser/import_commit3.sim
 ./test.sh -f general/parser/insert_tb.sim
 ./test.sh -f general/parser/first_last.sim
-# dyh is processing this script
-#./test.sh -f general/parser/import_file.sim
 ./test.sh -f general/parser/lastrow.sim
 ./test.sh -f general/parser/nchar.sim
 ./test.sh -f general/parser/null_char.sim
@@ -145,7 +143,6 @@ cd ../../../debug; make
 ./test.sh -f general/parser/groupby.sim
 ./test.sh -f general/parser/set_tag_vals.sim
 #./test.sh -f general/parser/sliding.sim
-./test.sh -f general/parser/tags_dynamically_specifiy.sim
 ./test.sh -f general/parser/tags_filter.sim
 ./test.sh -f general/parser/slimit_alter_tags.sim
 ./test.sh -f general/parser/join.sim

tests/script/sh/deploy.sh

@@ -125,7 +125,6 @@ echo "mqttDebugFlag       131"            >> $TAOS_CFG
 echo "qdebugFlag          135"            >> $TAOS_CFG
 echo "rpcDebugFlag        135"            >> $TAOS_CFG
 echo "tmrDebugFlag        131"            >> $TAOS_CFG
-echo "cDebugFlag          135"            >> $TAOS_CFG
 echo "udebugFlag          135"            >> $TAOS_CFG
 echo "sdebugFlag          135"            >> $TAOS_CFG
 echo "wdebugFlag          135"            >> $TAOS_CFG

tests/script/sh/exec-random-fail.sh

+#!/bin/bash
+
+# if [ $# != 4 || $# != 5 ]; then 
+  # echo "argument list need input : "
+  # echo "  -n nodeName"
+  # echo "  -s start/stop"
+  # echo "  -c clear"
+  # exit 1
+# fi
+
+NODE_NAME=
+EXEC_OPTON=
+CLEAR_OPTION="false"
+while getopts "n:s:u:x:ct" arg 
+do
+  case $arg in
+    n)
+      NODE_NAME=$OPTARG
+      ;;
+    s)
+      EXEC_OPTON=$OPTARG
+      ;;
+    c)
+      CLEAR_OPTION="clear"
+      ;;
+    t)
+      SHELL_OPTION="true"
+      ;;
+    u)
+      USERS=$OPTARG
+      ;;
+    x)
+      SIGNAL=$OPTARG
+      ;;
+    ?)
+      echo "unkown argument"
+      ;;
+  esac
+done
+
+SCRIPT_DIR=`dirname $0`
+cd $SCRIPT_DIR/../
+SCRIPT_DIR=`pwd`
+
+IN_TDINTERNAL="community"
+if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then
+  cd ../../..
+else
+  cd ../../
+fi
+
+TAOS_DIR=`pwd`
+TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1`
+
+if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then
+  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3`
+else
+  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2`
+fi
+
+BUILD_DIR=$TAOS_DIR/$BIN_DIR/build
+
+SIM_DIR=$TAOS_DIR/sim
+NODE_DIR=$SIM_DIR/$NODE_NAME
+EXE_DIR=$BUILD_DIR/bin
+CFG_DIR=$NODE_DIR/cfg
+LOG_DIR=$NODE_DIR/log
+DATA_DIR=$NODE_DIR/data
+MGMT_DIR=$NODE_DIR/data/mgmt
+TSDB_DIR=$NODE_DIR/data/tsdb
+
+TAOS_CFG=$NODE_DIR/cfg/taos.cfg
+
+echo ------------ $EXEC_OPTON $NODE_NAME
+
+TAOS_FLAG=$SIM_DIR/tsim/flag
+if [ -f "$TAOS_FLAG" ]; then 
+  EXE_DIR=/usr/local/bin/taos
+fi
+
+if [ "$CLEAR_OPTION" = "clear" ]; then 
+  echo rm -rf $MGMT_DIR $TSDB_DIR  
+  rm -rf $TSDB_DIR
+  rm -rf $MGMT_DIR
+fi
+
+if [ "$EXEC_OPTON" = "start" ]; then 
+  echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR
+  
+  if [ "$SHELL_OPTION" = "true" ]; then 
+    nohup valgrind --log-file=${LOG_DIR}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no  --track-origins=yes --show-leak-kinds=all  -v  --workaround-gcc296-bugs=yes   $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 &   
+  else
+    nohup $EXE_DIR/taosd -c $CFG_DIR --random-file-fail-factor 5 > /dev/null 2>&1 & 
+  fi
+  
+else
+  #relative path
+  RCFG_DIR=sim/$NODE_NAME/cfg
+  PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'`
+  while [ -n "$PID" ]
+  do
+    if [ "$SIGNAL" = "SIGINT" ]; then 
+      echo try to kill by signal SIGINT
+      kill -SIGINT $PID
+    else
+      echo try to kill by signal SIGKILL
+      kill -9 $PID
+    fi
+    sleep 1
+    PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'`
+  done 
+fi
+