Merge branch 'develop' of https://github.com/taosdata/TDengine into develop

Jenkinsfile

@@ -42,12 +42,12 @@ def pre_test(){
     killall -9 taosd ||echo "no taosd running"
     killall -9 gdb || echo "no gdb running"
     cd ${WKC}
-    git checkout develop
     git reset --hard HEAD~10 >/dev/null 
+    git checkout develop
     git pull >/dev/null
     git fetch origin +refs/pull/${CHANGE_ID}/merge
     git checkout -qf FETCH_HEAD
-    find ${WKC}/tests/pytest -name \'*\'.sql -exec rm -rf {} \\;
+    git clean -dfx
     cd ${WK}
     git reset --hard HEAD~10
     git checkout develop 
@@ -55,7 +55,7 @@ def pre_test(){
     cd ${WK}
     export TZ=Asia/Harbin
     date
-    rm -rf ${WK}/debug
+    git clean -dfx
     mkdir debug
     cd debug
     cmake .. > /dev/null

cmake/install.inc

@@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS)
   #INSTALL(TARGETS taos RUNTIME DESTINATION driver)
   #INSTALL(TARGETS shell RUNTIME DESTINATION .)
   IF (TD_MVN_INSTALLED)
-    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.24-dist.jar DESTINATION connector/jdbc)
+    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.25-dist.jar DESTINATION connector/jdbc)
   ENDIF ()
 ELSEIF (TD_DARWIN)
   SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")

documentation20/cn/12.taos-sql/docs.md

@@ -29,21 +29,21 @@ taos> DESCRIBE meters;
 
 ## <a class="anchor" id="data-type"></a>支持的数据类型
 
-使用TDengine，最重要的是时间戳。创建并插入记录、查询历史记录的时候，均需要指定时间戳。时间戳有如下规则：
+使用 TDengine，最重要的是时间戳。创建并插入记录、查询历史记录的时候，均需要指定时间戳。时间戳有如下规则：
 
-- 时间格式为```YYYY-MM-DD HH:mm:ss.MS```, 默认时间分辨率为毫秒。比如：```2017-08-12 18:25:58.128```
-- 内部函数now是服务器的当前时间
-- 插入记录时，如果时间戳为now，插入数据时使用服务器当前时间
-- Epoch Time: 时间戳也可以是一个长整数，表示从1970-01-01 08:00:00.000开始的毫秒数
-- 时间可以加减，比如 now-2h，表明查询时刻向前推2个小时(最近2小时)。 数字后面的时间单位可以是 a(毫秒)、s(秒)、 m(分)、h(小时)、d(天)、w(周)。 比如select * from t1 where ts > now-2w and ts <= now-1w, 表示查询两周前整整一周的数据。 在指定降频操作(down sampling)的时间窗口(interval)时，时间单位还可以使用 n(自然月) 和 y(自然年)。
+- 时间格式为 ```YYYY-MM-DD HH:mm:ss.MS```，默认时间分辨率为毫秒。比如：```2017-08-12 18:25:58.128```
+- 内部函数 now 是客户端的当前时间
+- 插入记录时，如果时间戳为 now，插入数据时使用提交这条记录的客户端的当前时间
+- Epoch Time：时间戳也可以是一个长整数，表示从 1970-01-01 08:00:00.000 开始的毫秒数
+- 时间可以加减，比如 now-2h，表明查询时刻向前推 2 个小时（最近 2 小时）。数字后面的时间单位可以是 u(微秒)、a(毫秒)、s(秒)、m(分)、h(小时)、d(天)、w(周)。 比如 `select * from t1 where ts > now-2w and ts <= now-1w`，表示查询两周前整整一周的数据。在指定降频操作（down sampling）的时间窗口（interval）时，时间单位还可以使用 n(自然月) 和 y(自然年)。
 
-TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMicrosecond就可支持微秒。
+TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableMicrosecond 就可以支持微秒。
 
-在TDengine中，普通表的数据模型中可使用以下10种数据类型。 
+在TDengine中，普通表的数据模型中可使用以下 10 种数据类型。 
 
 |      |   类型    | Bytes  | 说明                                                         |
 | ---- | :-------: | ------ | ------------------------------------------------------------ |
-| 1    | TIMESTAMP | 8      | 时间戳。缺省精度毫秒，可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始，计时不能早于该时间。 |
+| 1    | TIMESTAMP | 8      | 时间戳。缺省精度毫秒，可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始，计时不能早于该时间。（从 2.0.18 版本开始，已经去除了这一时间范围限制） |
 | 2    |    INT    | 4      | 整型，范围 [-2^31+1,   2^31-1], -2^31 用作 NULL           |
 | 3    |  BIGINT   | 8      | 长整型，范围 [-2^63+1,   2^63-1], -2^63 用于 NULL                                 |
 | 4    |   FLOAT   | 4      | 浮点型，有效位数 6-7，范围 [-3.4E38, 3.4E38]                  |

packaging/tools/remove.sh

@@ -120,7 +120,7 @@ function clean_service_on_systemd() {
   
     if [ "$verMode" == "cluster" ]; then
 		  nginx_service_config="${service_config_dir}/${nginx_service_name}.service"	
-   	 	if [ -d ${bin_dir}/web ]; then
+   	 	if [ -d ${install_nginxd_dir} ]; then
    	    if systemctl is-active --quiet ${nginx_service_name}; then
    	      echo "Nginx for TDengine is running, stopping it..."
    	      ${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null

src/connector/jdbc/CMakeLists.txt

@@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED)
   ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME}
     POST_BUILD
     COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
-    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.24-dist.jar ${LIBRARY_OUTPUT_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.25-dist.jar ${LIBRARY_OUTPUT_PATH}
     COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
     COMMENT "build jdbc driver")
   ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME})

src/connector/jdbc/deploy-pom.xml

@@ -5,7 +5,7 @@
 
   <groupId>com.taosdata.jdbc</groupId>
   <artifactId>taos-jdbcdriver</artifactId>
-  <version>2.0.24</version>
+  <version>2.0.25</version>
   <packaging>jar</packaging>
 
   <name>JDBCDriver</name>

src/connector/jdbc/pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.taosdata.jdbc</groupId>
     <artifactId>taos-jdbcdriver</artifactId>
-    <version>2.0.24</version>
+    <version>2.0.25</version>
     <packaging>jar</packaging>
     <name>JDBCDriver</name>
     <url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>
@@ -62,6 +62,14 @@
     </dependencies>
 
     <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <excludes>
+                    <exclude>**/*.md</exclude>
+                </excludes>
+            </resource>
+        </resources>
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>

src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java

@@ -15,7 +15,7 @@ public abstract class AbstractParameterMetaData extends WrapperImpl implements P
 
     @Override
     public int getParameterCount() throws SQLException {
-        return parameters.length;
+        return parameters == null ? 0 : parameters.length;
     }
 
     @Override

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java

@@ -29,45 +29,25 @@ public class TSDBJNIConnector {
     private static volatile Boolean isInitialized = false;
 
     private TaosInfo taosInfo = TaosInfo.getInstance();
+    // Connection pointer used in C
+    private long taos = TSDBConstants.JNI_NULL_POINTER;
+    // result set status in current connection
+    private boolean isResultsetClosed = true;
+    private int affectedRows = -1;
 
     static {
         System.loadLibrary("taos");
         System.out.println("java.library.path:" + System.getProperty("java.library.path"));
     }
 
-    /**
-     * Connection pointer used in C
-     */
-    private long taos = TSDBConstants.JNI_NULL_POINTER;
-
-    /**
-     * Result set pointer for the current connection
-     */
-//    private long taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER;
-
-    /**
-     * result set status in current connection
-     */
-    private boolean isResultsetClosed = true;
-    private int affectedRows = -1;
-
-    /**
-     * Whether the connection is closed
-     */
     public boolean isClosed() {
         return this.taos == TSDBConstants.JNI_NULL_POINTER;
     }
 
-    /**
-     * Returns the status of last result set in current connection
-     */
     public boolean isResultsetClosed() {
         return this.isResultsetClosed;
     }
 
-    /**
-     * Initialize static variables in JNI to optimize performance
-     */
     public static void init(String configDir, String locale, String charset, String timezone) throws SQLWarning {
         synchronized (isInitialized) {
             if (!isInitialized) {
@@ -93,11 +73,6 @@ public class TSDBJNIConnector {
 
     public static native String getTsCharset();
 
-    /**
-     * Get connection pointer
-     *
-     * @throws SQLException
-     */
     public boolean connect(String host, int port, String dbName, String user, String password) throws SQLException {
         if (this.taos != TSDBConstants.JNI_NULL_POINTER) {
 //            this.closeConnectionImp(this.taos);
@@ -185,13 +160,6 @@ public class TSDBJNIConnector {
 
     private native String getErrMsgImp(long pSql);
 
-    /**
-     * Get resultset pointer
-     * Each connection should have a single open result set at a time
-     */
-//    public long getResultSet() {
-//        return taosResultSetPointer;
-//    }
     private native long getResultSetImp(long connection, long pSql);
 
     public boolean isUpdateQuery(long pSql) {
@@ -231,6 +199,7 @@ public class TSDBJNIConnector {
 //        }
 //        return resCode;
 //    }
+
     private native int freeResultSetImp(long connection, long result);
 
     /**
@@ -323,8 +292,7 @@ public class TSDBJNIConnector {
      * Validate if a <I>create table</I> sql statement is correct without actually creating that table
      */
     public boolean validateCreateTableSql(String sql) {
-        long connection = taos;
-        int res = validateCreateTableSqlImp(connection, sql.getBytes());
+        int res = validateCreateTableSqlImp(taos, sql.getBytes());
         return res != 0 ? false : true;
     }
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java

@@ -84,9 +84,9 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet {
             case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP:
                 return new Timestamp(row.getDate(colIndex).getTime());
             case TSDBConstants.TSDB_DATA_TYPE_BINARY:
-                return row.getString(colIndex).getBytes();
+                return row.getString(colIndex) == null ? null : row.getString(colIndex).getBytes();
             case TSDBConstants.TSDB_DATA_TYPE_NCHAR:
-                return row.getString(colIndex);
+                return row.getString(colIndex) == null ? null : row.getString(colIndex);
             default:
                 return row.get(colIndex);
         }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java

+package com.taosdata.jdbc.utils;
+
+public class OSUtils {
+    private static final String OS = System.getProperty("os.name").toLowerCase();
+
+    public static boolean isWindows() {
+        return OS.indexOf("win") >= 0;
+    }
+
+    public static boolean isMac() {
+        return OS.indexOf("mac") >= 0;
+    }
+
+    public static boolean isLinux() {
+        return OS.indexOf("nux") >= 0;
+    }
+}

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java

@@ -17,7 +17,6 @@ public class DriverAutoloadTest {
 
     @Test
     public void testRestful() throws SQLException {
-//        Class.forName("com.taosdata.jdbc.rs.RestfulDriver");
         final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata";
         Connection conn = DriverManager.getConnection(url, properties);
         Assert.assertNotNull(conn);

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcRestfulTest.java

+package com.taosdata.jdbc.cases;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.*;
+
+public class NullValueInResultSetForJdbcRestfulTest {
+
+    private static final String host = "127.0.0.1";
+    Connection conn;
+
+    @Test
+    public void test() {
+        try (Statement stmt = conn.createStatement()) {
+            ResultSet rs = stmt.executeQuery("select * from weather");
+            ResultSetMetaData meta = rs.getMetaData();
+            while (rs.next()) {
+                for (int i = 1; i <= meta.getColumnCount(); i++) {
+                    Object value = rs.getObject(i);
+                    System.out.print(meta.getColumnLabel(i) + ": " + value + "\t");
+                }
+                System.out.println();
+            }
+
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Before
+    public void before() throws SQLException {
+        final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata";
+        conn = DriverManager.getConnection(url);
+        try (Statement stmt = conn.createStatement()) {
+            stmt.execute("drop database if exists test_null");
+            stmt.execute("create database if not exists test_null");
+            stmt.execute("use test_null");
+            stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64))");
+            stmt.executeUpdate("insert into weather(ts, f1) values(now+1s, 1)");
+            stmt.executeUpdate("insert into weather(ts, f2) values(now+2s, 2)");
+            stmt.executeUpdate("insert into weather(ts, f3) values(now+3s, 3.0)");
+            stmt.executeUpdate("insert into weather(ts, f4) values(now+4s, 4.0)");
+            stmt.executeUpdate("insert into weather(ts, f5) values(now+5s, 5)");
+            stmt.executeUpdate("insert into weather(ts, f6) values(now+6s, 6)");
+            stmt.executeUpdate("insert into weather(ts, f7) values(now+7s, true)");
+            stmt.executeUpdate("insert into weather(ts, f8) values(now+8s, 'hello')");
+            stmt.executeUpdate("insert into weather(ts, f9) values(now+9s, '涛思数据')");
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @After
+    public void after() {
+        try {
+            if (conn != null)
+                conn.close();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+}

src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java

+package com.taosdata.jdbc.utils;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class OSUtilsTest {
+
+    private String OS;
+
+    @Test
+    public void inWindows() {
+        Assert.assertEquals(OS.indexOf("win") >= 0, OSUtils.isWindows());
+    }
+
+    @Test
+    public void isMac() {
+        Assert.assertEquals(OS.indexOf("mac") >= 0, OSUtils.isMac());
+    }
+
+    @Test
+    public void isLinux() {
+        Assert.assertEquals(OS.indexOf("nux") >= 0, OSUtils.isLinux());
+    }
+
+    @Before
+    public void before() {
+        OS = System.getProperty("os.name").toLowerCase();
+    }
+}
\ No newline at end of file

src/dnode/src/dnodeMain.c

@@ -237,6 +237,18 @@ static int32_t dnodeInitStorage() {
     return -1;
   }
 
+  TDIR *tdir = tfsOpendir("vnode_bak/.staging");
+  if (tfsReaddir(tdir) != NULL) {
+    dError("vnode_bak/.staging dir not empty, fix it first.");
+    tfsClosedir(tdir);
+    return -1;
+  }
+
+  if (tfsMkdir("vnode_bak/.staging") < 0) {
+    dError("failed to create vnode_bak/.staging dir since %s", tstrerror(terrno));
+    return -1;
+  }
+
   dnodeCheckDataDirOpenned(tsDnodeDir);
 
   dInfo("dnode storage is initialized at %s", tsDnodeDir);

src/dnode/src/dnodeVWrite.c

@@ -205,7 +205,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) {
              pWrite->rpcMsg.ahandle, taosMsg[pWrite->pHead.msgType], qtypeStr[qtype], pWrite->pHead.version);
 
       pWrite->code = vnodeProcessWrite(pVnode, &pWrite->pHead, qtype, pWrite);
-      if (pWrite->code <= 0) pWrite->processedCount = 1;
+      if (pWrite->code <= 0) atomic_add_fetch_32(&pWrite->processedCount, 1);
       if (pWrite->code > 0) pWrite->code = 0;
       if (pWrite->code == 0 && pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true;
 

src/inc/taosdef.h

@@ -259,7 +259,7 @@ do { \
 #define TSDB_MIN_TABLES                 4
 #define TSDB_MAX_TABLES                 10000000
 #define TSDB_DEFAULT_TABLES             1000000
-#define TSDB_TABLES_STEP                100
+#define TSDB_TABLES_STEP                1000
 
 #define TSDB_MIN_DAYS_PER_FILE          1
 #define TSDB_MAX_DAYS_PER_FILE          3650 

src/inc/tsync.h

@@ -79,9 +79,6 @@ typedef void     (*FStopSyncFile)(int32_t vgId, uint64_t fversion);
 // get file version
 typedef int32_t  (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver);
 
-// reset version
-typedef int32_t  (*FResetVersion)(int32_t vgId, uint64_t fver);
-
 typedef int32_t  (*FSendFile)(void *tsdb, SOCKET socketFd);
 typedef int32_t  (*FRecvFile)(void *tsdb, SOCKET socketFd);
 
@@ -99,7 +96,6 @@ typedef struct {
   FStartSyncFile    startSyncFileFp;
   FStopSyncFile     stopSyncFileFp;
   FGetVersion       getVersionFp;
-  FResetVersion     resetVersionFp;
   FSendFile         sendFileFp;
   FRecvFile         recvFileFp;
 } SSyncInfo;

src/sync/inc/syncInt.h

@@ -117,7 +117,6 @@ typedef struct SSyncNode {
   FStartSyncFile    startSyncFileFp;
   FStopSyncFile     stopSyncFileFp;
   FGetVersion       getVersionFp;
-  FResetVersion     resetVersionFp;
   FSendFile         sendFileFp;
   FRecvFile         recvFileFp;
   pthread_mutex_t   mutex;

src/sync/src/syncMain.c

@@ -182,7 +182,6 @@ int64_t syncStart(const SSyncInfo *pInfo) {
   pNode->startSyncFileFp = pInfo->startSyncFileFp;
   pNode->stopSyncFileFp = pInfo->stopSyncFileFp;
   pNode->getVersionFp = pInfo->getVersionFp;
-  pNode->resetVersionFp = pInfo->resetVersionFp;
   pNode->sendFileFp = pInfo->sendFileFp;
   pNode->recvFileFp = pInfo->recvFileFp;
 

src/sync/src/syncRestore.c

@@ -238,7 +238,6 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
 
   (*pNode->stopSyncFileFp)(pNode->vgId, fversion);
   nodeVersion = fversion;
-  if (pNode->resetVersionFp) (*pNode->resetVersionFp)(pNode->vgId, fversion);
 
   sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion);
   uint64_t wver = 0;

src/vnode/inc/vnodeBackup.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_VNODE_BACKUP_H
+#define TDENGINE_VNODE_BACKUP_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "vnodeInt.h"
+
+int32_t vnodeInitBackup();
+void    vnodeCleanupBackup();
+int32_t vnodeBackup(int32_t vgId);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/vnode/inc/vnodeSync.h

@@ -30,7 +30,6 @@ void     vnodeStopSyncFile(int32_t vgId, uint64_t fversion);
 void     vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code);
 int32_t  vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam);
 int32_t  vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver);
-int32_t  vnodeResetVersion(int32_t vgId, uint64_t fver);
 
 void     vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code, bool force);
 

src/vnode/src/vnodeBackup.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "taosmsg.h"
+#include "tutil.h"
+#include "tqueue.h"
+#include "tglobal.h"
+#include "tfs.h"
+#include "vnodeBackup.h"
+#include "vnodeMain.h"
+
+typedef struct {
+  int32_t vgId;
+} SVBackupMsg;
+
+typedef struct {
+  pthread_t thread;
+  int32_t   workerId;
+} SVBackupWorker;
+
+typedef struct {
+  int32_t    num;
+  SVBackupWorker *worker;
+} SVBackupWorkerPool;
+
+static SVBackupWorkerPool tsVBackupPool;
+static taos_qset     tsVBackupQset;
+static taos_queue    tsVBackupQueue;
+
+static void vnodeProcessBackupMsg(SVBackupMsg *pMsg) {
+  int32_t vgId = pMsg->vgId;
+  char newDir[TSDB_FILENAME_LEN] = {0};
+  char stagingDir[TSDB_FILENAME_LEN] = {0};
+
+  sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId);
+  sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId);
+
+  if (tsEnableVnodeBak) {
+    tfsRmdir(newDir);
+    tfsRename(stagingDir, newDir);
+  } else {
+    vInfo("vgId:%d, vnode backup not enabled", vgId);
+
+    tfsRmdir(stagingDir);
+  }
+}
+
+static void *vnodeBackupFunc(void *param) {
+  while (1) {
+    SVBackupMsg *pMsg = NULL;
+    if (taosReadQitemFromQset(tsVBackupQset, NULL, (void **)&pMsg, NULL) == 0) {
+      vDebug("qset:%p, vbackup got no message from qset, exiting", tsVBackupQset);
+      break;
+    }
+
+    vTrace("vgId:%d, will be processed in vbackup queue", pMsg->vgId);
+    vnodeProcessBackupMsg(pMsg);
+
+    vTrace("vgId:%d, disposed in vbackup worker", pMsg->vgId);
+    taosFreeQitem(pMsg);
+  }
+
+  return NULL;
+}
+
+static int32_t vnodeStartBackup() {
+  tsVBackupQueue = taosOpenQueue();
+  if (tsVBackupQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
+
+  taosAddIntoQset(tsVBackupQset, tsVBackupQueue, NULL);
+
+  for (int32_t i = 0; i < tsVBackupPool.num; ++i) {
+    SVBackupWorker *pWorker = tsVBackupPool.worker + i;
+    pWorker->workerId = i;
+
+    pthread_attr_t thAttr;
+    pthread_attr_init(&thAttr);
+    pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
+
+    if (pthread_create(&pWorker->thread, &thAttr, vnodeBackupFunc, pWorker) != 0) {
+      vError("failed to create thread to process vbackup queue, reason:%s", strerror(errno));
+    }
+
+    pthread_attr_destroy(&thAttr);
+
+    vDebug("vbackup:%d is launched, total:%d", pWorker->workerId, tsVBackupPool.num);
+  }
+
+  vDebug("vbackup queue:%p is allocated", tsVBackupQueue);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t vnodeWriteIntoBackupWorker(int32_t vgId) {
+  SVBackupMsg *pMsg = taosAllocateQitem(sizeof(SVBackupMsg));
+  if (pMsg == NULL) return TSDB_CODE_VND_OUT_OF_MEMORY;
+
+  pMsg->vgId = vgId;
+
+  int32_t code = taosWriteQitem(tsVBackupQueue, TAOS_QTYPE_RPC, pMsg);
+  if (code == 0) code = TSDB_CODE_DND_ACTION_IN_PROGRESS;
+
+  return code;
+}
+
+int32_t vnodeBackup(int32_t vgId) {
+  vTrace("vgId:%d, will backup", vgId);
+  return vnodeWriteIntoBackupWorker(vgId);
+}
+
+int32_t vnodeInitBackup() {
+  tsVBackupQset = taosOpenQset();
+
+  tsVBackupPool.num = 1;
+  tsVBackupPool.worker = calloc(sizeof(SVBackupWorker), tsVBackupPool.num);
+
+  if (tsVBackupPool.worker == NULL) return -1;
+  for (int32_t i = 0; i < tsVBackupPool.num; ++i) {
+    SVBackupWorker *pWorker = tsVBackupPool.worker + i;
+    pWorker->workerId = i;
+    vDebug("vbackup:%d is created", i);
+  }
+
+  vDebug("vbackup is initialized, num:%d qset:%p", tsVBackupPool.num, tsVBackupQset);
+
+  return vnodeStartBackup();
+}
+
+void vnodeCleanupBackup() {
+  for (int32_t i = 0; i < tsVBackupPool.num; ++i) {
+    SVBackupWorker *pWorker = tsVBackupPool.worker + i;
+    if (taosCheckPthreadValid(pWorker->thread)) {
+      taosQsetThreadResume(tsVBackupQset);
+    }
+    vDebug("vbackup:%d is closed", i);
+  }
+
+  for (int32_t i = 0; i < tsVBackupPool.num; ++i) {
+    SVBackupWorker *pWorker = tsVBackupPool.worker + i;
+    vDebug("vbackup:%d start to join", i);
+    if (taosCheckPthreadValid(pWorker->thread)) {
+      pthread_join(pWorker->thread, NULL);
+    }
+    vDebug("vbackup:%d join success", i);
+  }
+
+  vDebug("vbackup is closed, qset:%p", tsVBackupQset);
+
+  taosCloseQset(tsVBackupQset);
+  tsVBackupQset = NULL;
+
+  tfree(tsVBackupPool.worker);
+
+  vDebug("vbackup queue:%p is freed", tsVBackupQueue);
+  taosCloseQueue(tsVBackupQueue);
+  tsVBackupQueue = NULL;
+}

src/vnode/src/vnodeMain.c

@@ -27,6 +27,7 @@
 #include "vnodeVersion.h"
 #include "vnodeMgmt.h"
 #include "vnodeWorker.h"
+#include "vnodeBackup.h"
 #include "vnodeMain.h"
 
 static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno);
@@ -364,7 +365,6 @@ int32_t vnodeOpen(int32_t vgId) {
   syncInfo.startSyncFileFp = vnodeStartSyncFile;
   syncInfo.stopSyncFileFp = vnodeStopSyncFile;
   syncInfo.getVersionFp = vnodeGetVersion;
-  syncInfo.resetVersionFp = vnodeResetVersion;
   syncInfo.sendFileFp = tsdbSyncSend;
   syncInfo.recvFileFp = tsdbSyncRecv;
   syncInfo.pTsdb = pVnode->tsdb;
@@ -448,18 +448,14 @@ void vnodeDestroy(SVnodeObj *pVnode) {
 
   if (pVnode->dropped) {
     char rootDir[TSDB_FILENAME_LEN] = {0};    
-    char newDir[TSDB_FILENAME_LEN] = {0};
+    char stagingDir[TSDB_FILENAME_LEN] = {0};
     sprintf(rootDir, "%s/vnode%d", "vnode", vgId);
-    sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId);
+    sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId);
 
-    if (0 == tsEnableVnodeBak) {
-      vInfo("vgId:%d, vnode backup not enabled", pVnode->vgId);
-    } else {
-      tfsRmdir(newDir);
-      tfsRename(rootDir, newDir);
-    }
+    tfsRename(rootDir, stagingDir);
+
+    vnodeBackup(vgId);
 
-    tfsRmdir(rootDir);
     dnodeSendStatusMsgToMnode();
   }
 

src/vnode/src/vnodeMgmt.c

@@ -17,6 +17,7 @@
 #include "os.h"
 #include "dnode.h"
 #include "vnodeStatus.h"
+#include "vnodeBackup.h"
 #include "vnodeWorker.h"
 #include "vnodeRead.h"
 #include "vnodeWrite.h"
@@ -29,6 +30,7 @@ static void    vnodeCleanupHash(void);
 static void    vnodeIncRef(void *ptNode);
 
 static SStep tsVnodeSteps[] = {
+  {"vnode-backup", vnodeInitBackup,    vnodeCleanupBackup},
   {"vnode-worker", vnodeInitMWorker,    vnodeCleanupMWorker},
   {"vnode-write",  vnodeInitWrite,      vnodeCleanupWrite},
   {"vnode-read",   vnodeInitRead,       vnodeCleanupRead},

src/vnode/src/vnodeSync.c

@@ -107,8 +107,9 @@ void vnodeStopSyncFile(int32_t vgId, uint64_t fversion) {
   pVnode->fversion = fversion;
   pVnode->version = fversion;
   vnodeSaveVersion(pVnode);
+  walResetVersion(pVnode->wal, fversion);
 
-  vDebug("vgId:%d, datafile is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion);
+  vInfo("vgId:%d, datafile is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion);
   vnodeSetReadyStatus(pVnode);
 
   vnodeRelease(pVnode);
@@ -158,22 +159,6 @@ int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) {
   return code;
 }
 
-int32_t vnodeResetVersion(int32_t vgId, uint64_t fver) {
-  SVnodeObj *pVnode = vnodeAcquire(vgId);
-  if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while reset version", vgId);
-    return -1;
-  }
-
-  pVnode->fversion = fver;
-  pVnode->version = fver;
-  walResetVersion(pVnode->wal, fver);
-  vInfo("vgId:%d, version reset to %" PRIu64, vgId, fver);
-
-  vnodeRelease(pVnode);
-  return 0;
-}
-
 void vnodeConfirmForward(void *vparam, uint64_t version, int32_t code, bool force) {
   SVnodeObj *pVnode = vparam;
   syncConfirmForward(pVnode->sync, version, code, force);

src/vnode/src/vnodeWrite.c

@@ -96,6 +96,7 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara
   // write into WAL
   code = walWrite(pVnode->wal, pHead);
   if (code < 0) {
+    if (syncCode > 0) atomic_sub_fetch_32(&pWrite->processedCount, 1);
     vError("vgId:%d, hver:%" PRIu64 " vver:%" PRIu64 " code:0x%x", pVnode->vgId, pHead->version, pVnode->version, code);
     return code;
   }
@@ -104,7 +105,10 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara
 
   // write data locally
   code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, pRspRet);
-  if (code < 0) return code;
+  if (code < 0) {
+    if (syncCode > 0) atomic_sub_fetch_32(&pWrite->processedCount, 1);
+    return code;
+  }
 
   return syncCode;
 }

tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java

@@ -68,12 +68,12 @@ public class JDBCDemo {
     }
 
     private void insert() {
-        final String sql = "insert into test.weather (ts, temperature, humidity) values(now, 20.5, 34)";
+        final String sql = "insert into  " + dbName + "." + tbName + " (ts, temperature, humidity) values(now, 20.5, 34)";
         exuete(sql);
     }
 
     private void select() {
-        final String sql = "select * from test.weather";
+        final String sql = "select * from "+ dbName + "." + tbName;
         executeQuery(sql);
     }
 

tests/pytest/fulltest.sh

@@ -139,6 +139,18 @@ python3 ./test.py -f import_merge/importInsertThenImport.py
 python3 ./test.py -f import_merge/importCSV.py
 #======================p1-end===============
 #======================p2-start===============
+# tools
+python3 test.py -f tools/taosdumpTest.py
+
+python3 test.py -f tools/taosdemoTest.py
+python3 test.py -f tools/taosdemoTestWithoutMetric.py
+python3 test.py -f tools/taosdemoTestWithJson.py
+python3 test.py -f tools/taosdemoTestLimitOffset.py
+python3 test.py -f tools/taosdemoTestTblAlt.py
+python3 test.py -f tools/taosdemoTestSampleData.py
+python3 test.py -f tools/taosdemoTestInterlace.py
+python3 test.py -f tools/taosdemoTestQuery.py
+
 # update
 python3 ./test.py -f update/allow_update.py
 python3 ./test.py -f update/allow_update-0.py
@@ -201,6 +213,8 @@ python3 ./test.py -f query/bug2119.py
 python3 ./test.py -f query/isNullTest.py
 python3 ./test.py -f query/queryWithTaosdKilled.py
 python3 ./test.py -f query/floatCompare.py
+python3 ./test.py -f query/query1970YearsAf.py
+python3 ./test.py -f query/bug3351.py
 python3 ./test.py -f query/bug3375.py
 
 
@@ -247,18 +261,6 @@ python3 test.py -f subscribe/supertable.py
 #======================p3-end===============
 #======================p4-start===============
 
-# tools
-python3 test.py -f tools/taosdumpTest.py
-
-python3 test.py -f tools/taosdemoTest.py
-python3 test.py -f tools/taosdemoTestWithoutMetric.py
-python3 test.py -f tools/taosdemoTestWithJson.py
-python3 test.py -f tools/taosdemoTestLimitOffset.py
-python3 test.py -f tools/taosdemoTest2.py
-python3 test.py -f tools/taosdemoTestSampleData.py
-python3 test.py -f tools/taosdemoTestInterlace.py
-python3 test.py -f tools/taosdemoTestQuery.py
-
 python3 ./test.py -f update/merge_commit_data-0.py
 # wal
 python3 ./test.py -f wal/addOldWalTest.py

tests/pytest/query/bug3351.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+        tdSql.execute("drop database if exists db")
+        tdSql.execute("create database  if not exists db keep 36500")
+        tdSql.execute("use db")
+        tdLog.printNoPrefix("==========step1:create table && insert data")
+
+        tdSql.execute(
+            "create table stb1 (ts timestamp, c1 int) TAGS(t1 int)"
+        )
+        tdSql.execute("create table t0 using stb1 tags(1)")
+        tdSql.execute("insert into t0 values (-865000000, 1)")
+        tdSql.execute("insert into t0 values (-864000000, 2)")
+        tdSql.execute("insert into t0 values (-863000000, 3)")
+        tdSql.execute("insert into t0 values (-15230000, 4)")
+        tdSql.execute("insert into t0 values (-15220000, 5)")
+        tdSql.execute("insert into t0 values (-15210000, 6)")
+
+        tdLog.printNoPrefix("==========step2:query")
+        # bug1:when ts > -864000000, return 0 rows;
+        # bug2:when ts = -15220000, return 0 rows.
+        tdSql.query('select * from t0 where ts < -864000000')
+        tdSql.checkRows(1)
+        tdSql.query('select * from t0 where ts <= -864000000')
+        tdSql.checkRows(2)
+        tdSql.query('select * from t0 where ts = -864000000')
+        tdSql.checkRows(1)
+        tdSql.query('select * from t0 where ts > -864000000')
+        tdSql.checkRows(4)
+        tdSql.query('select * from t0 where ts >= -864000000')
+        tdSql.checkRows(5)
+        tdSql.query('select * from t0 where ts < -15220000')
+        tdSql.checkRows(4)
+        tdSql.query('select * from t0 where ts <= -15220000')
+        tdSql.checkRows(5)
+        tdSql.query('select * from t0 where ts = -15220000')
+        tdSql.checkRows(1)
+        tdSql.query('select * from t0 where ts > -15220000')
+        tdSql.checkRows(1)
+        tdSql.query('select * from t0 where ts >= -15220000')
+        tdSql.checkRows(2)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

tests/pytest/query/query1970YearsAf.py

@@ -22,25 +22,34 @@ import datetime
 from util.log import *
 from util.sql import *
 from util.cases import *
+from util.dnodes import *
+from util.dnodes import TDDnode
 
 class TDTestCase:
 
+    def __init__(self):
+        self.path = ""
+
     def init(self, conn, logSql):
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())
 
-    def getCfgDir(self, path):
+    def getcfgPath(self, path):
         binPath = os.path.dirname(os.path.realpath(__file__))
         binPath = binPath + "/../../../debug/"
-        tdLog.debug("binPath %s" % (binPath))
+        tdLog.debug(f"binPath {binPath}")
         binPath = os.path.realpath(binPath)
-        tdLog.debug("binPath real path %s" % (binPath))
+        tdLog.debug(f"binPath real path {binPath}")
+
         if path == "":
             self.path = os.path.abspath(binPath + "../../")
         else:
             self.path = os.path.realpath(path)
+        return self.path
 
-        self.cfgDir = "%s/sim/psim/cfg" % (self.path)
+    def getCfgDir(self):
+        self.getcfgPath(self.path)
+        self.cfgDir = f"{self.path}/sim/psim/cfg"
         return self.cfgDir
 
     def creatcfg(self):
@@ -63,7 +72,7 @@ class TDTestCase:
             "update": 0
         }
 
-        # 设置创建的超级表格式
+        # set stable schema
         stable1 = {
             "name": "stb2",
             "child_table_exists": "no",
@@ -75,35 +84,49 @@ class TDTestCase:
             "insert_rows": 5000,
             "multi_thread_write_one_tbl": "no",
             "number_of_tbl_in_one_sql": 0,
-            "rows_per_tbl": 1000,
+            "rows_per_tbl": 1,
             "max_sql_len": 65480,
             "disorder_ratio": 0,
             "disorder_range": 1000,
-            "timestamp_step": 19000000,
-            "start_timestamp": "1969-01-01 00:00:00.000",
+            "timestamp_step": 20000,
+            "start_timestamp": "1969-12-31 00:00:00.000",
             "sample_format": "csv",
             "sample_file": "./sample.csv",
             "tags_file": "",
             "columns": [
-                {"type": "INT"},
-                {"type": "DOUBLE", "count": 10},
-                {"type": "BINARY", "len": 16, "count": 3},
-                {"type": "BINARY", "len": 32, "count": 6}
+                {"type": "INT", "count": 2},
+                {"type": "DOUBLE", "count": 2},
+                {"type": "BIGINT", "count": 2},
+                {"type": "FLOAT", "count": 2},
+                {"type": "SMALLINT", "count": 2},
+                {"type": "TINYINT", "count": 2},
+                {"type": "BOOL", "count": 2},
+                {"type": "NCHAR", "len": 3, "count": 1},
+                {"type": "BINARY", "len": 8, "count": 1}
+
             ],
             "tags": [
+                {"type": "INT", "count": 2},
+                {"type": "DOUBLE", "count": 2},
+                {"type": "BIGINT", "count": 2},
+                {"type": "FLOAT", "count": 2},
+                {"type": "SMALLINT", "count": 2},
                 {"type": "TINYINT", "count": 2},
-                {"type": "BINARY", "len": 16, "count": 5}
+                {"type": "BOOL", "count": 2},
+                {"type": "NCHAR", "len": 3, "count": 1},
+                {"type": "BINARY", "len": 8, "count": 1}
             ]
         }
 
-        # 需要创建多个超级表时，只需创建不同的超级表格式并添加至super_tables
-        super_tables = [stable1]
+        # create different stables like stable1 and add to list super_tables
+        super_tables = []
+        super_tables.append(stable1)
         database = {
             "dbinfo": dbinfo,
             "super_tables": super_tables
         }
 
-        cfgdir = self.getCfgDir("")
+        cfgdir = self.getCfgDir()
         create_table = {
             "filetype": "insert",
             "cfgdir": cfgdir,
@@ -121,105 +144,110 @@ class TDTestCase:
         }
         return create_table
 
-    def inserttable(self):
+    def createinsertfile(self):
         create_table = self.creatcfg()
         date = datetime.datetime.now().strftime("%Y%m%d%H%M")
         file_create_table = f"/tmp/insert_{date}.json"
 
         with open(file_create_table, 'w') as f:
             json.dump(create_table, f)
+        return file_create_table
 
-        create_table_cmd = f"taosdemo -f {file_create_table}"
+    def inserttable(self, filepath):
+        create_table_cmd = f"taosdemo -f {filepath}  > /dev/null 2>&1"
         _ = subprocess.check_output(create_table_cmd, shell=True).decode("utf-8")
 
-
-    def run(self):
-        s = 'reset query cache'
-        tdSql.execute(s)
-        s = 'create database if not exists db'
-        tdSql.execute(s)
-        s = 'use db'
-        tdSql.execute(s)
-
-        tdLog.info("==========step1:create table stable and child table,then insert data automatically")
-        self.inserttable()
-        # tdSql.execute(
-        #     '''create table if not exists supt
-        #     (ts timestamp, c1 int, c2 float, c3 bigint, c4 double, c5 smallint, c6 tinyint)
-        #     tags(location binary(64), type int, isused bool , family nchar(64))'''
-        # )
-        # tdSql.execute("create table t1 using supt tags('beijing', 1, 1, '自行车')")
-        # tdSql.execute("create table t2 using supt tags('shanghai', 2, 0, '拖拉机')")
-        # tdSql.execute(
-        #     f"insert into t1 values (-31564800000, 6, 5, 4, 3, 2, 1)"
-        # )
-
-
-
-        tdLog.info("==========step2:query join")
-
+    def sqlsquery(self):
         # stable query
         tdSql.query(
             "select * from stb2 where stb2.ts < '1970-01-01 00:00:00.000' "
         )
-        tdSql.checkRows(16600)
+        tdSql.checkRows(43200)
 
         tdSql.query(
             "select * from stb2 where stb2.ts >= '1970-01-01 00:00:00.000' "
         )
-        tdSql.checkRows(33400)
+        tdSql.checkRows(6800)
 
         tdSql.query(
-            "select * from stb2 where stb2.ts > '1969-12-01 00:00:00.000' and stb2.ts <'1970-01-31 00:00:00.000' "
+            "select * from stb2 where stb2.ts > '1969-12-31 23:00:00.000' and stb2.ts <'1970-01-01 01:00:00.000' "
         )
-        tdSql.checkRows(2780)
+        tdSql.checkRows(3590)
 
-        # child-table query
+        # child-tables query
         tdSql.query(
             "select * from t0 where t0.ts < '1970-01-01 00:00:00.000' "
         )
-        tdSql.checkRows(1660)
+        tdSql.checkRows(4320)
 
         tdSql.query(
             "select * from t1 where t1.ts >= '1970-01-01 00:00:00.000' "
         )
-        tdSql.checkRows(3340)
+        tdSql.checkRows(680)
 
         tdSql.query(
-            "select * from t9 where t9.ts > '1969-12-01 00:00:00.000' and t9.ts <'1970-01-31 00:00:00.000' "
+            "select * from t9 where t9.ts > '1969-12-31 22:00:00.000' and t9.ts <'1970-01-01 02:00:00.000' "
         )
-        tdSql.checkRows(278)
-
+        tdSql.checkRows(719)
+        
         tdSql.query(
             "select * from t0,t1 where t0.ts=t1.ts and t1.ts >= '1970-01-01 00:00:00.000' "
         )
-        tdSql.checkRows(3340)
-
+        tdSql.checkRows(680)
+        
         tdSql.query(
             "select diff(col1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' "
         )
-        tdSql.checkRows(3339)
+        tdSql.checkRows(679)
 
         tdSql.query(
             "select t0,col1 from stb2 where stb2.ts < '1970-01-01 00:00:00.000' order by ts"
         )
-        tdSql.checkRows(16600)
+        tdSql.checkRows(43200)
 
         # query with timestamp in 'where ...'
         tdSql.query(
             "select * from stb2 where stb2.ts > -28800000 "
         )
-        tdSql.checkRows(33400)
+        tdSql.checkRows(6790)
 
         tdSql.query(
             "select * from stb2 where stb2.ts > -28800000 and stb2.ts < '1970-01-01 08:00:00.000' "
         )
-        tdSql.checkRows(20)
+        tdSql.checkRows(6790)
 
         tdSql.query(
-            "select * from stb2 where stb2.ts < -28800000 and stb2.ts > '1969-12-31 16:00:00.000' "
+            "select * from stb2 where stb2.ts < -28800000 and stb2.ts > '1969-12-31 22:00:00.000' "
         )
+        tdSql.checkRows(3590)
+
+    def run(self):
+        s = 'reset query cache'
+        tdSql.execute(s)
+        s = 'create database if not exists db'
+        tdSql.execute(s)
+        s = 'use db'
+        tdSql.execute(s)
+
+        tdLog.info("==========step1:create table stable and child table,then insert data automatically")
+        insertfile = self.createinsertfile()
+        self.inserttable(insertfile)
+
+        tdLog.info("==========step2:query join")
+        self.sqlsquery()
+
+        # after wal and sync, check again
+        tdSql.query("show dnodes")
+        index = tdSql.getData(0, 0)
+        tdDnodes.stop(index)
+        tdDnodes.start(index)
+
+        tdLog.info("==========step3: query join again")
+        self.sqlsquery()
 
+        # delete temporary file
+        rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1"
+        _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8")
 
     def stop(self):
         tdSql.close()

tests/pytest/tools/taosdemoTest2.py → tests/pytest/tools/taosdemoTestTblAlt.py

@@ -29,10 +29,33 @@ class TDTestCase:
         self.numberOfTables = 10
         self.numberOfRecords = 1000000
 
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+
     def insertDataAndAlterTable(self, threadID):
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        binPath = buildPath + "/build/bin/"
+
         if(threadID == 0):
-            os.system("taosdemo -y -t %d -n %d" %
-                      (self.numberOfTables, self.numberOfRecords))
+            os.system("%staosdemo -y -t %d -n %d" %
+                      (binPath, self.numberOfTables, self.numberOfRecords))
         if(threadID == 1):
             time.sleep(2)
             print("use test")
@@ -47,7 +70,13 @@ class TDTestCase:
 
             # check if all the tables have heen created
             while True:
-                tdSql.query("show tables")
+                try:
+                    tdSql.query("show tables")
+                except Exception as e:
+                    tdLog.info("show tables test failed")
+                    time.sleep(1)
+                    continue
+
                 rows = tdSql.queryRows
                 print("number of tables: %d" % rows)
                 if(rows == self.numberOfTables):
@@ -56,16 +85,23 @@ class TDTestCase:
             # check if there are any records in the last created table
             while True:
                 print("query started")
-                tdSql.query("select * from test.t9")
+                try:
+                    tdSql.query("select * from test.t9")
+                except Exception as e:
+                    tdLog.info("select * test failed")
+                    time.sleep(2)
+                    continue
+
                 rows = tdSql.queryRows
                 print("number of records: %d" % rows)
                 if(rows > 0):
                     break
                 time.sleep(1)
+
             print("alter table test.meters add column col10 int")
             tdSql.execute("alter table test.meters add column col10 int")
-            print("insert into test.t0 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)")
-            tdSql.execute("insert into test.t0 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)")
+            print("insert into test.t9 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)")
+            tdSql.execute("insert into test.t9 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)")
 
     def run(self):
         tdSql.prepare()
@@ -78,6 +114,8 @@ class TDTestCase:
         t1.join()
         t2.join()
 
+        time.sleep(3)
+
         tdSql.query("select count(*) from test.meters")
         tdSql.checkData(0, 0, self.numberOfRecords * self.numberOfTables + 1)
 

tests/test-all.sh

@@ -25,29 +25,24 @@ function stopTaosd {
 function dohavecore(){
   corefile=`find $corepath -mmin 1`  
   core_file=`echo $corefile|cut -d " " -f2`
-  echo "corefile:$core_file"
-  echo "corepath:$corepath"
-  ls -l $corepath
   proc=`echo $corefile|cut -d "_" -f3`
   if [ -n "$corefile" ];then
     echo 'taosd or taos has generated core'
+    rm case.log
     if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]] && [[ $1 == 1 ]]; then
       cd ../../../
       tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug/build/bin/taosd debug/build/bin/tsim debug/build/lib/libtaos*so*
       if [[ $2 == 1 ]];then
         cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"`
-        rm -rf sim/case.log
       else
         cd community
         cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" `
-        rm -rf sim/case.log
       fi
     else 
       cd ../../
       if [[ $1 == 1 ]];then 
         tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug/build/bin/taosd debug/build/bin/tsim debug/build/lib/libtaos*so*
         cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" `
-        rm -rf sim/case.log
       fi
     fi
     if [[ $1 == 1 ]];then
@@ -82,7 +77,6 @@ function runSimCaseOneByOne {
       # fi
       end_time=`date +%s`
       echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log
-      dohavecore 0
     fi
   done < $1
 }
@@ -99,26 +93,25 @@ function runSimCaseOneByOnefq {
       date +%F\ %T | tee -a out.log
       if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
         echo -n $case
-        ./test.sh -f $case > ../../../sim/case.log 2>&1 && \
+        ./test.sh -f $case > case.log 2>&1 && \
         ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log  ||  echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
         ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log )  || \
-        ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat ../../../sim/case.log )
+        ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log )
       else
         echo -n $case
         ./test.sh -f $case > ../../sim/case.log 2>&1 && \
         ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log  ||  echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
         ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log )  || \
-        ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' &&  cat ../../sim/case.log )
+        ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' &&  cat case.log )
       fi
       
       out_log=`tail -1 out.log  `
       if [[ $out_log =~ 'failed' ]];then
+        rm case.log
         if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
           cp -r ../../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"`
-          rm -rf ../../../sim/case.log
         else 
           cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" `
-          rm -rf ../../sim/case.log
         fi
         dohavecore $2 1
         if [[ $2 == 1 ]];then
@@ -159,7 +152,6 @@ function runPyCaseOneByOne {
       else
         $line > /dev/null 2>&1
       fi
-      dohavecore 0
     fi
   done < $1
 }
@@ -185,7 +177,7 @@ function runPyCaseOneByOnefq() {
         start_time=`date +%s`
         date +%F\ %T | tee -a pytest-out.log
         echo -n $case
-        $line > ../../sim/case.log 2>&1 && \
+        $line > case.log 2>&1 && \
           echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \
           echo -e "${RED} failed${NC}" | tee -a pytest-out.log 
         end_time=`date +%s`
@@ -193,8 +185,8 @@ function runPyCaseOneByOnefq() {
         if [[ $out_log =~ 'failed' ]];then
           cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" `
           echo '=====================log===================== '
-          cat ../../sim/case.log
-          rm -rf ../../sim/case.log
+          cat case.log
+          rm -rf case.log
           dohavecore $2 2
           if [[ $2 == 1 ]];then
             exit 8