Merge pull request #6092 from taosdata/feature/m2d

Feature/m2d

cmake/install.inc

@@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS)
   #INSTALL(TARGETS taos RUNTIME DESTINATION driver)
   #INSTALL(TARGETS shell RUNTIME DESTINATION .)
   IF (TD_MVN_INSTALLED)
-    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.28-dist.jar DESTINATION connector/jdbc)
+    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.29.jar DESTINATION connector/jdbc)
   ENDIF ()
 ELSEIF (TD_DARWIN)
   SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")

src/client/inc/tschemautil.h

@@ -21,8 +21,8 @@ extern "C" {
 #endif
 
 #include "taosmsg.h"
-#include "tstoken.h"
 #include "tsclient.h"
+#include "ttoken.h"
 
 /**
  * get the number of tags of this table

src/client/src/tscParseInsert.c

@@ -29,8 +29,7 @@
 #include "taosdef.h"
 
 #include "tscLog.h"
-#include "tscSubquery.h"
-#include "tstoken.h"
+#include "ttoken.h"
 
 #include "tdataformat.h"
 
@@ -464,23 +463,24 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, SSqlCmd *pCmd, int1
     // Remove quotation marks
     if (TK_STRING == sToken.type) {
       // delete escape character: \\, \', \"
-      char    delim = sToken.z[0];
+      char delim = sToken.z[0];
+
       int32_t cnt = 0;
       int32_t j = 0;
       for (uint32_t k = 1; k < sToken.n - 1; ++k) {
-        if (sToken.z[k] == delim || sToken.z[k] == '\\') {
-          if (sToken.z[k + 1] == delim) {
-            cnt++;
+        if (sToken.z[k] == '\\' || (sToken.z[k] == delim && sToken.z[k + 1] == delim)) {
             tmpTokenBuf[j] = sToken.z[k + 1];
-            j++;
-            k++;
-            continue;
-          }
+
+          cnt++;
+          j++;
+          k++;
+          continue;
         }
 
         tmpTokenBuf[j] = sToken.z[k];
         j++;
       }
+
       tmpTokenBuf[j] = 0;
       sToken.z = tmpTokenBuf;
       sToken.n -= 2 + cnt;
@@ -1006,7 +1006,7 @@ int validateTableName(char *tblName, int len, SStrToken* psTblToken) {
 
   psTblToken->n    = len;
   psTblToken->type = TK_ID;
-  tSQLGetToken(psTblToken->z, &psTblToken->type);
+  tGetToken(psTblToken->z, &psTblToken->type);
 
   return tscValidateName(psTblToken);
 }

src/client/src/tscPrepare.c

@@ -174,7 +174,7 @@ static int normalStmtPrepare(STscStmt* stmt) {
 
   while (sql[i] != 0) {
     SStrToken token = {0};
-    token.n = tSQLGetToken(sql + i, &token.type);
+    token.n = tGetToken(sql + i, &token.type);
 
     if (token.type == TK_QUESTION) {
       sql[i] = 0;

src/client/src/tscSQLParser.c

@@ -21,19 +21,19 @@
 #endif // __APPLE__
 
 #include "os.h"
-#include "ttype.h"
-#include "texpr.h"
 #include "taos.h"
 #include "taosmsg.h"
 #include "tcompare.h"
+#include "texpr.h"
 #include "tname.h"
 #include "tscLog.h"
 #include "tscUtil.h"
 #include "tschemautil.h"
 #include "tsclient.h"
-#include "tstoken.h"
 #include "tstrbuild.h"
+#include "ttoken.h"
 #include "ttokendef.h"
+#include "ttype.h"
 #include "qUtil.h"
 
 #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0"
@@ -432,7 +432,6 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
       }
-
       // additional msg has been attached already
       code = tscSetTableFullName(pTableMetaInfo, pToken, pSql);
       if (code != TSDB_CODE_SUCCESS) {
@@ -984,11 +983,10 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableNam
   const char* msg3 = "no acctId";
   const char* msg4 = "db name too long";
   const char* msg5 = "table name too long";
-  
 
   SSqlCmd* pCmd = &pSql->cmd;
   int32_t  code = TSDB_CODE_SUCCESS;
-  int32_t idx = getDelimiterIndex(pTableName); 
+  int32_t  idx  = getDelimiterIndex(pTableName);
   if (idx != -1) { // db has been specified in sql string so we ignore current db path
     char* acctId = getAccountId(pSql);
     if (acctId == NULL || strlen(acctId) <= 0) {
@@ -1002,9 +1000,9 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableNam
     if (idx >= TSDB_DB_NAME_LEN) { 
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
     }
-   
+
     if (pTableName->n - 1 - idx >= TSDB_TABLE_NAME_LEN) {
-       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
+      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
     }
     
     char name[TSDB_TABLE_FNAME_LEN] = {0};
@@ -1351,8 +1349,8 @@ static char* cloneCurrentDBName(SSqlObj* pSql) {
 
 /* length limitation, strstr cannot be applied */
 static int32_t getDelimiterIndex(SStrToken* pTableName) {
-  for (uint32_t i = 0; i < pTableName->n; ++i) {
-    if (pTableName->z[i] == TS_PATH_DELIMITER[0]) {
+  for (uint32_t i = 0; i < pTableName->n; ++i) { 
+    if (pTableName->z[i] == TS_PATH_DELIMITER[0]) { 
       return i;
     }
   }
@@ -4646,7 +4644,7 @@ int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t t
       }
     } else {
       SStrToken token = {.z = pRight->value.pz, .n = pRight->value.nLen, .type = TK_ID};
-      int32_t   len = tSQLGetToken(pRight->value.pz, &token.type);
+      int32_t   len = tGetToken(pRight->value.pz, &token.type);
 
       if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->value.nLen) {
         return TSDB_CODE_TSC_INVALID_SQL;
@@ -5533,13 +5531,13 @@ int32_t validateLocalConfig(SMiscInfo* pOptions) {
 }
 
 int32_t validateColumnName(char* name) {
-  bool ret = isKeyWord(name, (int32_t)strlen(name));
+  bool ret = taosIsKeyWordToken(name, (int32_t)strlen(name));
   if (ret) {
     return TSDB_CODE_TSC_INVALID_SQL;
   }
 
   SStrToken token = {.z = name};
-  token.n = tSQLGetToken(name, &token.type);
+  token.n = tGetToken(name, &token.type);
 
   if (token.type != TK_STRING && token.type != TK_ID) {
     return TSDB_CODE_TSC_INVALID_SQL;
@@ -5550,7 +5548,7 @@ int32_t validateColumnName(char* name) {
     strntolower(token.z, token.z, token.n);
     token.n = (uint32_t)strtrim(token.z);
 
-    int32_t k = tSQLGetToken(token.z, &token.type);
+    int32_t k = tGetToken(token.z, &token.type);
     if (k != token.n) {
       return TSDB_CODE_TSC_INVALID_SQL;
     }
@@ -7527,4 +7525,3 @@ bool hasNormalColumnFilter(SQueryInfo* pQueryInfo) {
 
   return false;
 }
-

src/client/src/tscServer.c

@@ -1881,6 +1881,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
+  assert(pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE || pTableMeta->tableType == TSDB_NORMAL_TABLE || pTableMeta->tableType == TSDB_STREAM_TABLE);
+
   if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
     // check if super table hashmap or not
     int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN);
@@ -2451,6 +2453,7 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) {
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
     pTableMetaInfo->pTableMeta = (STableMeta *)tmp;
+    memset(pTableMetaInfo->pTableMeta, 0, size);
     pTableMetaInfo->tableMetaSize = size;
   } else {
     //uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta);

src/client/src/tscSql.c

@@ -963,7 +963,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t
     len = (int32_t)strtrim(tblName);
 
     SStrToken sToken = {.n = len, .type = TK_ID, .z = tblName};
-    tSQLGetToken(tblName, &sToken.type);
+    tGetToken(tblName, &sToken.type);
 
     // Check if the table name available or not
     if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) {

src/client/src/tscUtil.c

@@ -1895,7 +1895,7 @@ void tscColumnListDestroy(SArray* pColumnList) {
 static int32_t validateQuoteToken(SStrToken* pToken) {
   tscDequoteAndTrimToken(pToken);
 
-  int32_t k = tSQLGetToken(pToken->z, &pToken->type);
+  int32_t k = tGetToken(pToken->z, &pToken->type);
 
   if (pToken->type == TK_STRING) {
     return tscValidateName(pToken);
@@ -1963,7 +1963,7 @@ int32_t tscValidateName(SStrToken* pToken) {
       tscStrToLower(pToken->z, pToken->n);
       //pToken->n = (uint32_t)strtrim(pToken->z);
        
-      int len = tSQLGetToken(pToken->z, &pToken->type);
+      int len = tGetToken(pToken->z, &pToken->type);
 
       // single token, validate it
       if (len == pToken->n) {
@@ -1989,7 +1989,7 @@ int32_t tscValidateName(SStrToken* pToken) {
       pToken->n = (uint32_t)strtrim(pToken->z);
     }
 
-    pToken->n = tSQLGetToken(pToken->z, &pToken->type);
+    pToken->n = tGetToken(pToken->z, &pToken->type);
     if (pToken->z[pToken->n] != TS_PATH_DELIMITER[0]) {
       return TSDB_CODE_TSC_INVALID_SQL;
     }
@@ -2006,7 +2006,7 @@ int32_t tscValidateName(SStrToken* pToken) {
 
     pToken->z = sep + 1;
     pToken->n = (uint32_t)(oldLen - (sep - pStr) - 1);
-    int32_t len = tSQLGetToken(pToken->z, &pToken->type);
+    int32_t len = tGetToken(pToken->z, &pToken->type);
     if (len != pToken->n || (pToken->type != TK_STRING && pToken->type != TK_ID)) {
       return TSDB_CODE_TSC_INVALID_SQL;
     }

src/client/tests/timeParseTest.cpp

@@ -4,7 +4,7 @@
 #include <iostream>
 
 #include "taos.h"
-#include "tstoken.h"
+#include "ttoken.h"
 #include "tutil.h"
 
 int main(int argc, char** argv) {

src/common/inc/tname.h

@@ -18,7 +18,7 @@
 
 #include "os.h"
 #include "taosmsg.h"
-#include "tstoken.h"
+#include "ttoken.h"
 #include "tvariant.h"
 
 typedef struct SDataStatis {

src/common/inc/tvariant.h

@@ -16,8 +16,8 @@
 #ifndef TDENGINE_TVARIANT_H
 #define TDENGINE_TVARIANT_H
 
-#include "tstoken.h"
 #include "tarray.h"
+#include "ttoken.h"
 
 #ifdef __cplusplus
 extern "C" {

src/common/src/tname.c

@@ -2,7 +2,7 @@
 #include "tutil.h"
 
 #include "tname.h"
-#include "tstoken.h"
+#include "ttoken.h"
 #include "tvariant.h"
 
 #define VALIDNUMOFCOLS(x)  ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS)

src/common/src/tvariant.c

@@ -14,14 +14,14 @@
  */
 #include "os.h"
 
-#include "tvariant.h"
 #include "hash.h"
 #include "taos.h"
 #include "taosdef.h"
-#include "tstoken.h"
+#include "ttoken.h"
 #include "ttokendef.h"
-#include "tutil.h"
 #include "ttype.h"
+#include "tutil.h"
+#include "tvariant.h"
 
 void tVariantCreate(tVariant *pVar, SStrToken *token) {
   int32_t ret = 0;
@@ -49,7 +49,7 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) {
       ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, true);
       if (ret != 0) {
         SStrToken t = {0};
-        tSQLGetToken(token->z, &t.type);
+        tGetToken(token->z, &t.type);
         if (t.type == TK_MINUS) {  // it is a signed number which is greater than INT64_MAX or less than INT64_MIN
           pVar->nType = -1;   // -1 means error type
           return;
@@ -460,7 +460,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result
     *result = (int64_t) pVariant->dKey;
   } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) {
     SStrToken token = {.z = pVariant->pz, .n = pVariant->nLen};
-    /*int32_t n = */tSQLGetToken(pVariant->pz, &token.type);
+    /*int32_t n = */tGetToken(pVariant->pz, &token.type);
 
     if (token.type == TK_NULL) {
       if (releaseVariantPtr) {
@@ -495,10 +495,10 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result
     wchar_t *endPtr = NULL;
     
     SStrToken token = {0};
-    token.n = tSQLGetToken(pVariant->pz, &token.type);
+    token.n = tGetToken(pVariant->pz, &token.type);
     
     if (token.type == TK_MINUS || token.type == TK_PLUS) {
-      token.n = tSQLGetToken(pVariant->pz + token.n, &token.type);
+      token.n = tGetToken(pVariant->pz + token.n, &token.type);
     }
     
     if (token.type == TK_FLOAT) {

src/connector/jdbc/CMakeLists.txt

@@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED)
   ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME}
     POST_BUILD
     COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
-    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.28-dist.jar ${LIBRARY_OUTPUT_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.29.jar ${LIBRARY_OUTPUT_PATH}
     COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
     COMMENT "build jdbc driver")
   ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME})

src/connector/jdbc/deploy-pom.xml

@@ -5,7 +5,7 @@
 
   <groupId>com.taosdata.jdbc</groupId>
   <artifactId>taos-jdbcdriver</artifactId>
-  <version>2.0.28</version>
+  <version>2.0.29</version>
   <packaging>jar</packaging>
 
   <name>JDBCDriver</name>

src/connector/jdbc/pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.taosdata.jdbc</groupId>
     <artifactId>taos-jdbcdriver</artifactId>
-    <version>2.0.28</version>
+    <version>2.0.29</version>
     <packaging>jar</packaging>
     <name>JDBCDriver</name>
     <url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java

@@ -35,8 +35,8 @@ public class TSDBJNIConnector {
     private long taos = TSDBConstants.JNI_NULL_POINTER;
     
     // result set status in current connection
-    private boolean isResultsetClosed = true;
-    
+    private boolean isResultsetClosed;
+
     private int affectedRows = -1;
 
     static {
@@ -132,6 +132,7 @@ public class TSDBJNIConnector {
 
         // Try retrieving result set for the executed SQL using the current connection pointer. 
         pSql = this.getResultSetImp(this.taos, pSql);
+        // if pSql == 0L that means resultset is closed
         isResultsetClosed = (pSql == TSDBConstants.JNI_NULL_POINTER);
 
         return pSql;

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java

@@ -109,6 +109,8 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet {
     public void close() throws SQLException {
         if (isClosed)
             return;
+        if (this.statement == null)
+            return;
         if (this.jniConnector != null) {
             int code = this.jniConnector.freeResultSet(this.resultSetPointer);
             if (code == TSDBConstants.JNI_CONNECTION_NULL) {
@@ -461,12 +463,13 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet {
     }
 
     public boolean isClosed() throws SQLException {
-        if (isClosed)
-            return true;
-        if (jniConnector != null) {
-            isClosed = jniConnector.isResultsetClosed();
-        }
         return isClosed;
+//        if (isClosed)
+//            return true;
+//        if (jniConnector != null) {
+//            isClosed = jniConnector.isResultsetClosed();
+//        }
+//        return isClosed;
     }
 
     public String getNString(int columnIndex) throws SQLException {

src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java

 package com.taosdata.jdbc;
 
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -19,6 +20,7 @@ public class SubscribeTest {
     String tName = "t0";
     String host = "127.0.0.1";
     String topic = "test";
+    private long ts;
 
     @Test
     public void subscribe() {
@@ -27,26 +29,40 @@ public class SubscribeTest {
             TSDBConnection conn = connection.unwrap(TSDBConnection.class);
             TSDBSubscribe subscribe = conn.subscribe(topic, rawSql, false);
 
-            int a = 0;
-            while (true) {
-                TimeUnit.MILLISECONDS.sleep(1000);
+            for (int j = 0; j < 10; j++) {
+                TimeUnit.SECONDS.sleep(1);
                 TSDBResultSet resSet = subscribe.consume();
+
+                int rowCnt = 0;
                 while (resSet.next()) {
-                    for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) {
-                        System.out.printf(i + ": " + resSet.getString(i) + "\t");
+                    if (rowCnt == 0) {
+                        long cur_ts = resSet.getTimestamp(1).getTime();
+                        int k = resSet.getInt(2);
+                        int v = resSet.getInt(3);
+                        Assert.assertEquals(ts, cur_ts);
+                        Assert.assertEquals(100, k);
+                        Assert.assertEquals(1, v);
                     }
-                    System.out.println("\n======" + a + "==========");
-                }
-                a++;
-                if (a >= 2) {
-                    break;
+                    if (rowCnt == 1) {
+                        long cur_ts = resSet.getTimestamp(1).getTime();
+                        int k = resSet.getInt(2);
+                        int v = resSet.getInt(3);
+                        Assert.assertEquals(ts + 1, cur_ts);
+                        Assert.assertEquals(101, k);
+                        Assert.assertEquals(2, v);
+
+                    }
+                    rowCnt++;
                 }
+                if (j == 0)
+                    Assert.assertEquals(2, rowCnt);
                 resSet.close();
             }
-
             subscribe.close(true);
-        } catch (Exception e) {
-            e.printStackTrace();
+
+
+        } catch (SQLException | InterruptedException throwables) {
+            throwables.printStackTrace();
         }
     }
 
@@ -62,7 +78,7 @@ public class SubscribeTest {
         statement.execute("drop database if exists " + dbName);
         statement.execute("create database if not exists " + dbName);
         statement.execute("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)");
-        long ts = System.currentTimeMillis();
+        ts = System.currentTimeMillis();
         statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", 100, 1)");
         statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + (ts + 1) + ", 101, 2)");
     }

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java

@@ -12,7 +12,7 @@ public class InsertSpecialCharacterJniTest {
     private static String tbname1 = "test";
     private static String tbname2 = "weather";
     private static String special_character_str_1 = "$asd$$fsfsf$";
-    private static String special_character_str_2 = "\\asdfsfsf\\\\";
+    private static String special_character_str_2 = "\\\\asdfsfsf\\\\";
     private static String special_character_str_3 = "\\\\asdfsfsf\\";
     private static String special_character_str_4 = "?asd??fsf?sf?";
     private static String special_character_str_5 = "?#sd@$f(('<(s[P)>\"){]}f?s[]{}%vaew|\"fsfs^a&d*jhg)(j))(f@~!?$";
@@ -70,7 +70,7 @@ public class InsertSpecialCharacterJniTest {
             String f1 = new String(rs.getBytes(2));
             //TODO: bug to be fixed
 //            Assert.assertEquals(special_character_str_2, f1);
-            Assert.assertEquals(special_character_str_2.substring(0, special_character_str_1.length() - 2), f1);
+            Assert.assertEquals(special_character_str_2.substring(1, special_character_str_1.length() - 1), f1);
             String f2 = rs.getString(3);
             Assert.assertNull(f2);
         }

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java

@@ -13,7 +13,7 @@ public class InsertSpecialCharacterRestfulTest {
     private static String tbname1 = "test";
     private static String tbname2 = "weather";
     private static String special_character_str_1 = "$asd$$fsfsf$";
-    private static String special_character_str_2 = "\\asdfsfsf\\\\";
+    private static String special_character_str_2 = "\\\\asdfsfsf\\\\";
     private static String special_character_str_3 = "\\\\asdfsfsf\\";
     private static String special_character_str_4 = "?asd??fsf?sf?";
     private static String special_character_str_5 = "?#sd@$f(('<(s[P)>\"){]}f?s[]{}%vaew|\"fsfs^a&d*jhg)(j))(f@~!?$";
@@ -49,7 +49,7 @@ public class InsertSpecialCharacterRestfulTest {
     @Test
     public void testCase02() throws SQLException {
         //TODO:
-        // Expected :\asdfsfsf\\
+        // Expected :\asdfsfsf\
         // Actual   :\asdfsfsf\
 
         final long now = System.currentTimeMillis();
@@ -71,7 +71,7 @@ public class InsertSpecialCharacterRestfulTest {
             String f1 = new String(rs.getBytes(2));
             //TODO: bug to be fixed
 //            Assert.assertEquals(special_character_str_2, f1);
-            Assert.assertEquals(special_character_str_2.substring(0, special_character_str_1.length() - 2), f1);
+            Assert.assertEquals(special_character_str_2.substring(1, special_character_str_1.length() - 1), f1);
             String f2 = rs.getString(3);
             Assert.assertNull(f2);
         }

src/kit/taosdemo/taosdemo.c

@@ -770,49 +770,48 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) {
       }
       arguments->sqlFile = argv[++i];
     } else if (strcmp(argv[i], "-q") == 0) {
-      if ((argc == i+1)
-              || (!isStringNumber(argv[i+1]))) {
+      if ((argc == i+1) ||
+        (!isStringNumber(argv[i+1]))) {
         printHelp();
         errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n");
         exit(EXIT_FAILURE);
       }
       arguments->async_mode = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-T") == 0) {
-      if ((argc == i+1)
-              || (!isStringNumber(argv[i+1]))) {
+      if ((argc == i+1) ||
+        (!isStringNumber(argv[i+1]))) {
         printHelp();
         errorPrint("%s", "\n\t-T need a number following!\n");
         exit(EXIT_FAILURE);
       }
       arguments->num_of_threads = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-i") == 0) {
-      if ((argc == i+1)
-              || (!isStringNumber(argv[i+1]))) {
+      if ((argc == i+1) ||
+        (!isStringNumber(argv[i+1]))) {
         printHelp();
         errorPrint("%s", "\n\t-i need a number following!\n");
         exit(EXIT_FAILURE);
       }
       arguments->insert_interval = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-qt") == 0) {
-      if ((argc == i+1)
-              || (!isStringNumber(argv[i+1]))
-              || (atoi(argv[i+1]) <= 0)) {
+      if ((argc == i+1) ||
+        (!isStringNumber(argv[i+1]))) {
         printHelp();
-        errorPrint("%s", "\n\t-qt need a valid (>0) number following!\n");
+        errorPrint("%s", "\n\t-qt need a number following!\n");
         exit(EXIT_FAILURE);
       }
       arguments->query_times = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-B") == 0) {
-      if ((argc == i+1)
-              || (!isStringNumber(argv[i+1]))) {
+      if ((argc == i+1) ||
+        (!isStringNumber(argv[i+1]))) {
         printHelp();
         errorPrint("%s", "\n\t-B need a number following!\n");
         exit(EXIT_FAILURE);
       }
       arguments->interlace_rows = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-r") == 0) {
-      if ((argc == i+1)
-              || (!isStringNumber(argv[i+1]))) {
+      if ((argc == i+1) ||
+        (!isStringNumber(argv[i+1]))) {
         printHelp();
         errorPrint("%s", "\n\t-r need a number following!\n");
         exit(EXIT_FAILURE);
@@ -1072,7 +1071,7 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
   if (code != 0) {
     if (!quiet) {
       debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
-      errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res));
+      errorPrint("Failed to run %s, reason: %s\n", command, taos_errstr(res));
     }
     taos_free_result(res);
     //taos_close(taos);
@@ -1102,6 +1101,7 @@ static void appendResultBufToFile(char *resultBuf, char *resultFile)
     }
   }
 
+
   fprintf(fp, "%s", resultBuf);
   tmfclose(fp);
 }
@@ -1137,11 +1137,13 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) {
     totalLen += len;
   }
 
+  verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", __func__, __LINE__, databuf, resultFile);
   appendResultBufToFile(databuf, resultFile);
   free(databuf);
 }
 
-static void selectAndGetResult(threadInfo *pThreadInfo, char *command, char* resultFileName) {
+static void selectAndGetResult(threadInfo *pThreadInfo, char *command, char* resultFile)
+{
   if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) {
     TAOS_RES *res = taos_query(pThreadInfo->taos, command);
     if (res == NULL || taos_errno(res) != 0) {
@@ -1151,14 +1153,16 @@ static void selectAndGetResult(threadInfo *pThreadInfo, char *command, char* res
         return;
     }
 
-    appendResultToFile(res, resultFileName);
+    if ((resultFile) && (strlen(resultFile))) {
+      appendResultToFile(res, resultFile);
+    }
     taos_free_result(res);
 
   } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
       int retCode = postProceSql(
               g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port,
-              g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq],
-              resultFileName);
+              command,
+              resultFile);
       if (0 != retCode) {
         printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID);
       }
@@ -4067,9 +4071,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) {
 
   cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
   if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
-    if (gQueryTimes->valueint <= 0) {
-      errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
-        __func__, __LINE__, gQueryTimes->valueint);
+    if (gQueryTimes->valueint < 0) {
+      errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
+        __func__, __LINE__);
       goto PARSE_OVER;
     }
     g_args.query_times = gQueryTimes->valueint;
@@ -4118,9 +4122,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) {
     cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery,
         "query_times");
     if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
-      if (specifiedQueryTimes->valueint <= 0) {
-        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
-                __func__, __LINE__, specifiedQueryTimes->valueint);
+      if (specifiedQueryTimes->valueint < 0) {
+        errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
+          __func__, __LINE__);
         goto PARSE_OVER;
 
       }
@@ -4262,9 +4266,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) {
 
     cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
     if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
-      if (superQueryTimes->valueint <= 0) {
-        errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n",
-                __func__, __LINE__, superQueryTimes->valueint);
+      if (superQueryTimes->valueint < 0) {
+        errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
+          __func__, __LINE__);
         goto PARSE_OVER;
       }
       g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
@@ -5228,13 +5232,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
 
     startTs = taosGetTimestampMs();
 
-    if (recOfBatch == 0) {
-      errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n",
-              pThreadInfo->threadID, __func__, __LINE__,
-              recOfBatch);
-      errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n");
-      goto free_of_interlace;
-    }
     int64_t affectedRows = execInsert(pThreadInfo, buffer, recOfBatch);
 
     endTs = taosGetTimestampMs();
@@ -6455,7 +6452,8 @@ static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int c
     return;
   }
 
-  appendResultToFile(res, (char*)param);
+  if (param)
+    appendResultToFile(res, (char*)param);
   // tao_unscribe() will free result.
 }
 
@@ -6515,59 +6513,63 @@ static void *superSubscribe(void *sarg) {
     return NULL;
   }
 
-  //int64_t st = 0;
-  //int64_t et = 0;
-  do {
-    //if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) < g_queryInfo.specifiedQueryInfo.queryInterval) {
-    //  taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval- (et - st)); // ms
-    //  //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to);
-    //}
-
-    //st = taosGetTimestampMs();
-    char topic[32] = {0};
-    for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
-      sprintf(topic, "taosdemo-subscribe-%d", i);
+  char topic[32] = {0};
+  for (uint64_t i = pThreadInfo->start_table_from;
+          i <= pThreadInfo->end_table_to; i++) {
+    for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
+      sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%d", i, j);
       memset(subSqlstr,0,sizeof(subSqlstr));
-      replaceChildTblName(g_queryInfo.superQueryInfo.sql[i], subSqlstr, i);
+      replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], subSqlstr, i);
       char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
-      if (g_queryInfo.superQueryInfo.result[i][0] != 0) {
+      if (g_queryInfo.superQueryInfo.result[j][0] != 0) {
         sprintf(tmpFile, "%s-%d",
-                g_queryInfo.superQueryInfo.result[i], pThreadInfo->threadID);
+                g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID);
       }
-      tsub[i] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, tmpFile);
-      if (NULL == tsub[i]) {
+
+      uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j;
+      debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n",
+              __func__, __LINE__, subSeq, subSqlstr);
+      tsub[subSeq] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, tmpFile);
+      if (NULL == tsub[subSeq]) {
         taos_close(pThreadInfo->taos);
         return NULL;
       }
     }
-    //et = taosGetTimestampMs();
-    //printf("========thread[%"PRIu64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0);
-  } while(0);
+  }
 
   // start loop to consume result
   TAOS_RES* res = NULL;
   while(1) {
-    for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
-      if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
-        continue;
-      }
+    for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) {
+      for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
+        if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) {
+            continue;
+        }
 
-      res = taos_consume(tsub[i]);
-      if (res) {
-        char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
-        if (g_queryInfo.superQueryInfo.result[i][0] != 0) {
-          sprintf(tmpFile, "%s-%d",
-                  g_queryInfo.superQueryInfo.result[i],
+        uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j;
+        taosMsleep(100); // ms
+        res = taos_consume(tsub[subSeq]);
+        if (res) {
+            char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
+            if (g_queryInfo.superQueryInfo.result[j][0] != 0) {
+                sprintf(tmpFile, "%s-%d",
+                  g_queryInfo.superQueryInfo.result[j],
                   pThreadInfo->threadID);
+                appendResultToFile(res, tmpFile);
+            }
         }
-        appendResultToFile(res, tmpFile);
       }
     }
   }
   taos_free_result(res);
 
-  for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) {
-    taos_unsubscribe(tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress);
+  for (uint64_t i = pThreadInfo->start_table_from;
+          i <= pThreadInfo->end_table_to; i++) {
+    for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) {
+        uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j;
+        taos_unsubscribe(tsub[subSeq],
+                g_queryInfo.superQueryInfo.subscribeKeepProgress);
+    }
   }
 
   taos_close(pThreadInfo->taos);
@@ -6605,17 +6607,8 @@ static void *specifiedSubscribe(void *sarg) {
     return NULL;
   }
 
-  //int64_t st = 0;
-  //int64_t et = 0;
-  do {
-    //if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) < g_queryInfo.specifiedQueryInfo.queryInterval) {
-    //  taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval- (et - st)); // ms
-    //  //printf("========sleep duration:%"PRIu64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to);
-    //}
-
-    //st = taosGetTimestampMs();
-    char topic[32] = {0};
-    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
+  char topic[32] = {0};
+  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) {
       sprintf(topic, "taosdemo-subscribe-%d", i);
       char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
       if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) {
@@ -6628,11 +6621,7 @@ static void *specifiedSubscribe(void *sarg) {
         taos_close(pThreadInfo->taos);
         return NULL;
       }
-    }
-    //et = taosGetTimestampMs();
-    //printf("========thread[%"PRIu64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0);
-  } while(0);
-
+  }
   // start loop to consume result
   TAOS_RES* res = NULL;
   while(1) {
@@ -6641,14 +6630,15 @@ static void *specifiedSubscribe(void *sarg) {
         continue;
       }
 
+      taosMsleep(1000); // ms
       res = taos_consume(tsub[i]);
       if (res) {
         char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
         if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) {
           sprintf(tmpFile, "%s-%d",
                   g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID);
+          appendResultToFile(res, tmpFile);
         }
-        appendResultToFile(res, tmpFile);
       }
     }
   }
@@ -6697,31 +6687,35 @@ static int subscribeTestProcess() {
 
   pthread_t  *pids = NULL;
   threadInfo *infos = NULL;
-  //==== create sub threads for query from super table
-  if ((g_queryInfo.specifiedQueryInfo.sqlCount <= 0) ||
-          (g_queryInfo.specifiedQueryInfo.concurrent <= 0)) {
-    errorPrint("%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n",
+  //==== create sub threads for query for specified table
+  if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) {
+    printf("%s() LN%d, sepcified query sqlCount %"PRIu64".\n",
               __func__, __LINE__,
-              g_queryInfo.specifiedQueryInfo.sqlCount,
-              g_queryInfo.specifiedQueryInfo.concurrent);
-    exit(-1);
-  }
+              g_queryInfo.specifiedQueryInfo.sqlCount);
+  } else {
+    if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) {
+        errorPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n",
+              __func__, __LINE__,
+              g_queryInfo.specifiedQueryInfo.sqlCount);
+        exit(-1);
+    }
 
-  pids  = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t));
-  infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo));
-  if ((NULL == pids) || (NULL == infos)) {
-      errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
-      exit(-1);
-  }
+    pids  = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t));
+    infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo));
+    if ((NULL == pids) || (NULL == infos)) {
+        errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__);
+        exit(-1);
+    }
 
-  for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) {
+    for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) {
       threadInfo *t_info = infos + i;
       t_info->threadID = i;
       t_info->taos = NULL;  // TODO: workaround to use separate taos connection;
       pthread_create(pids + i, NULL, specifiedSubscribe, t_info);
+    }
   }
 
-  //==== create sub threads for query from sub table
+  //==== create sub threads for super table query
   pthread_t  *pidsOfSub  = NULL;
   threadInfo *infosOfSub = NULL;
   if ((g_queryInfo.superQueryInfo.sqlCount > 0)

src/mnode/inc/mnodeDef.h

@@ -249,7 +249,7 @@ typedef struct SAcctObj {
 } SAcctObj;
 
 typedef struct {
-  char     db[TSDB_DB_NAME_LEN];
+  char     db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
   int8_t   type;
   int16_t  numOfColumns;
   int32_t  index;

src/mnode/src/mnodeShow.c

@@ -129,7 +129,7 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
   SShowObj *pShow = calloc(1, showObjSize);
   pShow->type       = pShowMsg->type;
   pShow->payloadLen = htons(pShowMsg->payloadLen);
-  tstrncpy(pShow->db, pShowMsg->db, TSDB_DB_NAME_LEN);
+  tstrncpy(pShow->db, pShowMsg->db, TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN);
   memcpy(pShow->payload, pShowMsg->payload, pShow->payloadLen);
 
   pShow = mnodePutShowObj(pShow);

src/os/tests/test.cpp

@@ -4,7 +4,7 @@
 #include <iostream>
 
 #include "taos.h"
-#include "tstoken.h"
+#include "ttoken.h"
 #include "tutil.h"
 
 int main(int argc, char** argv) {

src/query/inc/qSqlparser.h

@@ -22,8 +22,8 @@ extern "C" {
 
 #include "taos.h"
 #include "taosmsg.h"
-#include "tstoken.h"
 #include "tstrbuild.h"
+#include "ttoken.h"
 #include "tvariant.h"
 
 #define ParseTOKENTYPE SStrToken

src/query/src/qSqlParser.c

@@ -13,13 +13,13 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
 #include "qSqlparser.h"
+#include "os.h"
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tcmdtype.h"
-#include "tstoken.h"
 #include "tstrbuild.h"
+#include "ttoken.h"
 #include "ttokendef.h"
 #include "tutil.h"
 
@@ -38,7 +38,7 @@ SSqlInfo qSqlParse(const char *pStr) {
       goto abort_parse;
     }
 
-    t0.n = tSQLGetToken((char *)&pStr[i], &t0.type);
+    t0.n = tGetToken((char *)&pStr[i], &t0.type);
     t0.z = (char *)(pStr + i);
     i += t0.n;
 

src/query/src/sql.c

@@ -25,14 +25,14 @@
 #include <stdio.h>
 /************ Begin %include sections from the grammar ************************/
 
+#include <assert.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <assert.h>
-#include <stdbool.h>
 #include "qSqlparser.h"
 #include "tcmdtype.h"
-#include "tstoken.h"
+#include "ttoken.h"
 #include "ttokendef.h"
 #include "tutil.h"
 #include "tvariant.h"

src/query/tests/tsBufTest.cpp

@@ -3,10 +3,10 @@
 #include <cassert>
 #include <iostream>
 
+#include "qTsbuf.h"
 #include "taos.h"
 #include "tsdb.h"
-#include "qTsbuf.h"
-#include "tstoken.h"
+#include "ttoken.h"
 #include "tutil.h"
 
 namespace {

src/query/tests/unitTest.cpp

@@ -21,7 +21,7 @@ int32_t testValidateName(char* name) {
   token.n = strlen(name);
   token.type = 0;
 
-  tSQLGetToken(name, &token.type);
+  tGetToken(name, &token.type);
   return tscValidateName(&token);
 }
 }
@@ -691,32 +691,32 @@ TEST(testCase, tGetToken_Test) {
   char* s = ".123 ";
   uint32_t type = 0;
 
-  int32_t len = tSQLGetToken(s, &type);
+  int32_t len = tGetToken(s, &type);
   EXPECT_EQ(type, TK_FLOAT);
   EXPECT_EQ(len, strlen(s) - 1);
 
   char s1[] = "1.123e10 ";
-  len = tSQLGetToken(s1, &type);
+  len = tGetToken(s1, &type);
   EXPECT_EQ(type, TK_FLOAT);
   EXPECT_EQ(len, strlen(s1) - 1);
 
   char s4[] = "0xff ";
-  len = tSQLGetToken(s4, &type);
+  len = tGetToken(s4, &type);
   EXPECT_EQ(type, TK_HEX);
   EXPECT_EQ(len, strlen(s4) - 1);
 
   // invalid data type
   char s2[] = "e10 ";
-  len = tSQLGetToken(s2, &type);
+  len = tGetToken(s2, &type);
   EXPECT_FALSE(type == TK_FLOAT);
 
   char s3[] = "1.1.1.1";
-  len = tSQLGetToken(s3, &type);
+  len = tGetToken(s3, &type);
   EXPECT_EQ(type, TK_IPTOKEN);
   EXPECT_EQ(len, strlen(s3));
 
   char s5[] = "0x ";
-  len = tSQLGetToken(s5, &type);
+  len = tGetToken(s5, &type);
   EXPECT_FALSE(type == TK_HEX);
 }
 

src/util/inc/tstoken.h → src/util/inc/ttoken.h

@@ -37,13 +37,25 @@ typedef struct SStrToken {
   char    *z;
 } SStrToken;
 
+extern const char escapeChar[];
+
+/**
+ * check if it is a number or not
+ * @param pToken
+ * @return
+ */
+#define isNumber(tk) \
+((tk)->type == TK_INTEGER || (tk)->type == TK_FLOAT || (tk)->type == TK_HEX || (tk)->type == TK_BIN)
+
+#define GET_ESCAPE_CHAR(c) (escapeChar[(uint8_t)(c)])
+
 /**
  * tokenizer for sql string
  * @param z
  * @param tokenType
  * @return
  */
-uint32_t tSQLGetToken(char *z, uint32_t *tokenType);
+uint32_t tGetToken(char *z, uint32_t *tokenType);
 
 /**
  * enhanced tokenizer for sql string.
@@ -61,21 +73,12 @@ SStrToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr);
  * @param len
  * @return
  */
-bool isKeyWord(const char *z, int32_t len);
-
-/**
- * check if it is a number or not
- * @param pToken
- * @return
- */
-#define isNumber(tk) \
-((tk)->type == TK_INTEGER || (tk)->type == TK_FLOAT || (tk)->type == TK_HEX || (tk)->type == TK_BIN)
-
+bool taosIsKeyWordToken(const char *z, int32_t len);
 
 /**
  * check if it is a token or not
- * @param pToken
- * @return        token type, if it is not a number, TK_ILLEGAL will return
+ * @param   pToken
+ * @return  token type, if it is not a number, TK_ILLEGAL will return
  */
 static FORCE_INLINE int32_t tGetNumericStringType(const SStrToken* pToken) {
   const char* z = pToken->z;

src/query/src/qTokenizer.c → src/util/src/ttokenizer.c

@@ -18,7 +18,7 @@
 #include "hash.h"
 #include "hashfunc.h"
 #include "taosdef.h"
-#include "tstoken.h"
+#include "ttoken.h"
 #include "ttokendef.h"
 #include "tutil.h"
 
@@ -232,6 +232,18 @@ static const char isIdChar[] = {
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
 };
 
+const char escapeChar[] = {
+    /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, /* 0x */
+    0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, /* 1x */
+    0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, /* 2x */
+    0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, /* 3x */
+    0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F,/* 4x */
+    0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,/* 5x */
+    0x60, 0x07, 0x08, 0x63, 0x64, 0x65, 0x0C, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x0A, 0x6F,/* 6x */
+    0x70, 0x71, 0x0D, 0x73, 0x09, 0x75, 0x0B, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F,/* 7x */
+};
+
 static void* keywordHashTable = NULL;
 
 static void doInitKeywordsTable(void) {
@@ -247,7 +259,7 @@ static void doInitKeywordsTable(void) {
 
 static pthread_once_t keywordsHashTableInit = PTHREAD_ONCE_INIT;
 
-int tSQLKeywordCode(const char* z, int n) {
+static int32_t tKeywordCode(const char* z, int n) {
   pthread_once(&keywordsHashTableInit, doInitKeywordsTable);
   
   char key[512] = {0};
@@ -271,7 +283,7 @@ int tSQLKeywordCode(const char* z, int n) {
  * Return the length of the token that begins at z[0].
  * Store the token type in *type before returning.
  */
-uint32_t tSQLGetToken(char* z, uint32_t* tokenId) {
+uint32_t tGetToken(char* z, uint32_t* tokenId) {
   uint32_t i;
   switch (*z) {
     case ' ':
@@ -403,12 +415,12 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenId) {
       int  delim = z[0];
       bool strEnd = false;
       for (i = 1; z[i]; i++) {
-        if (z[i] == '\\') { 
+        if (z[i] == '\\') {   // ignore the escaped character that follows this backslash
           i++;
           continue;
         }
         
-        if (z[i] == delim ) {
+        if (z[i] == delim) {
           if (z[i + 1] == delim) {
             i++;
           } else {
@@ -551,7 +563,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenId) {
       }
       for (i = 1; ((z[i] & 0x80) == 0) && isIdChar[(uint8_t) z[i]]; i++) {
       }
-      *tokenId = tSQLKeywordCode(z, i);
+      *tokenId = tKeywordCode(z, i);
       return i;
     }
   }
@@ -606,7 +618,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) {
       t = str[++(*i)];
     }
 
-    t0.n = tSQLGetToken(&str[*i], &t0.type);
+    t0.n = tGetToken(&str[*i], &t0.type);
     break;
 
     // not support user specfied ignored symbol list
@@ -635,7 +647,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) {
 
   // support parse the 'db.tbl' format, notes: There should be no space on either side of the dot!
   if ('.' == str[*i + t0.n]) {
-    len = tSQLGetToken(&str[*i + t0.n + 1], &type);
+    len = tGetToken(&str[*i + t0.n + 1], &type);
 
     // only id and string are valid
     if ((TK_STRING != t0.type) && (TK_ID != t0.type)) {
@@ -650,7 +662,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) {
   } else {
     // support parse the -/+number format
     if ((isPrevOptr) && (t0.type == TK_MINUS || t0.type == TK_PLUS)) {
-      len = tSQLGetToken(&str[*i + t0.n], &type);
+      len = tGetToken(&str[*i + t0.n], &type);
       if (type == TK_INTEGER || type == TK_FLOAT) {
         t0.type = type;
         t0.n += len;
@@ -664,7 +676,9 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) {
   return t0;
 }
 
-bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); }
+bool taosIsKeyWordToken(const char* z, int32_t len) {
+  return (tKeywordCode((char*)z, len) != TK_ID);
+}
 
 void taosCleanupKeywordsTable() {
   void* m = keywordHashTable;

tests/pytest/fulltest.sh

@@ -305,6 +305,7 @@ python3 ./test.py -f functions/function_top.py -r 1
 python3 ./test.py -f functions/function_twa.py -r 1
 python3 ./test.py -f functions/function_twa_test2.py
 python3 ./test.py -f functions/function_stddev_td2555.py
+python3 ./test.py -f functions/showOfflineThresholdIs864000.py
 python3 ./test.py -f insert/metadataUpdate.py
 python3 ./test.py -f query/last_cache.py
 python3 ./test.py -f query/last_row_cache.py

tests/pytest/functions/showOfflineThresholdIs864000.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to execute {__file__}")
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.query("show variables")
+        tdSql.checkData(51, 1, 864000)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

tests/pytest/insert/nchar-unicode.py

@@ -57,12 +57,15 @@ class TDTestCase:
 
         # https://www.ltg.ed.ac.uk/~richard/unicode-sample.html
         # Basic Latin
-        data = r'! # $ % & ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~'
+        data = r'! # $ % & ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \\ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~'
         tdLog.info("insert Basic Latin %d length data: %s" % (len(data), data))
         tdSql.execute("insert into tb values (now, '%s')" % data)
         tdSql.query("select * from tb")
         tdSql.checkRows(3)
+
+        data = data.replace('\\\\', '\\')
         tdSql.checkData(2, 1, data)
+        # tdSql.execute("insert into tb values(now, 'abc')")
 
         # Latin-1 Supplement
         data = ' ¡ ¢ £ ¤ ¥ ¦ § ¨ © ª « ¬ ­ ® ¯ ° ± ² ³ ´ µ ¶ · ¸ ¹ º » ¼ ½ ¾ ¿ À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß à á â ã ä å æ ç è é ê ë ì í î ï ð ñ ò ó ô õ ö ÷ ø ù ú û ü ý þ ÿ'

tests/pytest/tools/taosdemoAllTest/speciQueryRestful.json

@@ -8,7 +8,7 @@
     "confirm_parameter_prompt": "no",
     "databases": "db",
     "query_times": 2,
-    "query_mode": "restful",
+    "query_mode": "rest",
     "specified_table_query": {
       "query_interval": 1,
       "concurrent": 3,
@@ -35,4 +35,4 @@
       ]
     }
   }
-  
+  
\ No newline at end of file

tests/script/general/parser/binary_escapeCharacter.sim

@@ -93,5 +93,15 @@ if $data41 != @udp005@ then
   print "[ERROR] expect: udp005, act:$data41"
 endi
 
+print ---------------------> TD-3967
+sql insert into tb values(now, '\\abc\\\\');
+sql insert into tb values(now, '\\abc\\\\');
+sql insert into tb values(now, '\\\\');
+
+print ------------->sim bug
+# sql_error insert into tb values(now, '\\\');
+sql_error insert into tb values(now, '\');
+#sql_error insert into tb values(now, '\\\n');
+sql insert into tb values(now, '\n');
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file

tests/script/general/parser/testSuite.sim

@@ -54,5 +54,9 @@ run general/parser/timestamp.sim
 run general/parser/sliding.sim
 run general/parser/function.sim
 run general/parser/stableOp.sim
+
 run general/parser/having.sim
-run general/parser/having_child.sim
+run general/parser/having_child.sim
\ No newline at end of file
+run general/parser/slimit_alter_tags.sim
+run general/parser/binary_escapeCharacter.sim
+