Merge branch 'develop' into feature/query

Jenkinsfile

@@ -41,6 +41,7 @@ def pre_test(){
     sh '''
     killall -9 taosd ||echo "no taosd running"
     killall -9 gdb || echo "no gdb running"
+    killall -9 python3.8 || echo "no python program running"
     cd ${WKC}
     git reset --hard HEAD~10 >/dev/null
     '''
@@ -223,12 +224,14 @@ pipeline {
           steps {
             pre_test()
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                timeout(time: 60, unit: 'MINUTES'){
                   sh '''
                   cd ${WKC}/tests/pytest
                   ./crash_gen.sh -a -p -t 4 -s 2000
                   '''
                 }
-
+            }
+            timeout(time: 60, unit: 'MINUTES'){
               sh '''
               cd ${WKC}/tests/pytest
               rm -rf /var/lib/taos/*
@@ -241,6 +244,7 @@ pipeline {
               rm -rf /var/log/taos/*
               ./handle_taosd_val_log.sh
               '''
+            }
             timeout(time: 45, unit: 'MINUTES'){
                 sh '''
                 date

documentation20/cn/02.getting-started/docs.md

@@ -126,7 +126,7 @@ taos> source <filename>;
 $ taosdemo
 ```
 
-该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupdId，groupdId 被设置为 1 到 10， location 被设置为 "beijing" 或者 "shanghai"。
+该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "beijing" 或者 "shanghai"。
 
 执行这条命令大概需要几分钟，最后共插入 1 亿条记录。
 
@@ -150,10 +150,10 @@ taos> select avg(current), max(voltage), min(phase) from test.meters;
 taos> select count(*) from test.meters where location="beijing";
 ```
 
-- 查询 groupdId=10 的所有记录的平均值、最大值、最小值等：
+- 查询 groupId=10 的所有记录的平均值、最大值、最小值等：
 
 ```mysql
-taos> select avg(current), max(voltage), min(phase) from test.meters where groupdId=10;
+taos> select avg(current), max(voltage), min(phase) from test.meters where groupId=10;
 ```
 
 - 对表 d10 按 10s 进行平均值、最大值和最小值聚合统计：

documentation20/cn/04.model/docs.md

@@ -33,7 +33,7 @@ USE power;
 一个物联网系统，往往存在多种类型的设备，比如对于电网，存在智能电表、变压器、母线、开关等等。为便于多表之间的聚合，使用TDengine, 需要对每个类型的数据采集点创建一超级表。以表一中的智能电表为例，可以使用如下的SQL命令创建超级表：
 
 ```mysql
-CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int);
+CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);
 ```
 
 **注意：**这一指令中的 STABLE 关键字，在 2.0.15 之前的版本中需写作 TABLE 。

documentation20/cn/07.advanced-features/docs.md

@@ -17,7 +17,7 @@ TDengine提供的连续查询与普通流计算中的时间窗口计算具有以
 下面以智能电表场景为例介绍连续查询的具体使用方法。假设我们通过下列SQL语句创建了超级表和子表：
 
 ```sql
-create table meters (ts timestamp, current float, voltage int, phase float) tags (location binary(64), groupdId int);
+create table meters (ts timestamp, current float, voltage int, phase float) tags (location binary(64), groupId int);
 create table D1001 using meters tags ("Beijing.Chaoyang", 2);
 create table D1002 using meters tags ("Beijing.Haidian", 2);
 ...

documentation20/cn/12.taos-sql/docs.md

@@ -34,7 +34,7 @@ taos> DESCRIBE meters;
 - 时间格式为 ```YYYY-MM-DD HH:mm:ss.MS```，默认时间分辨率为毫秒。比如：```2017-08-12 18:25:58.128```
 - 内部函数 now 是客户端的当前时间
 - 插入记录时，如果时间戳为 now，插入数据时使用提交这条记录的客户端的当前时间
-- Epoch Time：时间戳也可以是一个长整数，表示从 1970-01-01 08:00:00.000 开始的毫秒数
+- Epoch Time：时间戳也可以是一个长整数，表示从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始的毫秒数（相应地，如果所在 Database 的时间精度设置为“微秒”，则长整型格式的时间戳含义也就对应于从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始的微秒数）
 - 时间可以加减，比如 now-2h，表明查询时刻向前推 2 个小时（最近 2 小时）。数字后面的时间单位可以是 u(微秒)、a(毫秒)、s(秒)、m(分)、h(小时)、d(天)、w(周)。 比如 `select * from t1 where ts > now-2w and ts <= now-1w`，表示查询两周前整整一周的数据。在指定降频操作（down sampling）的时间窗口（interval）时，时间单位还可以使用 n(自然月) 和 y(自然年)。
 
 TDengine 缺省的时间戳是毫秒精度，但通过在 CREATE DATABASE 时传递的 PRECISION 参数就可以支持微秒。
@@ -414,13 +414,13 @@ INSERT INTO
     ```
     也可以在自动建表时，只是指定部分 TAGS 列的取值，未被指定的 TAGS 列将置为 NULL。例如：  
     ```mysql
-    INSERT INTO d21001 USING meters (groupdId) TAGS (2) VALUES ('2021-07-13 14:06:33.196', 10.15, 217, 0.33);
+    INSERT INTO d21001 USING meters (groupId) TAGS (2) VALUES ('2021-07-13 14:06:33.196', 10.15, 217, 0.33);
     ```
     自动建表语法也支持在一条语句中向多个表插入记录。例如：  
     ```mysql
     INSERT INTO d21001 USING meters TAGS ('Beijing.Chaoyang', 2) VALUES ('2021-07-13 14:06:34.630', 10.2, 219, 0.32) ('2021-07-13 14:06:35.779', 10.15, 217, 0.33)
-                d21002 USING meters (groupdId) TAGS (2) VALUES ('2021-07-13 14:06:34.255', 10.15, 217, 0.33)
-                d21003 USING meters (groupdId) TAGS (2) (ts, current, phase) VALUES ('2021-07-13 14:06:34.255', 10.27, 0.31);
+                d21002 USING meters (groupId) TAGS (2) VALUES ('2021-07-13 14:06:34.255', 10.15, 217, 0.33)
+                d21003 USING meters (groupId) TAGS (2) (ts, current, phase) VALUES ('2021-07-13 14:06:34.255', 10.27, 0.31);
     ```
     **说明：**在 2.0.20.5 版本之前，在使用自动建表语法并指定列时，子表的列名必须紧跟在子表名称后面，而不能如例子里那样放在 TAGS 和 VALUES 之间。从 2.0.20.5 版本开始，两种写法都可以，但不能在一条 SQL 语句中混用，否则会报语法错误。
 

documentation20/en/04.model/docs.md

@@ -32,7 +32,7 @@ Replace the database operating in the current connection with “power”, other
 An IoT system often has many types of devices, such as smart meters, transformers, buses, switches, etc. for power grids. In order to facilitate aggregation among multiple tables, using TDengine, it is necessary to create a STable for each type of data collection point. Taking the smart meter in Table 1 as an example, you can use the following SQL command to create a STable:
 
 ```mysql
-CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int);
+CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);
 ```
 
 **Note:** The STABLE keyword in this instruction needs to be written as TABLE in versions before 2.0.15.

documentation20/en/07.advanced-features/docs.md

@@ -17,7 +17,7 @@ The continuous query provided by TDengine differs from the time window calculati
 The following is an example of the smart meter scenario to introduce the specific use of continuous query. Suppose we create a STables and sub-tables through the following SQL statement:
 
 ```sql
-create table meters (ts timestamp, current float, voltage int, phase float) tags (location binary(64), groupdId int);
+create table meters (ts timestamp, current float, voltage int, phase float) tags (location binary(64), groupId int);
 create table D1001 using meters tags ("Beijing.Chaoyang", 2);
 create table D1002 using meters tags ("Beijing.Haidian", 2);
 ...

src/client/inc/tscUtil.h

@@ -340,10 +340,11 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild);
 uint32_t tscGetTableMetaSize(STableMeta* pTableMeta);
 CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta);
 uint32_t tscGetTableMetaMaxSize();
-int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf);
+int32_t tscCreateTableMetaFromSTableMeta(STableMeta** pChild, const char* name, size_t *tableMetaCapacity);
 STableMeta* tscTableMetaDup(STableMeta* pTableMeta);
 SVgroupsInfo* tscVgroupsInfoDup(SVgroupsInfo* pVgroupsInfo);
 
+int32_t tscGetTagFilterSerializeLen(SQueryInfo* pQueryInfo);
 int32_t tscGetColFilterSerializeLen(SQueryInfo* pQueryInfo);
 int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr);
 void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, STableGroupInfo* pTableGroupInfo, SOperatorInfo* pOperator, char* sql, void* addr, int32_t stage, uint64_t qId);

src/client/src/tscParseLineProtocol.c

@@ -458,9 +458,9 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSm
   schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false);
   schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false);
 
-  uint32_t size = tscGetTableMetaMaxSize();
-  STableMeta* tableMeta = calloc(1, size);
-  taosHashGetClone(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, tableMeta);
+  size_t size = 0; 
+  STableMeta* tableMeta = NULL; 
+  taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void **)&tableMeta, &size);
 
   tstrncpy(schema->sTableName, tableName, strlen(tableName)+1);
   schema->precision = tableMeta->tableInfo.precision;
@@ -1417,7 +1417,7 @@ static bool isTimeStamp(char *pVal, uint16_t len, SMLTimeStampType *tsType) {
   return false;
 }
 
-static bool convertStrToNumber(TAOS_SML_KV *pVal, char*str) {
+static bool convertStrToNumber(TAOS_SML_KV *pVal, char*str, SSmlLinesInfo* info) {
   errno = 0;
   uint8_t type = pVal->type;
   int16_t length = pVal->length;
@@ -1436,7 +1436,7 @@ static bool convertStrToNumber(TAOS_SML_KV *pVal, char*str) {
   }
 
   if (errno == ERANGE) {
-    tscError("Converted number out of range");
+    tscError("SML:0x%"PRIx64" Convert number(%s) out of range", info->id, str);
     return false;
   }
 
@@ -1518,7 +1518,7 @@ static bool convertStrToNumber(TAOS_SML_KV *pVal, char*str) {
 }
 //len does not include '\0' from value.
 static bool convertSmlValueType(TAOS_SML_KV *pVal, char *value,
-                                uint16_t len) {
+                                uint16_t len, SSmlLinesInfo* info) {
   if (len <= 0) {
     return false;
   }
@@ -1528,7 +1528,7 @@ static bool convertSmlValueType(TAOS_SML_KV *pVal, char *value,
     pVal->type = TSDB_DATA_TYPE_TINYINT;
     pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
     value[len - 2] = '\0';
-    if (!isValidInteger(value) || !convertStrToNumber(pVal, value)) {
+    if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) {
       return false;
     }
     return true;
@@ -1537,7 +1537,7 @@ static bool convertSmlValueType(TAOS_SML_KV *pVal, char *value,
     pVal->type = TSDB_DATA_TYPE_UTINYINT;
     pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
     value[len - 2] = '\0';
-    if (!isValidInteger(value) || !convertStrToNumber(pVal, value)) {
+    if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) {
       return false;
     }
     return true;
@@ -1546,7 +1546,7 @@ static bool convertSmlValueType(TAOS_SML_KV *pVal, char *value,
     pVal->type = TSDB_DATA_TYPE_SMALLINT;
     pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
     value[len - 3] = '\0';
-    if (!isValidInteger(value) || !convertStrToNumber(pVal, value)) {
+    if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) {
       return false;
     }
     return true;
@@ -1555,7 +1555,7 @@ static bool convertSmlValueType(TAOS_SML_KV *pVal, char *value,
     pVal->type = TSDB_DATA_TYPE_USMALLINT;
     pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
     value[len - 3] = '\0';
-    if (!isValidInteger(value) || !convertStrToNumber(pVal, value)) {
+    if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) {
       return false;
     }
     return true;
@@ -1564,7 +1564,7 @@ static bool convertSmlValueType(TAOS_SML_KV *pVal, char *value,
     pVal->type = TSDB_DATA_TYPE_INT;
     pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
     value[len - 3] = '\0';
-    if (!isValidInteger(value) || !convertStrToNumber(pVal, value)) {
+    if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) {
       return false;
     }
     return true;
@@ -1573,7 +1573,7 @@ static bool convertSmlValueType(TAOS_SML_KV *pVal, char *value,
     pVal->type = TSDB_DATA_TYPE_UINT;
     pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
     value[len - 3] = '\0';
-    if (!isValidInteger(value) || !convertStrToNumber(pVal, value)) {
+    if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) {
       return false;
     }
     return true;
@@ -1582,7 +1582,7 @@ static bool convertSmlValueType(TAOS_SML_KV *pVal, char *value,
     pVal->type = TSDB_DATA_TYPE_BIGINT;
     pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
     value[len - 3] = '\0';
-    if (!isValidInteger(value) || !convertStrToNumber(pVal, value)) {
+    if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) {
       return false;
     }
     return true;
@@ -1591,7 +1591,7 @@ static bool convertSmlValueType(TAOS_SML_KV *pVal, char *value,
     pVal->type = TSDB_DATA_TYPE_UBIGINT;
     pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
     value[len - 3] = '\0';
-    if (!isValidInteger(value) || !convertStrToNumber(pVal, value)) {
+    if (!isValidInteger(value) || !convertStrToNumber(pVal, value, info)) {
       return false;
     }
     return true;
@@ -1601,7 +1601,7 @@ static bool convertSmlValueType(TAOS_SML_KV *pVal, char *value,
     pVal->type = TSDB_DATA_TYPE_FLOAT;
     pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
     value[len - 3] = '\0';
-    if (!isValidFloat(value) || !convertStrToNumber(pVal, value)) {
+    if (!isValidFloat(value) || !convertStrToNumber(pVal, value, info)) {
       return false;
     }
     return true;
@@ -1610,7 +1610,7 @@ static bool convertSmlValueType(TAOS_SML_KV *pVal, char *value,
     pVal->type = TSDB_DATA_TYPE_DOUBLE;
     pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
     value[len - 3] = '\0';
-    if (!isValidFloat(value) || !convertStrToNumber(pVal, value)) {
+    if (!isValidFloat(value) || !convertStrToNumber(pVal, value, info)) {
       return false;
     }
     return true;
@@ -1646,7 +1646,7 @@ static bool convertSmlValueType(TAOS_SML_KV *pVal, char *value,
   if (isValidInteger(value) || isValidFloat(value)) {
     pVal->type = TSDB_DATA_TYPE_FLOAT;
     pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
-    if (!convertStrToNumber(pVal, value)) {
+    if (!convertStrToNumber(pVal, value, info)) {
       return false;
     }
     return true;
@@ -1702,7 +1702,7 @@ static int32_t getTimeStampValue(char *value, uint16_t len,
 }
 
 static int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value,
-                                   uint16_t len) {
+                                   uint16_t len, SSmlLinesInfo* info) {
   int32_t ret;
   SMLTimeStampType type;
   int64_t tsVal;
@@ -1715,7 +1715,7 @@ static int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value,
   if (ret) {
     return ret;
   }
-  tscDebug("Timestamp after conversion:%"PRId64, tsVal);
+  tscDebug("SML:0x%"PRIx64"Timestamp after conversion:%"PRId64, info->id, tsVal);
 
   pVal->type = TSDB_DATA_TYPE_TIMESTAMP;
   pVal->length = (int16_t)tDataTypes[pVal->type].bytes;
@@ -1724,7 +1724,7 @@ static int32_t convertSmlTimeStamp(TAOS_SML_KV *pVal, char *value,
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t parseSmlTimeStamp(TAOS_SML_KV **pTS, const char **index) {
+static int32_t parseSmlTimeStamp(TAOS_SML_KV **pTS, const char **index, SSmlLinesInfo* info) {
   const char *start, *cur;
   int32_t ret = TSDB_CODE_SUCCESS;
   int len = 0;
@@ -1744,7 +1744,7 @@ static int32_t parseSmlTimeStamp(TAOS_SML_KV **pTS, const char **index) {
     memcpy(value, start, len);
   }
 
-  ret = convertSmlTimeStamp(*pTS, value, len);
+  ret = convertSmlTimeStamp(*pTS, value, len, info);
   if (ret) {
     free(value);
     free(*pTS);
@@ -1757,7 +1757,7 @@ static int32_t parseSmlTimeStamp(TAOS_SML_KV **pTS, const char **index) {
   return ret;
 }
 
-static bool checkDuplicateKey(char *key, SHashObj *pHash) {
+static bool checkDuplicateKey(char *key, SHashObj *pHash, SSmlLinesInfo* info) {
   char *val = NULL;
   char *cur = key;
   char keyLower[TSDB_COL_NAME_LEN];
@@ -1771,7 +1771,7 @@ static bool checkDuplicateKey(char *key, SHashObj *pHash) {
 
   val = taosHashGet(pHash, keyLower, keyLen);
   if (val) {
-    tscError("Duplicate key:%s", keyLower);
+    tscError("SML:0x%"PRIx64" Duplicate key detected:%s", info->id, keyLower);
     return true;
   }
 
@@ -1781,19 +1781,19 @@ static bool checkDuplicateKey(char *key, SHashObj *pHash) {
   return false;
 }
 
-static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash) {
+static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash, SSmlLinesInfo* info) {
   const char *cur = *index;
   char key[TSDB_COL_NAME_LEN + 1];  // +1 to avoid key[len] over write
   uint16_t len = 0;
 
   //key field cannot start with digit
   if (isdigit(*cur)) {
-    tscError("Tag key cannnot start with digit\n");
+    tscError("SML:0x%"PRIx64" Tag key cannnot start with digit", info->id);
     return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
   }
   while (*cur != '\0') {
     if (len > TSDB_COL_NAME_LEN) {
-      tscDebug("Key field cannot exceeds 65 characters");
+      tscError("SML:0x%"PRIx64" Key field cannot exceeds 65 characters", info->id);
       return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
     }
     //unescaped '=' identifies a tag key
@@ -1810,20 +1810,20 @@ static int32_t parseSmlKey(TAOS_SML_KV *pKV, const char **index, SHashObj *pHash
   }
   key[len] = '\0';
 
-  if (checkDuplicateKey(key, pHash)) {
+  if (checkDuplicateKey(key, pHash, info)) {
     return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
   }
 
   pKV->key = calloc(len + 1, 1);
   memcpy(pKV->key, key, len + 1);
-  //tscDebug("Key:%s|len:%d", pKV->key, len);
+  //tscDebug("SML:0x%"PRIx64" Key:%s|len:%d", info->id, pKV->key, len);
   *index = cur + 1;
   return TSDB_CODE_SUCCESS;
 }
 
 
 static bool parseSmlValue(TAOS_SML_KV *pKV, const char **index,
-                          bool *is_last_kv) {
+                          bool *is_last_kv, SSmlLinesInfo* info) {
   const char *start, *cur;
   char *value = NULL;
   uint16_t len = 0;
@@ -1847,7 +1847,9 @@ static bool parseSmlValue(TAOS_SML_KV *pKV, const char **index,
   value = calloc(len + 1, 1);
   memcpy(value, start, len);
   value[len] = '\0';
-  if (!convertSmlValueType(pKV, value, len)) {
+  if (!convertSmlValueType(pKV, value, len, info)) {
+    tscError("SML:0x%"PRIx64" Failed to convert sml value string(%s) to any type",
+            info->id, value);
     //free previous alocated key field
     free(pKV->key);
     pKV->key = NULL;
@@ -1861,7 +1863,7 @@ static bool parseSmlValue(TAOS_SML_KV *pKV, const char **index,
 }
 
 static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index,
-                                   uint8_t *has_tags) {
+                                   uint8_t *has_tags, SSmlLinesInfo* info) {
   const char *cur = *index;
   uint16_t len = 0;
 
@@ -1870,7 +1872,7 @@ static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
   if (isdigit(*cur)) {
-    tscError("Measurement field cannnot start with digit");
+    tscError("SML:0x%"PRIx64" Measurement field cannnot start with digit", info->id);
     free(pSml->stableName);
     pSml->stableName = NULL;
     return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
@@ -1878,7 +1880,7 @@ static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index
 
   while (*cur != '\0') {
     if (len > TSDB_TABLE_NAME_LEN) {
-      tscError("Measurement field cannot exceeds 193 characters");
+      tscError("SML:0x%"PRIx64" Measurement field cannot exceeds 193 characters", info->id);
       free(pSml->stableName);
       pSml->stableName = NULL;
       return TSDB_CODE_TSC_LINE_SYNTAX_ERROR;
@@ -1902,7 +1904,7 @@ static int32_t parseSmlMeasurement(TAOS_SML_DATA_POINT *pSml, const char **index
   }
   pSml->stableName[len] = '\0';
   *index = cur + 1;
-  tscDebug("Stable name in measurement:%s|len:%d", pSml->stableName, len);
+  tscDebug("SML:0x%"PRIx64" Stable name in measurement:%s|len:%d", info->id, pSml->stableName, len);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -1921,7 +1923,8 @@ static int32_t isValidChildTableName(const char *pTbName, int16_t len) {
 
 static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs,
                                const char **index, bool isField,
-                               TAOS_SML_DATA_POINT* smlData, SHashObj *pHash) {
+                               TAOS_SML_DATA_POINT* smlData, SHashObj *pHash,
+                               SSmlLinesInfo* info) {
   const char *cur = *index;
   int32_t ret = TSDB_CODE_SUCCESS;
   TAOS_SML_KV *pkv;
@@ -1941,14 +1944,14 @@ static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs,
   }
 
   while (*cur != '\0') {
-    ret = parseSmlKey(pkv, &cur, pHash);
+    ret = parseSmlKey(pkv, &cur, pHash, info);
     if (ret) {
-      tscError("Unable to parse key field");
+      tscError("SML:0x%"PRIx64" Unable to parse key", info->id);
       goto error;
     }
-    ret = parseSmlValue(pkv, &cur, &is_last_kv);
+    ret = parseSmlValue(pkv, &cur, &is_last_kv, info);
     if (ret) {
-      tscError("Unable to parse value field");
+      tscError("SML:0x%"PRIx64" Unable to parse value", info->id);
       goto error;
     }
     if (!isField &&
@@ -1966,7 +1969,6 @@ static int32_t parseSmlKvPairs(TAOS_SML_KV **pKVs, int *num_kvs,
       *num_kvs += 1;
     }
     if (is_last_kv) {
-      //tscDebug("last key-value field detected");
       goto done;
     }
 
@@ -2024,50 +2026,50 @@ static void moveTimeStampToFirstKv(TAOS_SML_DATA_POINT** smlData, TAOS_SML_KV *t
   free(ts);
 }
 
-int32_t tscParseLine(const char* sql, TAOS_SML_DATA_POINT* smlData) {
+int32_t tscParseLine(const char* sql, TAOS_SML_DATA_POINT* smlData, SSmlLinesInfo* info) {
   const char* index = sql;
   int32_t ret = TSDB_CODE_SUCCESS;
   uint8_t has_tags = 0;
   TAOS_SML_KV *timestamp = NULL;
   SHashObj *keyHashTable = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false);
 
-  ret = parseSmlMeasurement(smlData, &index, &has_tags);
+  ret = parseSmlMeasurement(smlData, &index, &has_tags, info);
   if (ret) {
-    tscError("Unable to parse measurement");
+    tscError("SML:0x%"PRIx64" Unable to parse measurement", info->id);
     taosHashCleanup(keyHashTable);
     return ret;
   }
-  tscDebug("Parse measurement finished, has_tags:%d", has_tags);
+  tscDebug("SML:0x%"PRIx64" Parse measurement finished, has_tags:%d", info->id, has_tags);
 
   //Parse Tags
   if (has_tags) {
-    ret = parseSmlKvPairs(&smlData->tags, &smlData->tagNum, &index, false, smlData, keyHashTable);
+    ret = parseSmlKvPairs(&smlData->tags, &smlData->tagNum, &index, false, smlData, keyHashTable, info);
     if (ret) {
-      tscError("Unable to parse tag");
+      tscError("SML:0x%"PRIx64" Unable to parse tag", info->id);
       taosHashCleanup(keyHashTable);
       return ret;
     }
   }
-  tscDebug("Parse tags finished, num of tags:%d", smlData->tagNum);
+  tscDebug("SML:0x%"PRIx64" Parse tags finished, num of tags:%d", info->id, smlData->tagNum);
 
   //Parse fields
-  ret = parseSmlKvPairs(&smlData->fields, &smlData->fieldNum, &index, true, smlData, keyHashTable);
+  ret = parseSmlKvPairs(&smlData->fields, &smlData->fieldNum, &index, true, smlData, keyHashTable, info);
   if (ret) {
-    tscError("Unable to parse field");
+    tscError("SML:0x%"PRIx64" Unable to parse field", info->id);
     taosHashCleanup(keyHashTable);
     return ret;
   }
-  tscDebug("Parse fields finished, num of fields:%d", smlData->fieldNum);
+  tscDebug("SML:0x%"PRIx64" Parse fields finished, num of fields:%d", info->id, smlData->fieldNum);
   taosHashCleanup(keyHashTable);
 
   //Parse timestamp
-  ret = parseSmlTimeStamp(&timestamp, &index);
+  ret = parseSmlTimeStamp(&timestamp, &index, info);
   if (ret) {
-    tscError("Unable to parse timestamp");
+    tscError("SML:0x%"PRIx64" Unable to parse timestamp", info->id);
     return ret;
   }
   moveTimeStampToFirstKv(&smlData, timestamp);
-  tscDebug("Parse timestamp finished");
+  tscDebug("SML:0x%"PRIx64" Parse timestamp finished", info->id);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -2104,7 +2106,7 @@ void destroySmlDataPoint(TAOS_SML_DATA_POINT* point) {
 int32_t tscParseLines(char* lines[], int numLines, SArray* points, SArray* failedLines, SSmlLinesInfo* info) {
   for (int32_t i = 0; i < numLines; ++i) {
     TAOS_SML_DATA_POINT point = {0};
-    int32_t code = tscParseLine(lines[i], &point);
+    int32_t code = tscParseLine(lines[i], &point, info);
     if (code != TSDB_CODE_SUCCESS) {
       tscError("SML:0x%"PRIx64" data point line parse failed. line %d : %s", info->id, i, lines[i]);
       destroySmlDataPoint(&point);

src/client/src/tscProfile.c

@@ -255,10 +255,25 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
     pQdesc->qId = htobe64(pSql->res.qId);
     pQdesc->sqlObjId = htobe64(pSql->self);
     pQdesc->pid = pHeartbeat->pid;
-    if (pSql->cmd.pQueryInfo->stableQuery == true) {
+    pQdesc->stableQuery = pSql->cmd.pQueryInfo->stableQuery;
     pQdesc->numOfSub = pSql->subState.numOfSub;
+
+    char *p = pQdesc->subSqlInfo;
+    int32_t remainLen = sizeof(pQdesc->subSqlInfo);
+    if (pQdesc->numOfSub == 0) {
+      snprintf(p, remainLen, "N/A");
     } else {
-      pQdesc->numOfSub = 1;
+      int32_t len;
+      for (int32_t i = 0; i < pQdesc->numOfSub; ++i) {
+        len = snprintf(p, remainLen, "[%d]0x%" PRIx64 "(%c) ", i,
+                        pSql->pSubs[i]->self,
+                        pSql->subState.states[i] ? 'C' : 'I');
+        if (len > remainLen) {
+          break;
+        }
+        remainLen -= len;
+        p += len;
+      }
     }
     pQdesc->numOfSub = htonl(pQdesc->numOfSub);
 

src/client/src/tscSQLParser.c

@@ -900,6 +900,7 @@ int32_t tscValidateSqlInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
         SSqlNode* pSqlNode = taosArrayGetP(pInfo->list, i);
 
         tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size);
+
 //        normalizeSqlNode(pSqlNode); // normalize the column name in each function
         if ((code = validateSqlNode(pSql, pSqlNode, pQueryInfo)) != TSDB_CODE_SUCCESS) {
           return code;
@@ -2024,7 +2025,6 @@ static SUdfInfo* isValidUdf(SArray* pUdfInfo, const char* name, int32_t len) {
     tscError("udfinfo is null");
     return NULL;
   }
-  
   size_t t = taosArrayGetSize(pUdfInfo);
   for(int32_t i = 0; i < t; ++i) {
     SUdfInfo* pUdf = taosArrayGet(pUdfInfo, i);
@@ -5884,6 +5884,11 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
     tVariantListItem* pItem = taosArrayGet(pVarList, 1);
     SSchema* pTagsSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex);
+
+    if (IS_VAR_DATA_TYPE(pTagsSchema->type) && (pItem->pVar.nLen > pTagsSchema->bytes * TSDB_NCHAR_SIZE)) {
+      return invalidOperationMsg(pMsg, msg14);
+    }
+
     pAlterSQL->tagData.data = calloc(1, pTagsSchema->bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
 
     if (tVariantDump(&pItem->pVar, pAlterSQL->tagData.data, pTagsSchema->type, true) != TSDB_CODE_SUCCESS) {
@@ -8089,6 +8094,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) {
   SArray* pVgroupList   = NULL;
   SArray* plist         = NULL;
   STableMeta* pTableMeta = NULL;
+  size_t    tableMetaCapacity = 0;
   SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
 
   pCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
@@ -8115,18 +8121,14 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     }
   }
 
-  uint32_t maxSize = tscGetTableMetaMaxSize();
   char     name[TSDB_TABLE_FNAME_LEN] = {0};
 
-  assert(maxSize < 80 * TSDB_MAX_COLUMNS);
-  if (!pSql->pBuf) {
-    if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) {
-      code = TSDB_CODE_TSC_OUT_OF_MEMORY;
-      goto _end;
-    }
-  }
-
-  pTableMeta = calloc(1, maxSize);
+  //if (!pSql->pBuf) {
+  //  if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) {
+  //    code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+  //    goto _end;
+  //  }
+  //}
 
   plist = taosArrayInit(4, POINTER_BYTES);
   pVgroupList = taosArrayInit(4, POINTER_BYTES);
@@ -8140,16 +8142,16 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     tNameExtractFullName(pname, name);
 
     size_t len = strlen(name);
-    memset(pTableMeta, 0, maxSize);
-    taosHashGetClone(tscTableMetaMap, name, len, NULL, pTableMeta);
       
-    if (pTableMeta->id.uid > 0) {
+    taosHashGetCloneExt(tscTableMetaMap, name, len, NULL,  (void **)&pTableMeta, &tableMetaCapacity);
+
+    if (pTableMeta && pTableMeta->id.uid > 0) {
       tscDebug("0x%"PRIx64" retrieve table meta %s from local buf", pSql->self, name);
 
       // avoid mem leak, may should update pTableMeta
       void* pVgroupIdList = NULL;
       if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
-        code = tscCreateTableMetaFromSTableMeta(pTableMeta, name, pSql->pBuf);
+        code = tscCreateTableMetaFromSTableMeta((STableMeta **)(&pTableMeta), name, &tableMetaCapacity);
 
         // create the child table meta from super table failed, try load it from mnode
         if (code != TSDB_CODE_SUCCESS) {
@@ -8305,6 +8307,7 @@ static int32_t doLoadAllTableMeta(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNod
     STableMetaVgroupInfo* p = taosHashGet(pCmd->pTableMetaMap, fname, strnlen(fname, TSDB_TABLE_FNAME_LEN));
 
     pTableMetaInfo->pTableMeta        = tscTableMetaDup(p->pTableMeta);
+    pTableMetaInfo->tableMetaCapacity = tscGetTableMetaSize(pTableMetaInfo->pTableMeta);
     assert(pTableMetaInfo->pTableMeta != NULL);
 
     if (p->vgroupIdList != NULL) {
@@ -8405,6 +8408,7 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pS
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
   pTableMetaInfo1->pTableMeta        = extractTempTableMetaFromSubquery(pSub);
+  pTableMetaInfo1->tableMetaCapacity = tscGetTableMetaSize(pTableMetaInfo1->pTableMeta);
 
   if (subInfo->aliasName.n > 0) {
     if (subInfo->aliasName.n >= TSDB_TABLE_FNAME_LEN) {
@@ -8453,6 +8457,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
   const char* msg6 = "not support stddev/percentile/interp in the outer query yet";
   const char* msg7 = "derivative/twa/irate requires timestamp column exists in subquery";
   const char* msg8 = "condition missing for join query";
+  const char* msg9 = "not support 3 level select";
 
   int32_t code = TSDB_CODE_SUCCESS;
 
@@ -8483,6 +8488,13 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf
     // parse the subquery in the first place
     int32_t numOfSub = (int32_t)taosArrayGetSize(pSqlNode->from->list);
     for (int32_t i = 0; i < numOfSub; ++i) {
+      // check if there is 3 level select
+      SRelElementPair* subInfo = taosArrayGet(pSqlNode->from->list, i);
+      SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0);
+      if (p->from->type == SQL_NODE_FROM_SUBQUERY){
+        return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9);
+      }
+
       code = doValidateSubquery(pSqlNode, i, pSql, pQueryInfo, tscGetErrorMsgPayload(pCmd));
       if (code != TSDB_CODE_SUCCESS) {
         return code;

src/client/src/tscServer.c

@@ -676,6 +676,8 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) {
 
   int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo));
   int32_t srcColFilterSize = tscGetColFilterSerializeLen(pQueryInfo);
+  int32_t srcTagFilterSize = tscGetTagFilterSerializeLen(pQueryInfo);
+
   size_t  numOfExprs = tscNumOfExprs(pQueryInfo);
   int32_t exprSize = (int32_t)(sizeof(SSqlExpr) * numOfExprs * 2);
 
@@ -696,8 +698,8 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) {
     tableSerialize = totalTables * sizeof(STableIdInfo);
   }
 
-  return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + exprSize + tsBufSize +
-         tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen;
+  return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize +
+         exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen;
 }
 
 static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, char *pMsg,
@@ -1033,7 +1035,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
     SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid);
     if (pCond != NULL && pCond->cond != NULL) {
-      pQueryMsg->tagCondLen = htons(pCond->len);
+      pQueryMsg->tagCondLen = htonl(pCond->len);
       memcpy(pMsg, pCond->cond, pCond->len);
 
       pMsg += pCond->len;
@@ -2838,43 +2840,22 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
 int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) {
   assert(tIsValidName(&pTableMetaInfo->name));
 
-  uint32_t size = tscGetTableMetaMaxSize();
-  if (pTableMetaInfo->pTableMeta == NULL) {
-    pTableMetaInfo->pTableMeta    = calloc(1, size);
-    pTableMetaInfo->tableMetaSize = size;
-  } else if (pTableMetaInfo->tableMetaSize < size) {
-    char *tmp = realloc(pTableMetaInfo->pTableMeta, size);
-    if (tmp == NULL) {
-      return TSDB_CODE_TSC_OUT_OF_MEMORY;
-    }
-    pTableMetaInfo->pTableMeta = (STableMeta *)tmp;
-  }
-
-  memset(pTableMetaInfo->pTableMeta, 0, size);
-  pTableMetaInfo->tableMetaSize = size;
-
-  pTableMetaInfo->pTableMeta->tableType = -1;
-  pTableMetaInfo->pTableMeta->tableInfo.numOfColumns  = -1;
-
   char name[TSDB_TABLE_FNAME_LEN] = {0};
   tNameExtractFullName(&pTableMetaInfo->name, name);
 
   size_t len = strlen(name);
-  taosHashGetClone(tscTableMetaMap, name, len, NULL, pTableMetaInfo->pTableMeta);
-
-  // TODO resize the tableMeta
-  assert(size < 80 * TSDB_MAX_COLUMNS);
-  if (!pSql->pBuf) {
-    if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) {
-      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  if (pTableMetaInfo->tableMetaCapacity != 0) {
+    if (pTableMetaInfo->pTableMeta != NULL) {
+      memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity);
     } 
   } 
+  taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity);
 
   STableMeta* pMeta = pTableMetaInfo->pTableMeta;
-  if (pMeta->id.uid > 0) {
+  if (pMeta && pMeta->id.uid > 0) {
     // in case of child table, here only get the
     if (pMeta->tableType == TSDB_CHILD_TABLE) {
-      int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, pSql->pBuf);
+      int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity);
       if (code != TSDB_CODE_SUCCESS) {
         return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate);
       }

src/client/src/tscUtil.c

@@ -1658,6 +1658,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
 
     pTableMetaInfo->pTableMeta    = tscTableMetaDup(pDataBlock->pTableMeta);
     pTableMetaInfo->tableMetaSize = tscGetTableMetaSize(pDataBlock->pTableMeta);
+    pTableMetaInfo->tableMetaCapacity =  (size_t)(pTableMetaInfo->tableMetaSize);
   }
 
   /*
@@ -3415,6 +3416,8 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableM
   } else {
     pTableMetaInfo->tableMetaSize = tscGetTableMetaSize(pTableMeta);
   }
+  pTableMetaInfo->tableMetaCapacity = (size_t)(pTableMetaInfo->tableMetaSize);
+  
 
   if (vgroupList != NULL) {
     pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList);
@@ -4447,24 +4450,36 @@ CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta) {
   return cMeta;
 }
 
-int32_t tscCreateTableMetaFromSTableMeta(STableMeta* pChild, const char* name, void* buf) {
-  assert(pChild != NULL && buf != NULL);
+int32_t tscCreateTableMetaFromSTableMeta(STableMeta** ppChild, const char* name, size_t *tableMetaCapacity) {
+  assert(*ppChild != NULL);
+
+  STableMeta* p    = NULL;
+  size_t      sz   = 0;
+  STableMeta* pChild = *ppChild;
    
-  STableMeta* p = buf;
-  taosHashGetClone(tscTableMetaMap, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, p);
+  taosHashGetCloneExt(tscTableMetaMap, pChild->sTableName, strnlen(pChild->sTableName, TSDB_TABLE_FNAME_LEN), NULL, (void **)&p, &sz);
 
   // tableMeta exists, build child table meta according to the super table meta
   // the uid need to be checked in addition to the general name of the super table.
-  if (p->id.uid > 0 && pChild->suid == p->id.uid) {
+  if (p && p->id.uid > 0 && pChild->suid == p->id.uid) {
+
+    int32_t totalBytes    = (p->tableInfo.numOfColumns + p->tableInfo.numOfTags) * sizeof(SSchema);
+    int32_t tableMetaSize =  sizeof(STableMeta)  + totalBytes;
+    if (*tableMetaCapacity < tableMetaSize) {
+      pChild = realloc(pChild, tableMetaSize); 
+      *tableMetaCapacity = (size_t)tableMetaSize;
+    }
+
     pChild->sversion = p->sversion;
     pChild->tversion = p->tversion;
-
     memcpy(&pChild->tableInfo, &p->tableInfo, sizeof(STableComInfo));
-    int32_t total = pChild->tableInfo.numOfColumns + pChild->tableInfo.numOfTags;
+    memcpy(pChild->schema, p->schema, totalBytes);
 
-    memcpy(pChild->schema, p->schema, sizeof(SSchema) *total);
+    *ppChild = pChild;
+    tfree(p);
     return TSDB_CODE_SUCCESS;
   } else { // super table has been removed, current tableMeta is also expired. remove it here
+    tfree(p);
     taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN));
     return -1;
   }
@@ -4687,6 +4702,21 @@ int32_t tscGetColFilterSerializeLen(SQueryInfo* pQueryInfo) {
   return len;
 }
 
+int32_t tscGetTagFilterSerializeLen(SQueryInfo* pQueryInfo) {
+  // serialize tag column query condition
+  if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) {
+    STagCond* pTagCond = &pQueryInfo->tagCond;
+
+    STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+    STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
+    SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid);
+    if (pCond != NULL && pCond->cond != NULL) {
+      return pCond->len;
+    }
+  }
+  return 0;
+}
+
 int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr) {
   memset(pQueryAttr, 0, sizeof(SQueryAttr));
 

src/common/inc/tdataformat.h

@@ -325,7 +325,9 @@ typedef struct SDataCol {
 #define isAllRowsNull(pCol) ((pCol)->len == 0)
 static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
 
-void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
+int tdAllocMemForCol(SDataCol *pCol, int maxPoints);
+
+void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints);
 void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints);
 void dataColSetOffset(SDataCol *pCol, int nEle);
 
@@ -358,12 +360,10 @@ typedef struct {
   int maxRowSize;
   int maxCols;    // max number of columns
   int maxPoints;  // max number of points
-  int bufSize;
 
   int       numOfRows;
   int       numOfCols;  // Total number of cols
   int       sversion;   // TODO: set sversion
-  void *    buf;
   SDataCol *cols;
 } SDataCols;
 

src/common/src/tdataformat.c

@@ -22,6 +22,29 @@
 static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
                                int limit2, int tRows, bool forceSetNull);
 
+//TODO: change caller to use return val
+int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
+  int spaceNeeded = pCol->bytes * maxPoints;
+  if(IS_VAR_DATA_TYPE(pCol->type)) {
+    spaceNeeded += sizeof(VarDataOffsetT) * maxPoints;
+  }
+  if(pCol->spaceSize < spaceNeeded) {
+    void* ptr = realloc(pCol->pData, spaceNeeded);
+    if(ptr == NULL) {
+      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)pCol->spaceSize,
+             strerror(errno));
+      return -1;
+    } else {
+      pCol->pData = ptr;
+      pCol->spaceSize = spaceNeeded;
+    }
+  }
+  if(IS_VAR_DATA_TYPE(pCol->type)) {
+    pCol->dataOff = POINTER_SHIFT(pCol->pData, pCol->bytes * maxPoints);
+  }
+  return 0;
+}
+
 /**
  * Duplicate the schema and return a new object
  */
@@ -207,24 +230,13 @@ SMemRow tdMemRowDup(SMemRow row) {
   return trow;
 }
 
-void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) {
+void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) {
   pDataCol->type = colType(pCol);
   pDataCol->colId = colColId(pCol);
   pDataCol->bytes = colBytes(pCol);
   pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE;
 
   pDataCol->len = 0;
-  if (IS_VAR_DATA_TYPE(pDataCol->type)) {
-    pDataCol->dataOff = (VarDataOffsetT *)(*pBuf);
-    pDataCol->pData = POINTER_SHIFT(*pBuf, sizeof(VarDataOffsetT) * maxPoints);
-    pDataCol->spaceSize = pDataCol->bytes * maxPoints;
-    *pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize + sizeof(VarDataOffsetT) * maxPoints);
-  } else {
-    pDataCol->spaceSize = pDataCol->bytes * maxPoints;
-    pDataCol->dataOff = NULL;
-    pDataCol->pData = *pBuf;
-    *pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize);
-  }
 }
 // value from timestamp should be TKEY here instead of TSKEY
 void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) {
@@ -239,6 +251,8 @@ void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxP
     if (numOfRows > 0) {
       // Find the first not null value, fill all previouse values as NULL
       dataColSetNEleNull(pCol, numOfRows, maxPoints);
+    } else {
+      tdAllocMemForCol(pCol, maxPoints);
     }
   }
 
@@ -257,13 +271,14 @@ void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxP
 }
 
 bool isNEleNull(SDataCol *pCol, int nEle) {
+  if(isAllRowsNull(pCol)) return true;
   for (int i = 0; i < nEle; i++) {
     if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false;
   }
   return true;
 }
 
-FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
+static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->dataOff[index] = pCol->len;
     char *ptr = POINTER_SHIFT(pCol->pData, pCol->len);
@@ -276,6 +291,7 @@ FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
 }
 
 void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) {
+  tdAllocMemForCol(pCol, maxPoints);
 
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->len = 0;
@@ -319,56 +335,63 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
       tdFreeDataCols(pCols);
       return NULL;
     }
+    int i;
+    for(i = 0; i < maxCols; i++) {
+      pCols->cols[i].spaceSize = 0;
+      pCols->cols[i].len = 0;
+      pCols->cols[i].pData = NULL;
+      pCols->cols[i].dataOff = NULL;
+    }
 
     pCols->maxCols = maxCols;
   }
 
   pCols->maxRowSize = maxRowSize;
-  pCols->bufSize = maxRowSize * maxRows;
 
-  if (pCols->bufSize > 0) {
-    pCols->buf = malloc(pCols->bufSize);
-    if (pCols->buf == NULL) {
-      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols,
-             strerror(errno));
-      tdFreeDataCols(pCols);
-      return NULL;
-    }
-  }
 
   return pCols;
 }
 
 int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
-  if (schemaNCols(pSchema) > pCols->maxCols) {
+  int i;
+  int oldMaxCols = pCols->maxCols;
+  if (schemaNCols(pSchema) > oldMaxCols) {
     pCols->maxCols = schemaNCols(pSchema);
     pCols->cols = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols);
     if (pCols->cols == NULL) return -1;
+    for(i = oldMaxCols; i < pCols->maxCols; i++) {
+      pCols->cols[i].pData = NULL;
+      pCols->cols[i].dataOff = NULL;
+      pCols->cols[i].spaceSize = 0;
+    }
   }
 
   if (schemaTLen(pSchema) > pCols->maxRowSize) {
     pCols->maxRowSize = schemaTLen(pSchema);
-    pCols->bufSize = schemaTLen(pSchema) * pCols->maxPoints;
-    pCols->buf = realloc(pCols->buf, pCols->bufSize);
-    if (pCols->buf == NULL) return -1;
   }
 
   tdResetDataCols(pCols);
   pCols->numOfCols = schemaNCols(pSchema);
 
-  void *ptr = pCols->buf;
-  for (int i = 0; i < schemaNCols(pSchema); i++) {
-    dataColInit(pCols->cols + i, schemaColAt(pSchema, i), &ptr, pCols->maxPoints);
-    ASSERT((char *)ptr - (char *)(pCols->buf) <= pCols->bufSize);
+  for (i = 0; i < schemaNCols(pSchema); i++) {
+    dataColInit(pCols->cols + i, schemaColAt(pSchema, i), pCols->maxPoints);
   }
   
   return 0;
 }
 
 SDataCols *tdFreeDataCols(SDataCols *pCols) {
+  int i;
   if (pCols) {
-    tfree(pCols->buf);
-    tfree(pCols->cols);
+    if(pCols->cols) {
+      int maxCols = pCols->maxCols;
+      for(i = 0; i < maxCols; i++) {
+        SDataCol *pCol = &pCols->cols[i];
+        tfree(pCol->pData);
+      }
+      free(pCols->cols);
+      pCols->cols = NULL;
+    }
     free(pCols);
   }
   return NULL;
@@ -388,21 +411,14 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
     pRet->cols[i].bytes = pDataCols->cols[i].bytes;
     pRet->cols[i].offset = pDataCols->cols[i].offset;
 
-    pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize;
-    pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf)));
-
-    if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
-      ASSERT(pDataCols->cols[i].dataOff != NULL);
-      pRet->cols[i].dataOff =
-          (int32_t *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].dataOff) - (char *)(pDataCols->buf)));
-    }
-
     if (keepData) {
-      pRet->cols[i].len = pDataCols->cols[i].len;
       if (pDataCols->cols[i].len > 0) {
+        tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints);
+        pRet->cols[i].len = pDataCols->cols[i].len;
         memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len);
         if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) {
-          memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints);
+          int dataOffSize = sizeof(VarDataOffsetT) * pDataCols->maxPoints;
+          memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize);
         }
       }
     }
@@ -426,20 +442,9 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols
   int rcol = 0;
   int dcol = 0;
 
-  if (dataRowDeleted(row)) {
-    for (; dcol < pCols->numOfCols; dcol++) {
-      SDataCol *pDataCol = &(pCols->cols[dcol]);
-      if (dcol == 0) {
-        dataColAppendVal(pDataCol, dataRowTuple(row), pCols->numOfRows, pCols->maxPoints);
-      } else {
-        dataColSetNullAt(pDataCol, pCols->numOfRows);
-      }
-    }
-  } else {
   while (dcol < pCols->numOfCols) {
     SDataCol *pDataCol = &(pCols->cols[dcol]);
     if (rcol >= schemaNCols(pSchema)) {
-        // dataColSetNullAt(pDataCol, pCols->numOfRows);
       dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
       dcol++;
       continue;
@@ -455,13 +460,11 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols
       rcol++;
     } else {
       if(forceSetNull) {
-          //dataColSetNullAt(pDataCol, pCols->numOfRows);
         dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
       }
       dcol++;
     }
   }
-  }
   pCols->numOfRows++;
 }
 
@@ -471,22 +474,11 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo
   int rcol = 0;
   int dcol = 0;
 
-  if (kvRowDeleted(row)) {
-    for (; dcol < pCols->numOfCols; dcol++) {
-      SDataCol *pDataCol = &(pCols->cols[dcol]);
-      if (dcol == 0) {
-        dataColAppendVal(pDataCol, kvRowValues(row), pCols->numOfRows, pCols->maxPoints);
-      } else {
-        dataColSetNullAt(pDataCol, pCols->numOfRows);
-      }
-    }
-  } else {
   int nRowCols = kvRowNCols(row);
 
   while (dcol < pCols->numOfCols) {
     SDataCol *pDataCol = &(pCols->cols[dcol]);
     if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) {
-        // dataColSetNullAt(pDataCol, pCols->numOfRows);
       dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
       ++dcol;
       continue;
@@ -503,13 +495,11 @@ static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCo
       ++rcol;
     } else {
       if (forceSetNull) {
-          // dataColSetNullAt(pDataCol, pCols->numOfRows);
         dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
       }
       ++dcol;
     }
   }
-  }
   pCols->numOfRows++;
 }
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java

@@ -77,8 +77,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
     }
 
     public boolean supportsMixedCaseIdentifiers() throws SQLException {
-        //像database、table这些对象的标识符，在存储时是否采用大小写混合的模式
-        return false;
+        return false;   //像database、table这些对象的标识符，在存储时是否采用大小写混合的模式
     }
 
     public boolean storesUpperCaseIdentifiers() throws SQLException {
@@ -514,7 +513,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col6 = new ColumnMetaData();
         col6.setColIndex(colIndex);
         col6.setColName("TYPE_CAT");
-        col6.setColType(Types.NCHAR);
+        col6.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col6;
     }
 
@@ -522,7 +521,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col7 = new ColumnMetaData();
         col7.setColIndex(colIndex);
         col7.setColName("TYPE_SCHEM");
-        col7.setColType(Types.NCHAR);
+        col7.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col7;
     }
 
@@ -530,7 +529,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col8 = new ColumnMetaData();
         col8.setColIndex(colIndex);
         col8.setColName("TYPE_NAME");
-        col8.setColType(Types.NCHAR);
+        col8.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col8;
     }
 
@@ -538,7 +537,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col9 = new ColumnMetaData();
         col9.setColIndex(colIndex);
         col9.setColName("SELF_REFERENCING_COL_NAME");
-        col9.setColType(Types.NCHAR);
+        col9.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col9;
     }
 
@@ -546,7 +545,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col10 = new ColumnMetaData();
         col10.setColIndex(colIndex);
         col10.setColName("REF_GENERATION");
-        col10.setColType(Types.NCHAR);
+        col10.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col10;
     }
 
@@ -592,7 +591,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col4 = new ColumnMetaData();
         col4.setColIndex(colIndex);
         col4.setColName("TABLE_TYPE");
-        col4.setColType(Types.NCHAR);
+        col4.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col4;
     }
 
@@ -734,7 +733,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col1 = new ColumnMetaData();
         col1.setColIndex(colIndex);
         col1.setColName("TABLE_CAT");
-        col1.setColType(Types.NCHAR);
+        col1.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col1;
     }
 
@@ -742,7 +741,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col2 = new ColumnMetaData();
         col2.setColIndex(colIndex);
         col2.setColName("TABLE_SCHEM");
-        col2.setColType(Types.NCHAR);
+        col2.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col2;
     }
 
@@ -751,7 +750,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         col3.setColIndex(colIndex);
         col3.setColName("TABLE_NAME");
         col3.setColSize(193);
-        col3.setColType(Types.NCHAR);
+        col3.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col3;
     }
 
@@ -760,7 +759,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         col4.setColIndex(colIndex);
         col4.setColName("COLUMN_NAME");
         col4.setColSize(65);
-        col4.setColType(Types.NCHAR);
+        col4.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col4;
     }
 
@@ -768,7 +767,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col5 = new ColumnMetaData();
         col5.setColIndex(colIndex);
         col5.setColName("DATA_TYPE");
-        col5.setColType(Types.INTEGER);
+        col5.setColType(TSDBConstants.TSDB_DATA_TYPE_INT);
         return col5;
     }
 
@@ -776,7 +775,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col7 = new ColumnMetaData();
         col7.setColIndex(7);
         col7.setColName("COLUMN_SIZE");
-        col7.setColType(Types.INTEGER);
+        col7.setColType(TSDBConstants.TSDB_DATA_TYPE_INT);
         return col7;
     }
 
@@ -791,7 +790,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col9 = new ColumnMetaData();
         col9.setColIndex(9);
         col9.setColName("DECIMAL_DIGITS");
-        col9.setColType(Types.INTEGER);
+        col9.setColType(TSDBConstants.TSDB_DATA_TYPE_INT);
         return col9;
     }
 
@@ -799,7 +798,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col10 = new ColumnMetaData();
         col10.setColIndex(10);
         col10.setColName("NUM_PREC_RADIX");
-        col10.setColType(Types.INTEGER);
+        col10.setColType(TSDBConstants.TSDB_DATA_TYPE_INT);
         return col10;
     }
 
@@ -807,7 +806,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col11 = new ColumnMetaData();
         col11.setColIndex(11);
         col11.setColName("NULLABLE");
-        col11.setColType(Types.INTEGER);
+        col11.setColType(TSDBConstants.TSDB_DATA_TYPE_INT);
         return col11;
     }
 
@@ -815,7 +814,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col12 = new ColumnMetaData();
         col12.setColIndex(colIndex);
         col12.setColName("REMARKS");
-        col12.setColType(Types.NCHAR);
+        col12.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col12;
     }
 
@@ -823,7 +822,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col13 = new ColumnMetaData();
         col13.setColIndex(13);
         col13.setColName("COLUMN_DEF");
-        col13.setColType(Types.NCHAR);
+        col13.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col13;
     }
 
@@ -831,7 +830,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col14 = new ColumnMetaData();
         col14.setColIndex(14);
         col14.setColName("SQL_DATA_TYPE");
-        col14.setColType(Types.INTEGER);
+        col14.setColType(TSDBConstants.TSDB_DATA_TYPE_INT);
         return col14;
     }
 
@@ -839,7 +838,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col15 = new ColumnMetaData();
         col15.setColIndex(15);
         col15.setColName("SQL_DATETIME_SUB");
-        col15.setColType(Types.INTEGER);
+        col15.setColType(TSDBConstants.TSDB_DATA_TYPE_INT);
         return col15;
     }
 
@@ -847,7 +846,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col16 = new ColumnMetaData();
         col16.setColIndex(16);
         col16.setColName("CHAR_OCTET_LENGTH");
-        col16.setColType(Types.INTEGER);
+        col16.setColType(TSDBConstants.TSDB_DATA_TYPE_INT);
         return col16;
     }
 
@@ -855,7 +854,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col17 = new ColumnMetaData();
         col17.setColIndex(17);
         col17.setColName("ORDINAL_POSITION");
-        col17.setColType(Types.INTEGER);
+        col17.setColType(TSDBConstants.TSDB_DATA_TYPE_INT);
         return col17;
     }
 
@@ -863,7 +862,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col18 = new ColumnMetaData();
         col18.setColIndex(18);
         col18.setColName("IS_NULLABLE");
-        col18.setColType(Types.NCHAR);
+        col18.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col18;
     }
 
@@ -871,7 +870,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col19 = new ColumnMetaData();
         col19.setColIndex(19);
         col19.setColName("SCOPE_CATALOG");
-        col19.setColType(Types.NCHAR);
+        col19.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col19;
     }
 
@@ -879,7 +878,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col20 = new ColumnMetaData();
         col20.setColIndex(20);
         col20.setColName("SCOPE_SCHEMA");
-        col20.setColType(Types.NCHAR);
+        col20.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col20;
     }
 
@@ -887,7 +886,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col21 = new ColumnMetaData();
         col21.setColIndex(21);
         col21.setColName("SCOPE_TABLE");
-        col21.setColType(Types.NCHAR);
+        col21.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col21;
     }
 
@@ -903,7 +902,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col23 = new ColumnMetaData();
         col23.setColIndex(23);
         col23.setColName("IS_AUTOINCREMENT");
-        col23.setColType(Types.NCHAR);
+        col23.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col23;
     }
 
@@ -911,7 +910,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col24 = new ColumnMetaData();
         col24.setColIndex(24);
         col24.setColName("IS_GENERATEDCOLUMN");
-        col24.setColType(Types.NCHAR);
+        col24.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col24;
     }
 
@@ -1205,7 +1204,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col5 = new ColumnMetaData();
         col5.setColIndex(colIndex);
         col5.setColName("KEY_SEQ");
-        col5.setColType(Types.SMALLINT);
+        col5.setColType(TSDBConstants.TSDB_DATA_TYPE_SMALLINT);
         return col5;
     }
 
@@ -1213,7 +1212,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col6 = new ColumnMetaData();
         col6.setColIndex(colIndex);
         col6.setColName("PK_NAME");
-        col6.setColType(Types.NCHAR);
+        col6.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col6;
     }
 
@@ -1275,7 +1274,7 @@ public abstract class AbstractDatabaseMetaData extends WrapperImpl implements Da
         ColumnMetaData col4 = new ColumnMetaData();
         col4.setColIndex(colIndex);
         col4.setColName("SUPERTABLE_NAME");
-        col4.setColType(Types.NCHAR);
+        col4.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR);
         return col4;
     }
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java

@@ -16,7 +16,7 @@ package com.taosdata.jdbc;
 
 public class ColumnMetaData {
 
-    private int colType = 0;
+    private int colType = 0;         //taosType
     private String colName = null;
     private int colSize = -1;
     private int colIndex = 0;

src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java

@@ -68,71 +68,61 @@ public class DatabaseMetaDataResultSet extends AbstractResultSet {
     @Override
     public String getString(int columnIndex) throws SQLException {
         int colType = columnMetaDataList.get(columnIndex - 1).getColType();
-        int nativeType = TSDBConstants.jdbcType2TaosType(colType);
-        return rowCursor.getString(columnIndex, nativeType);
+        return rowCursor.getString(columnIndex, colType);
     }
 
     @Override
     public boolean getBoolean(int columnIndex) throws SQLException {
         int colType = columnMetaDataList.get(columnIndex - 1).getColType();
-        int nativeType = TSDBConstants.jdbcType2TaosType(colType);
-        return rowCursor.getBoolean(columnIndex, nativeType);
+        return rowCursor.getBoolean(columnIndex, colType);
     }
 
     @Override
     public byte getByte(int columnIndex) throws SQLException {
         int colType = columnMetaDataList.get(columnIndex - 1).getColType();
-        int nativeType = TSDBConstants.jdbcType2TaosType(colType);
-        return (byte) rowCursor.getInt(columnIndex, nativeType);
+        return (byte) rowCursor.getInt(columnIndex, colType);
     }
 
     @Override
     public short getShort(int columnIndex) throws SQLException {
         int colType = columnMetaDataList.get(columnIndex - 1).getColType();
-        int nativeType = TSDBConstants.jdbcType2TaosType(colType);
-        return (short) rowCursor.getInt(columnIndex, nativeType);
+        return (short) rowCursor.getInt(columnIndex, colType);
     }
 
     @Override
     public int getInt(int columnIndex) throws SQLException {
         int colType = columnMetaDataList.get(columnIndex - 1).getColType();
-        int nativeType = TSDBConstants.jdbcType2TaosType(colType);
-        return rowCursor.getInt(columnIndex, nativeType);
+        return rowCursor.getInt(columnIndex, colType);
     }
 
     @Override
     public long getLong(int columnIndex) throws SQLException {
         int colType = columnMetaDataList.get(columnIndex - 1).getColType();
-        int nativeType = TSDBConstants.jdbcType2TaosType(colType);
-        return rowCursor.getLong(columnIndex, nativeType);
+        return rowCursor.getLong(columnIndex, colType);
     }
 
     @Override
     public float getFloat(int columnIndex) throws SQLException {
         int colType = columnMetaDataList.get(columnIndex - 1).getColType();
-        int nativeType = TSDBConstants.jdbcType2TaosType(colType);
-        return rowCursor.getFloat(columnIndex, nativeType);
+        return rowCursor.getFloat(columnIndex, colType);
     }
 
     @Override
     public double getDouble(int columnIndex) throws SQLException {
         int colType = columnMetaDataList.get(columnIndex - 1).getColType();
-        int nativeType = TSDBConstants.jdbcType2TaosType(colType);
-        return rowCursor.getDouble(columnIndex, nativeType);
+        return rowCursor.getDouble(columnIndex, colType);
     }
 
     @Override
     public byte[] getBytes(int columnIndex) throws SQLException {
         int colType = columnMetaDataList.get(columnIndex - 1).getColType();
-        int nativeType = TSDBConstants.jdbcType2TaosType(colType);
-        return (rowCursor.getString(columnIndex, nativeType)).getBytes();
+        return (rowCursor.getString(columnIndex, colType)).getBytes();
     }
 
     @Override
     public Timestamp getTimestamp(int columnIndex) throws SQLException {
         int colType = columnMetaDataList.get(columnIndex - 1).getColType();
-        int nativeType = TSDBConstants.jdbcType2TaosType(colType);
-        return rowCursor.getTimestamp(columnIndex, nativeType);
+        return rowCursor.getTimestamp(columnIndex, colType);
     }
 
     @Override
@@ -158,8 +148,7 @@ public class DatabaseMetaDataResultSet extends AbstractResultSet {
     @Override
     public BigDecimal getBigDecimal(int columnIndex) throws SQLException {
         int colType = columnMetaDataList.get(columnIndex - 1).getColType();
-        int nativeType = TSDBConstants.jdbcType2TaosType(colType);
-        double value = rowCursor.getDouble(columnIndex, nativeType);
+        double value = rowCursor.getDouble(columnIndex, colType);
         return new BigDecimal(value);
     }
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java

@@ -129,8 +129,9 @@ public abstract class TSDBConstants {
                 return Types.TIMESTAMP;
             case TSDBConstants.TSDB_DATA_TYPE_NCHAR:
                 return Types.NCHAR;
+            default:
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE, "unknown taos type: " + taosType + " in tdengine");
         }
-        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE);
     }
 
     public static String taosType2JdbcTypeName(int taosType) throws SQLException {
@@ -160,7 +161,7 @@ public abstract class TSDBConstants {
             case TSDBConstants.TSDB_DATA_TYPE_NCHAR:
                 return "NCHAR";
             default:
-                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE);
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE, "unknown taos type: " + taosType + " in tdengine");
         }
     }
 
@@ -187,7 +188,7 @@ public abstract class TSDBConstants {
             case Types.NCHAR:
                 return TSDBConstants.TSDB_DATA_TYPE_NCHAR;
         }
-        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE);
+        throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE, "unknown sql type: " + jdbcType + " in tdengine");
     }
 
     public static String jdbcType2TaosTypeName(int jdbcType) throws SQLException {
@@ -213,7 +214,7 @@ public abstract class TSDBConstants {
             case Types.NCHAR:
                 return "NCHAR";
             default:
-                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE);
+                throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE, "unknown sql type: " + jdbcType + " in tdengine");
         }
     }
 

src/inc/taosdef.h

@@ -223,6 +223,7 @@ do { \
 #define TSDB_IPv4ADDR_LEN      	  16
 #define TSDB_FILENAME_LEN         128
 #define TSDB_SHOW_SQL_LEN         512
+#define TSDB_SHOW_SUBQUERY_LEN    1000
 #define TSDB_SLOW_QUERY_SQL_LEN   512
 
 #define TSDB_STEP_NAME_LEN        32

src/inc/taosmsg.h

@@ -489,7 +489,7 @@ typedef struct {
   int16_t     numOfCols;        // the number of columns will be load from vnode
   SInterval   interval;
   SSessionWindow sw;            // session window
-  uint16_t    tagCondLen;       // tag length in current query
+  uint32_t    tagCondLen;       // tag length in current query
   uint32_t    tbnameCondLen;    // table name filter condition string length
   int16_t     numOfGroupCols;   // num of group by columns
   int16_t     orderByIdx;
@@ -877,7 +877,9 @@ typedef struct {
   uint64_t sqlObjId;
   int32_t  pid;
   char     fqdn[TSDB_FQDN_LEN];
+  bool     stableQuery;
   int32_t  numOfSub;
+  char     subSqlInfo[TSDB_SHOW_SUBQUERY_LEN]; //include subqueries' index, Obj IDs and states(C-complete/I-imcomplete)
 } SQueryDesc;
 
 typedef struct {

src/inc/tsdb.h

@@ -94,7 +94,7 @@ STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH);
 int        tsdbCloseRepo(STsdbRepo *repo, int toCommit);
 int32_t    tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg);
 int        tsdbGetState(STsdbRepo *repo);
-bool       tsdbInCompact(STsdbRepo *repo);
+int8_t     tsdbGetCompactState(STsdbRepo *repo);
 // --------- TSDB TABLE DEFINITION
 typedef struct {
   uint64_t uid;  // the unique table ID

src/kit/taosdemo/taosdemo.c

@@ -5093,7 +5093,9 @@ static int getRowDataFromSample(
 
 static int64_t generateStbRowData(
         SSuperTable* stbInfo,
-        char* recBuf, int64_t timestamp)
+        char* recBuf,
+        int64_t remainderBufLen,
+        int64_t timestamp)
 {
     int64_t   dataLen = 0;
     char  *pstr = recBuf;
@@ -5121,6 +5123,7 @@ static int64_t generateStbRowData(
             rand_string(buf, stbInfo->columns[i].dataLen);
             dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\',", buf);
             tmfree(buf);
+
         } else {
             char *tmp;
 
@@ -5177,6 +5180,9 @@ static int64_t generateStbRowData(
             tstrncpy(pstr + dataLen, ",", 2);
             dataLen += 1;
         }
+
+        if (dataLen > (remainderBufLen - (DOUBLE_BUFF_LEN + 1)))
+            return 0;
     }
 
     dataLen -= 1;
@@ -5383,7 +5389,7 @@ static int32_t generateDataTailWithoutStb(
 
     int32_t k = 0;
     for (k = 0; k < batch;) {
-        char data[MAX_DATA_SIZE];
+        char *data = pstr;
         memset(data, 0, MAX_DATA_SIZE);
 
         int64_t retLen = 0;
@@ -5407,7 +5413,7 @@ static int32_t generateDataTailWithoutStb(
         if (len > remainderBufLen)
             break;
 
-        pstr += sprintf(pstr, "%s", data);
+        pstr += retLen;
         k++;
         len += retLen;
         remainderBufLen -= retLen;
@@ -5463,14 +5469,14 @@ static int32_t generateStbDataTail(
 
     int32_t k;
     for (k = 0; k < batch;) {
-        char data[MAX_DATA_SIZE];
-        memset(data, 0, MAX_DATA_SIZE);
+        char *data = pstr;
 
         int64_t lenOfRow = 0;
 
         if (tsRand) {
             if (superTblInfo->disorderRatio > 0) {
                 lenOfRow = generateStbRowData(superTblInfo, data,
+                        remainderBufLen,
                         startTime + getTSRandTail(
                             superTblInfo->timeStampStep, k,
                             superTblInfo->disorderRatio,
@@ -5478,6 +5484,7 @@ static int32_t generateStbDataTail(
                         );
             } else {
                 lenOfRow = generateStbRowData(superTblInfo, data,
+                        remainderBufLen,
                         startTime + superTblInfo->timeStampStep * k
                         );
             }
@@ -5490,11 +5497,15 @@ static int32_t generateStbDataTail(
                     pSamplePos);
         }
 
+        if (lenOfRow == 0) {
+            data[0] = '\0';
+            break;
+        }
         if ((lenOfRow + 1) > remainderBufLen) {
             break;
         }
 
-        pstr += snprintf(pstr , lenOfRow + 1, "%s", data);
+        pstr += lenOfRow;
         k++;
         len += lenOfRow;
         remainderBufLen -= lenOfRow;
@@ -6246,7 +6257,7 @@ static int32_t generateStbProgressiveData(
     assert(buffer != NULL);
     char *pstr = buffer;
 
-    memset(buffer, 0, *pRemainderBufLen);
+    memset(pstr, 0, *pRemainderBufLen);
 
     int64_t headLen = generateStbSQLHead(
             superTblInfo,
@@ -6640,7 +6651,7 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) {
                 return NULL;
             }
 
-            int64_t remainderBufLen = maxSqlLen;
+            int64_t remainderBufLen = maxSqlLen - 2000;
             char *pstr = pThreadInfo->buffer;
 
             int len = snprintf(pstr,
@@ -6822,10 +6833,14 @@ static void callBack(void *param, TAOS_RES *res, int code) {
                 && rand_num < pThreadInfo->superTblInfo->disorderRatio) {
             int64_t d = pThreadInfo->lastTs
                 - (taosRandom() % pThreadInfo->superTblInfo->disorderRange + 1);
-            generateStbRowData(pThreadInfo->superTblInfo, data, d);
+            generateStbRowData(pThreadInfo->superTblInfo, data,
+                    MAX_DATA_SIZE,
+                    d);
         } else {
             generateStbRowData(pThreadInfo->superTblInfo,
-                    data, pThreadInfo->lastTs += 1000);
+                    data,
+                    MAX_DATA_SIZE,
+                    pThreadInfo->lastTs += 1000);
         }
         pstr += sprintf(pstr, "%s", data);
         pThreadInfo->counter++;
@@ -7050,6 +7065,7 @@ static void startMultiThreadInsertData(int threads, char* db_name,
     for (int i = 0; i < threads; i++) {
         threadInfo *pThreadInfo = infos + i;
         pThreadInfo->threadID = i;
+
         tstrncpy(pThreadInfo->db_name, db_name, TSDB_DB_NAME_LEN);
         pThreadInfo->time_precision = timePrec;
         pThreadInfo->superTblInfo = superTblInfo;

src/kit/taosdump/taosdump.c

@@ -307,7 +307,7 @@ static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols,
         FILE *fp, char* dbName);
 static void taosDumpCreateMTableClause(STableDef *tableDes, char *metric,
         int numOfCols, FILE *fp, char* dbName);
-static int32_t taosDumpTable(char *table, char *metric,
+static int32_t taosDumpTable(char *tbName, char *metric,
         FILE *fp, TAOS* taosCon, char* dbName);
 static int taosDumpTableData(FILE *fp, char *tbName,
         TAOS* taosCon, char* dbName,
@@ -798,11 +798,11 @@ static int taosGetTableRecordInfo(
         tstrncpy(pTableRecordInfo->tableRecord.name,
                 (char *)row[TSDB_SHOW_TABLES_NAME_INDEX],
                 min(TSDB_TABLE_NAME_LEN,
-                    fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes) + 1);
+                    fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes + 1));
         tstrncpy(pTableRecordInfo->tableRecord.metric,
                 (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX],
                 min(TSDB_TABLE_NAME_LEN,
-                    fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes) + 1);
+                    fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes + 1));
         break;
     }
 
@@ -945,7 +945,7 @@ static int32_t taosSaveTableOfMetricToTempFile(
 
     int32_t  numOfThread = *totalNumOfThread;
     int      subFd = -1;
-    for (; numOfThread < maxThreads; numOfThread++) {
+    for (; numOfThread <= maxThreads; numOfThread++) {
         memset(tmpBuf, 0, MAX_FILE_NAME_LEN);
         sprintf(tmpBuf, ".tables.tmp.%d", numOfThread);
         subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
@@ -1084,7 +1084,7 @@ _dump_db_point:
         }
 
         tstrncpy(g_dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
-                min(TSDB_DB_NAME_LEN, fields[TSDB_SHOW_DB_NAME_INDEX].bytes) + 1);
+                min(TSDB_DB_NAME_LEN, fields[TSDB_SHOW_DB_NAME_INDEX].bytes + 1));
         if (g_args.with_property) {
             g_dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
             g_dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
@@ -1093,7 +1093,7 @@ _dump_db_point:
             g_dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
 
             tstrncpy(g_dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
-                    min(32, fields[TSDB_SHOW_DB_KEEP_INDEX].bytes) + 1);
+                    min(32, fields[TSDB_SHOW_DB_KEEP_INDEX].bytes + 1));
             //g_dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]);
             //g_dbInfos[count]->daysToKeep1;
             //g_dbInfos[count]->daysToKeep2;
@@ -1107,7 +1107,7 @@ _dump_db_point:
             g_dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));
 
             tstrncpy(g_dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
-                    min(8, fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes) + 1);
+                    min(8, fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes + 1));
             //g_dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]);
             g_dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
         }
@@ -1237,7 +1237,7 @@ _exit_failure:
 
 static int taosGetTableDes(
         char* dbName, char *table,
-        STableDef *tableDes, TAOS* taosCon, bool isSuperTable) {
+        STableDef *stableDes, TAOS* taosCon, bool isSuperTable) {
     TAOS_ROW row = NULL;
     TAOS_RES* res = NULL;
     int count = 0;
@@ -1256,18 +1256,18 @@ static int taosGetTableDes(
 
     TAOS_FIELD *fields = taos_fetch_fields(res);
 
-    tstrncpy(tableDes->name, table, TSDB_TABLE_NAME_LEN);
+    tstrncpy(stableDes->name, table, TSDB_TABLE_NAME_LEN);
     while ((row = taos_fetch_row(res)) != NULL) {
-        tstrncpy(tableDes->cols[count].field,
+        tstrncpy(stableDes->cols[count].field,
                 (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
                 min(TSDB_COL_NAME_LEN + 1,
                     fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes + 1));
-        tstrncpy(tableDes->cols[count].type,
+        tstrncpy(stableDes->cols[count].type,
                 (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
                 min(16, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes + 1));
-        tableDes->cols[count].length =
+        stableDes->cols[count].length =
             *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
-        tstrncpy(tableDes->cols[count].note,
+        tstrncpy(stableDes->cols[count].note,
                 (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
                 min(COL_NOTE_LEN,
                     fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes + 1));
@@ -1284,11 +1284,11 @@ static int taosGetTableDes(
 
     // if chidl-table have tag, using  select tagName from table to get tagValue
     for (int i = 0 ; i < count; i++) {
-        if (strcmp(tableDes->cols[i].note, "TAG") != 0) continue;
+        if (strcmp(stableDes->cols[i].note, "TAG") != 0) continue;
 
 
         sprintf(sqlstr, "select %s from %s.%s",
-                tableDes->cols[i].field, dbName, table);
+                stableDes->cols[i].field, dbName, table);
 
         res = taos_query(taosCon, sqlstr);
         code = taos_errno(res);
@@ -1310,7 +1310,7 @@ static int taosGetTableDes(
         }
 
         if (row[0] == NULL) {
-            sprintf(tableDes->cols[i].note, "%s", "NULL");
+            sprintf(stableDes->cols[i].note, "%s", "NULL");
             taos_free_result(res);
             res = NULL;
             continue;
@@ -1321,47 +1321,47 @@ static int taosGetTableDes(
         //int32_t* length = taos_fetch_lengths(tmpResult);
         switch (fields[0].type) {
             case TSDB_DATA_TYPE_BOOL:
-                sprintf(tableDes->cols[i].note, "%d",
+                sprintf(stableDes->cols[i].note, "%d",
                         ((((int32_t)(*((char *)row[0]))) == 1) ? 1 : 0));
                 break;
             case TSDB_DATA_TYPE_TINYINT:
-                sprintf(tableDes->cols[i].note, "%d", *((int8_t *)row[0]));
+                sprintf(stableDes->cols[i].note, "%d", *((int8_t *)row[0]));
                 break;
             case TSDB_DATA_TYPE_SMALLINT:
-                sprintf(tableDes->cols[i].note, "%d", *((int16_t *)row[0]));
+                sprintf(stableDes->cols[i].note, "%d", *((int16_t *)row[0]));
                 break;
             case TSDB_DATA_TYPE_INT:
-                sprintf(tableDes->cols[i].note, "%d", *((int32_t *)row[0]));
+                sprintf(stableDes->cols[i].note, "%d", *((int32_t *)row[0]));
                 break;
             case TSDB_DATA_TYPE_BIGINT:
-                sprintf(tableDes->cols[i].note, "%" PRId64 "", *((int64_t *)row[0]));
+                sprintf(stableDes->cols[i].note, "%" PRId64 "", *((int64_t *)row[0]));
                 break;
             case TSDB_DATA_TYPE_FLOAT:
-                sprintf(tableDes->cols[i].note, "%f", GET_FLOAT_VAL(row[0]));
+                sprintf(stableDes->cols[i].note, "%f", GET_FLOAT_VAL(row[0]));
                 break;
             case TSDB_DATA_TYPE_DOUBLE:
-                sprintf(tableDes->cols[i].note, "%f", GET_DOUBLE_VAL(row[0]));
+                sprintf(stableDes->cols[i].note, "%f", GET_DOUBLE_VAL(row[0]));
                 break;
             case TSDB_DATA_TYPE_BINARY:
                 {
-                    memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note));
-                    tableDes->cols[i].note[0] = '\'';
+                    memset(stableDes->cols[i].note, 0, sizeof(stableDes->cols[i].note));
+                    stableDes->cols[i].note[0] = '\'';
                     char tbuf[COL_NOTE_LEN];
                     converStringToReadable((char *)row[0], length[0], tbuf, COL_NOTE_LEN);
-                    char* pstr = stpcpy(&(tableDes->cols[i].note[1]), tbuf);
+                    char* pstr = stpcpy(&(stableDes->cols[i].note[1]), tbuf);
                     *(pstr++) = '\'';
                     break;
                 }
             case TSDB_DATA_TYPE_NCHAR:
                 {
-                    memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note));
+                    memset(stableDes->cols[i].note, 0, sizeof(stableDes->cols[i].note));
                     char tbuf[COL_NOTE_LEN-2];    // need reserve 2 bytes for ' '
                     convertNCharToReadable((char *)row[0], length[0], tbuf, COL_NOTE_LEN);
-                    sprintf(tableDes->cols[i].note, "\'%s\'", tbuf);
+                    sprintf(stableDes->cols[i].note, "\'%s\'", tbuf);
                     break;
                 }
             case TSDB_DATA_TYPE_TIMESTAMP:
-                sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]);
+                sprintf(stableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]);
 #if 0
                 if (!g_args.mysqlFlag) {
                     sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]);
@@ -1386,7 +1386,7 @@ static int taosGetTableDes(
     return count;
 }
 
-static int convertSchemaToAvroSchema(STableDef *tableDes, char **avroSchema)
+static int convertSchemaToAvroSchema(STableDef *stableDes, char **avroSchema)
 {
     errorPrint("%s() LN%d TODO: covert table schema to avro schema\n",
             __func__, __LINE__);
@@ -1394,7 +1394,7 @@ static int convertSchemaToAvroSchema(STableDef *tableDes, char **avroSchema)
 }
 
 static int32_t taosDumpTable(
-        char *table, char *metric,
+        char *tbName, char *metric,
         FILE *fp, TAOS* taosCon, char* dbName) {
     int count = 0;
 
@@ -1415,7 +1415,7 @@ static int32_t taosDumpTable(
            memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
            */
 
-        count = taosGetTableDes(dbName, table, tableDes, taosCon, false);
+        count = taosGetTableDes(dbName, tbName, tableDes, taosCon, false);
 
         if (count < 0) {
             free(tableDes);
@@ -1426,7 +1426,7 @@ static int32_t taosDumpTable(
         taosDumpCreateMTableClause(tableDes, metric, count, fp, dbName);
 
     } else {  // dump table definition
-        count = taosGetTableDes(dbName, table, tableDes, taosCon, false);
+        count = taosGetTableDes(dbName, tbName, tableDes, taosCon, false);
 
         if (count < 0) {
             free(tableDes);
@@ -1446,7 +1446,7 @@ static int32_t taosDumpTable(
 
     int32_t ret = 0;
     if (!g_args.schemaonly) {
-        ret = taosDumpTableData(fp, table, taosCon, dbName,
+        ret = taosDumpTableData(fp, tbName, taosCon, dbName,
             jsonAvroSchema);
     }
 
@@ -1648,26 +1648,27 @@ static void taosStartDumpOutWorkThreads(int32_t  numOfThread, char *dbName)
 static int32_t taosDumpStable(char *table, FILE *fp,
         TAOS* taosCon, char* dbName) {
 
-    uint64_t sizeOfTableDes = (uint64_t)(sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
-    STableDef *tableDes = (STableDef *)calloc(1, sizeOfTableDes);
-    if (NULL == tableDes) {
+    uint64_t sizeOfTableDes =
+        (uint64_t)(sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
+    STableDef *stableDes = (STableDef *)calloc(1, sizeOfTableDes);
+    if (NULL == stableDes) {
         errorPrint("%s() LN%d, failed to allocate %"PRIu64" memory\n",
                 __func__, __LINE__, sizeOfTableDes);
         exit(-1);
     }
 
-    int count = taosGetTableDes(dbName, table, tableDes, taosCon, true);
+    int count = taosGetTableDes(dbName, table, stableDes, taosCon, true);
 
     if (count < 0) {
-        free(tableDes);
+        free(stableDes);
         errorPrint("%s() LN%d, failed to get stable[%s] schema\n",
                __func__, __LINE__, table);
         exit(-1);
     }
 
-    taosDumpCreateTableClause(tableDes, count, fp, dbName);
+    taosDumpCreateTableClause(stableDes, count, fp, dbName);
 
-    free(tableDes);
+    free(stableDes);
     return 0;
 }
 
@@ -1707,7 +1708,7 @@ static int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE
         memset(&tableRecord, 0, sizeof(STableRecord));
         tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX],
                 min(TSDB_TABLE_NAME_LEN,
-                    fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes) + 1);
+                    fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes + 1));
         taosWrite(fd, &tableRecord, sizeof(STableRecord));
     }
 
@@ -1782,10 +1783,10 @@ static int taosDumpDb(SDbInfo *dbInfo, FILE *fp, TAOS *taosCon) {
         memset(&tableRecord, 0, sizeof(STableRecord));
         tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX],
                 min(TSDB_TABLE_NAME_LEN,
-                    fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes) + 1);
+                    fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes + 1));
         tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX],
                 min(TSDB_TABLE_NAME_LEN,
-                    fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes) + 1);
+                    fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes + 1));
 
         taosWrite(fd, &tableRecord, sizeof(STableRecord));
 

src/mnode/src/mnodeProfile.c

@@ -32,7 +32,8 @@
 #define CONN_KEEP_TIME  (tsShellActivityTimer * 3)
 #define CONN_CHECK_TIME (tsShellActivityTimer * 2)
 #define QUERY_ID_SIZE   20
-#define QUERY_OBJ_ID_SIZE   10
+#define QUERY_OBJ_ID_SIZE   18
+#define SUBQUERY_INFO_SIZE   6
 #define QUERY_STREAM_SAVE_SIZE 20
 
 static SCacheObj *tsMnodeConnCache = NULL;
@@ -380,12 +381,24 @@ static int32_t mnodeGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
+  pShow->bytes[cols] = 1;
+  pSchema[cols].type = TSDB_DATA_TYPE_BOOL;
+  strcpy(pSchema[cols].name, "stable_query");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "sub_queries");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
+  pShow->bytes[cols] = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "sub_query_info");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
   pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE;
   pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
   strcpy(pSchema[cols].name, "sql");
@@ -459,12 +472,8 @@ static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, v
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       *(int64_t *)pWrite = htobe64(pDesc->useconds);
       cols++;
-      /*
-      pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-      *(int64_t *)pWrite = htobe64(pDesc->sqlObjId);
-      cols++;
-      */
-      snprintf(str, tListLen(str), "0x%08" PRIx64, htobe64(pDesc->sqlObjId));
+
+      snprintf(str, tListLen(str), "0x%" PRIx64, htobe64(pDesc->sqlObjId));
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->bytes[cols]);
       cols++;
@@ -479,10 +488,18 @@ static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, v
       STR_WITH_MAXSIZE_TO_VARSTR(pWrite, epBuf, pShow->bytes[cols]);
       cols++;
 
+      pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+      *(bool *)pWrite = pDesc->stableQuery;
+      cols++;
+
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       *(int32_t *)pWrite = htonl(pDesc->numOfSub);
       cols++;
 
+      pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+      STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->subSqlInfo, pShow->bytes[cols]);
+      cols++;
+
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
       STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->sql, pShow->bytes[cols]);
       cols++;

src/query/inc/qTableMeta.h

@@ -72,6 +72,7 @@ typedef struct STableMeta {
 typedef struct STableMetaInfo {
   STableMeta    *pTableMeta;      // table meta, cached in client side and acquired by name
   uint32_t      tableMetaSize;
+  size_t        tableMetaCapacity; 
   SVgroupsInfo  *vgroupList;
   SArray        *pVgroupTables;   // SArray<SVgroupTableInfo>
 

src/query/src/qExecutor.c

@@ -6753,19 +6753,20 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) {
       if (isNull(val, type)) {
         continue;
       }
-
+      char* p = val;
       size_t keyLen = 0;
       if (IS_VAR_DATA_TYPE(pOperator->pExpr->base.colType)) {
         tstr* var = (tstr*)(val);
+        p = var->data;
         keyLen = varDataLen(var);
       } else {
         keyLen = bytes;
       }
 
       int dummy;
-      void* res = taosHashGet(pInfo->pSet, val, keyLen);
+      void* res = taosHashGet(pInfo->pSet, p, keyLen);
       if (res == NULL) {
-        taosHashPut(pInfo->pSet, val, keyLen, &dummy, sizeof(dummy));
+        taosHashPut(pInfo->pSet, p, keyLen, &dummy, sizeof(dummy));
         char* start = pResultColInfoData->pData + bytes * pInfo->pRes->info.rows;
         memcpy(start, val, bytes);
         pRes->info.rows += 1;
@@ -6990,7 +6991,7 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) {
   pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols);
   pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput);
   pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols);
-  pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen);
+  pQueryMsg->tagCondLen = htonl(pQueryMsg->tagCondLen);
   pQueryMsg->tsBuf.tsOffset = htonl(pQueryMsg->tsBuf.tsOffset);
   pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen);
   pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks);

src/tsdb/inc/tsdbRowMergeBuf.h

@@ -29,7 +29,9 @@ typedef void* SMergeBuf;
 SDataRow tsdbMergeTwoRows(SMergeBuf *pBuf, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2);
 
 static FORCE_INLINE int tsdbMergeBufMakeSureRoom(SMergeBuf *pBuf, STSchema* pSchema1, STSchema* pSchema2) {
-  return tsdbMakeRoom(pBuf, MAX(dataRowMaxBytesFromSchema(pSchema1), dataRowMaxBytesFromSchema(pSchema2)));
+  size_t len1 = dataRowMaxBytesFromSchema(pSchema1);
+  size_t len2 = dataRowMaxBytesFromSchema(pSchema2);
+  return tsdbMakeRoom(pBuf, MAX(len1, len2));
 }
 
 static FORCE_INLINE void tsdbFreeMergeBuf(SMergeBuf buf) {

src/tsdb/inc/tsdbint.h

@@ -94,8 +94,9 @@ struct STsdbRepo {
   pthread_mutex_t mutex;
   bool            repoLocked;
   int32_t         code;  // Commit code
+
   SMergeBuf       mergeBuf;  //used when update=2
-  bool            inCompact;  // is in compact process?
+  int8_t          compactState;  // compact state: inCompact/noCompact/waitingCompact?
 };
 
 #define REPO_ID(r) (r)->config.tsdbId

src/tsdb/src/tsdbCompact.c

@@ -58,6 +58,7 @@ static int  tsdbCompactFSetImpl(SCompactH *pComph);
 static int  tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf,
                                       void **ppCBuf);
 
+enum { TSDB_NO_COMPACT, TSDB_IN_COMPACT, TSDB_WAITING_COMPACT};
 int tsdbCompact(STsdbRepo *pRepo) { return tsdbAsyncCompact(pRepo); }
 
 void *tsdbCompactImpl(STsdbRepo *pRepo) {
@@ -89,16 +90,21 @@ _err:
 }
 
 static int tsdbAsyncCompact(STsdbRepo *pRepo) {
+  if (pRepo->compactState != TSDB_NO_COMPACT) {
+    tsdbInfo("vgId:%d not compact tsdb again ", REPO_ID(pRepo));
+    return 0; 
+  } 
+  pRepo->compactState = TSDB_WAITING_COMPACT;   
   tsem_wait(&(pRepo->readyToCommit));
   return tsdbScheduleCommit(pRepo, COMPACT_REQ);
 }
 
 static void tsdbStartCompact(STsdbRepo *pRepo) {
-  ASSERT(!pRepo->inCompact);
+  assert(pRepo->compactState != TSDB_IN_COMPACT);
   tsdbInfo("vgId:%d start to compact!", REPO_ID(pRepo));
   tsdbStartFSTxn(pRepo, 0, 0);
   pRepo->code = TSDB_CODE_SUCCESS;
-  pRepo->inCompact = true;
+  pRepo->compactState = TSDB_IN_COMPACT;
 }
 
 static void tsdbEndCompact(STsdbRepo *pRepo, int eno) {
@@ -107,7 +113,7 @@ static void tsdbEndCompact(STsdbRepo *pRepo, int eno) {
   } else {
     tsdbEndFSTxn(pRepo);
   }
-  pRepo->inCompact = false;
+  pRepo->compactState = TSDB_NO_COMPACT;
   tsdbInfo("vgId:%d compact over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed");
   tsem_post(&(pRepo->readyToCommit));
 }

src/tsdb/src/tsdbMain.c

@@ -200,7 +200,7 @@ STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo) { return NULL; }
 
 int tsdbGetState(STsdbRepo *repo) { return repo->state; }
 
-bool tsdbInCompact(STsdbRepo *repo) { return repo->inCompact; }
+int8_t tsdbGetCompactState(STsdbRepo *repo) { return (int8_t)(repo->compactState); }
 
 void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) {
   ASSERT(repo != NULL);
@@ -541,7 +541,7 @@ static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) {
 
   pRepo->state = TSDB_STATE_OK;
   pRepo->code = TSDB_CODE_SUCCESS;
-  pRepo->inCompact = false;
+  pRepo->compactState = 0;
   pRepo->config = *pCfg;
   if (pAppH) {
     pRepo->appH = *pAppH;

src/tsdb/src/tsdbMeta.c

@@ -1035,6 +1035,8 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro
       }
     }
   }
+  pMeta->maxCols = maxCols;
+  pMeta->maxRowBytes = maxRowBytes;
 
   if (lock) tsdbUnlockRepoMeta(pRepo);
   tsdbDebug("vgId:%d table %s uid %" PRIu64 " is removed from meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_UID(pTable));

src/tsdb/src/tsdbReadImpl.c

@@ -518,6 +518,8 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32
     return -1;
   }
 
+  tdAllocMemForCol(pDataCol, maxPoints);
+
   // Decode the data
   if (comp) {
     // Need to decompress

src/util/inc/hash.h

@@ -127,6 +127,16 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen);
  */
 void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d);
 
+/**
+ * @param pHashObj
+ * @param key
+ * @param keyLen
+ * @param fp
+ * @param d
+ * @param sz 
+ * @return
+ */
+void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void** d, size_t *sz);
 /**
  * remove item with the specified key
  * @param pHashObj

src/util/src/hash.c

@@ -18,6 +18,8 @@
 #include "tulog.h"
 #include "taosdef.h"
 
+#define EXT_SIZE 1024 
+
 #define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR)
 
 #define DO_FREE_HASH_NODE(_n) \
@@ -296,6 +298,68 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
 void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) {
   return taosHashGetClone(pHashObj, key, keyLen, NULL, NULL);
 }
+//TODO(yihaoDeng), merge with taosHashGetClone
+void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void** d, size_t *sz) {
+  if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) {
+    return NULL;
+  }
+
+  uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen);
+
+  // only add the read lock to disable the resize process
+  __rd_lock(&pHashObj->lock, pHashObj->type);
+
+  int32_t     slot = HASH_INDEX(hashVal, pHashObj->capacity);
+  SHashEntry *pe = pHashObj->hashList[slot];
+
+  // no data, return directly
+  if (atomic_load_32(&pe->num) == 0) {
+    __rd_unlock(&pHashObj->lock, pHashObj->type);
+    return NULL;
+  }
+
+  char *data = NULL;
+
+  // lock entry
+  if (pHashObj->type == HASH_ENTRY_LOCK) {
+    taosRLockLatch(&pe->latch);
+  }
+
+  if (pe->num > 0) {
+    assert(pe->next != NULL);
+  } else {
+    assert(pe->next == NULL);
+  }
+
+  SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal);
+  if (pNode != NULL) {
+    if (fp != NULL) {
+      fp(GET_HASH_NODE_DATA(pNode));
+    }
+     
+    if (*d == NULL) {
+      *sz =  pNode->dataLen + EXT_SIZE;
+      *d  =  calloc(1, *sz);   
+    } else if (*sz < pNode->dataLen){
+      *sz = pNode->dataLen + EXT_SIZE;
+      *d  = realloc(*d, *sz); 
+    }
+    memcpy((char *)(*d), GET_HASH_NODE_DATA(pNode), pNode->dataLen);
+    // just make runtime happy 
+    if ((*sz) - pNode->dataLen > 0) {
+      memset((char *)(*d) + pNode->dataLen, 0, (*sz) - pNode->dataLen);
+    } 
+
+    data = GET_HASH_NODE_DATA(pNode);
+  }
+
+  if (pHashObj->type == HASH_ENTRY_LOCK) {
+    taosRUnLockLatch(&pe->latch);
+  }
+
+  __rd_unlock(&pHashObj->lock, pHashObj->type);
+  return data;
+}
 
 void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d) {
   if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) {

src/vnode/src/vnodeMgmt.c

@@ -162,7 +162,7 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) {
   pLoad->status = pVnode->status;
   pLoad->role = pVnode->role;
   pLoad->replica = pVnode->syncCfg.replica;  
-  pLoad->compact = (pVnode->tsdb != NULL) && tsdbInCompact(pVnode->tsdb) ? 1 : 0; 
+  pLoad->compact = (pVnode->tsdb != NULL) ? tsdbGetCompactState(pVnode->tsdb) : 0; 
 }
 
 int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {

tests/perftest-scripts/perftest-query.sh

 #!/bin/bash
 
+
 branch=
 if [ x$1 != x ];then
         branch=$1
@@ -8,17 +9,19 @@ else
         echo "Please enter branch name as a parameter"
         exit 1
 fi
-jemalloc=
+
+type=
 if [ x$2 != x ];then
-		jemalloc=jemalloc
+    	type=jemalloc
         echo "Building TDengine using jemalloc"
 else
+		type=glibc
         echo "Building TDengine using glibc"
 fi
 
 today=`date +"%Y%m%d"`
-WORK_DIR=/home/ubuntu/pxiao
-PERFORMANCE_TEST_REPORT=$WORK_DIR/TDengine/tests/performance-test-report-$today.log
+WORK_DIR=/root/pxiao
+PERFORMANCE_TEST_REPORT=$WORK_DIR/TDengine/tests/performance-report-$branch-$type-$today.log
 
 # Coloured Echoes                                                                                                       #
 function red_echo      { echo -e "\033[31m$@\033[0m";   }                                                               #
@@ -64,52 +67,41 @@ function buildTDengine {
 	echo "REMOTE: $REMOTE_COMMIT"
 	if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
 		echo "repo up-to-date"
-	else
-		echo "repo need to pull"
-		git pull > /dev/null 2>&1
+	fi
 
-		LOCAL_COMMIT=`git rev-parse --short @`
-		if [ $jemalloc = "jemalloc" ];then
+	git pull > /dev/null 2>&1
+	if [ $type = "jemalloc" ];then
 		echo "git submodule update --init --recursive"
 		git submodule update --init --recursive
 	fi
-		
+	LOCAL_COMMIT=`git rev-parse --short @`
 	cd debug
 	rm -rf *
-
-		if [ $jemalloc = "jemalloc" ];then
+	if [ $type = "jemalloc" ];then
 		echo "cmake .. -DJEMALLOC_ENABLED=true > /dev/null"
 		cmake .. -DJEMALLOC_ENABLED=true > /dev/null
 	else
 		cmake .. > /dev/null
 	fi
-		make && make install > /dev/null		
-	fi
+	make > /dev/null 2>&1	
+	make install > /dev/null 2>&1
+	echo "Build TDengine on remote server"	
+	ssh perftest "./buildTDengine.sh $branch > /dev/null"
 }
 
 function runQueryPerfTest {
 	[ -f $PERFORMANCE_TEST_REPORT ] && rm $PERFORMANCE_TEST_REPORT
-	nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/taosperf/ > /dev/null 2>&1 &
+	nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/perf/ > /dev/null 2>&1 &
 	echoInfo "Wait TDengine to start"
 	sleep 60
 	echoInfo "Run Performance Test"	
 	cd $WORK_DIR/TDengine/tests/pytest	
 
-	python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
+	python3 query/queryPerformance.py -c $LOCAL_COMMIT -b $branch -T $type | tee -a $PERFORMANCE_TEST_REPORT
 
-	mkdir -p /var/lib/perf/
-	mkdir -p /var/log/perf/
-	rm -rf /var/lib/perf/*
-	rm -rf /var/log/perf/*
-	nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/perf/ > /dev/null 2>&1 &	
-	echoInfo "Wait TDengine to start"
-	sleep 10
-	echoInfo "Run Performance Test"	
-	cd $WORK_DIR/TDengine/tests/pytest
+	python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT -b $branch -T $type | tee -a $PERFORMANCE_TEST_REPORT
 	
-	python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
-	
-	python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT
+	python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -b $branch -T $type | tee -a $PERFORMANCE_TEST_REPORT
 }
 
 
@@ -122,8 +114,7 @@ function sendReport {
 
 	sed -i 's/\x1b\[[0-9;]*m//g' $PERFORMANCE_TEST_REPORT
 	BODY_CONTENT=`cat $PERFORMANCE_TEST_REPORT`
-	
-	echo -e "From: <support@taosdata.com>\nto: ${receiver}\nsubject: Query Performace Report ${branch} ${jemalloc} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+	echo -e "From: <support@taosdata.com>\nto: ${receiver}\nsubject: Query Performace Report ${branch} ${jemalloc} commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
 	(cat - && uuencode $PERFORMANCE_TEST_REPORT performance-test-report-$today.log) | \
 	/usr/sbin/ssmtp "${receiver}" && echo "Report Sent!"
 }

tests/pytest/fulltest.sh

@@ -148,6 +148,9 @@ python3 ./test.py -f import_merge/importTPORestart.py
 python3 ./test.py -f import_merge/importTRestart.py
 python3 ./test.py -f import_merge/importInsertThenImport.py
 python3 ./test.py -f import_merge/importCSV.py
+python3 ./test.py -f import_merge/import_update_0.py
+python3 ./test.py -f import_merge/import_update_1.py
+python3 ./test.py -f import_merge/import_update_2.py
 #======================p1-end===============
 #======================p2-start===============
 # tools
@@ -254,6 +257,7 @@ python3 ./test.py -f query/nestedQuery/queryWithOrderLimit.py
 python3 ./test.py -f query/nestquery_last_row.py
 python3 ./test.py -f query/queryCnameDisplay.py
 python3 ./test.py -f query/operator_cost.py
+python3 ./test.py -f query/long_where_query.py
 python3 test.py -f query/nestedQuery/queryWithSpread.py
 
 #stream
@@ -376,6 +380,7 @@ python3 test.py -f alter/alter_cacheLastRow.py
 python3 ./test.py -f query/querySession.py 
 python3 test.py -f  alter/alter_create_exception.py
 python3 ./test.py -f insert/flushwhiledrop.py
+python3 ./test.py -f insert/schemalessInsert.py
 
 #======================p4-end===============
 python3 test.py -f  tools/taosdemoAllTest/pytest.py

tests/pytest/insert/insertFromCSVPerformance.py

@@ -22,11 +22,12 @@ import argparse
 import os.path
 
 class insertFromCSVPerformace:
-    def __init__(self, commitID, dbName, stbName, branchName):
+    def __init__(self, commitID, dbName, tbName, branchName, buildType):
         self.commitID = commitID
         self.dbName = dbName
-        self.stbName = stbName
+        self.tbName = tbName
         self.branchName = branchName
+        self.type = buildType
         self.ts = 1500074556514
         self.host = "127.0.0.1"
         self.user = "root"
@@ -37,6 +38,12 @@ class insertFromCSVPerformace:
             self.user,
             self.password,            
             self.config)
+        self.host2 = "192.168.1.179"    
+        self.conn2 = taos.connect(
+            host = self.host2,
+            user = self.user,
+            password = self.password,
+            config = self.config)
 
     def writeCSV(self):
         with open('test3.csv','w', encoding='utf-8', newline='') as csvFile:
@@ -52,46 +59,42 @@ class insertFromCSVPerformace:
         data = data.drop([0])
         data.to_csv("ordered.csv", header = False, index = False)
     
-    def createTables(self):
-        cursor = self.conn.cursor()
-                    
-        cursor.execute("create database if not exists %s" % self.dbName)
-        cursor.execute("use %s" % self.dbName)
-        cursor.execute("create table if not exists %s(ts timestamp, in_order_time float, out_of_order_time float, commit_id binary(50)) tags(branch binary(50))" % self.stbName)
-        cursor.execute("create table if not exists %s using %s tags('%s')" % (self.branchName, self.stbName, self.branchName))
-
-        cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
-        cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
-
-        cursor.close()
-    
     def run(self):
         cursor = self.conn.cursor()
+        cursor.execute("create database if not exists %s" % self.dbName)
         cursor.execute("use %s" % self.dbName)
         print("==================== CSV insert performance ====================")
         
         totalTime = 0
         for i in range(10):
+            cursor.execute("drop table if exists t1")  
             cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
             startTime = time.time()
             cursor.execute("insert into t1 file 'outoforder.csv'")
             totalTime += time.time() - startTime                      
-            cursor.execute("drop table if exists t1")            
         out_of_order_time = (float) (totalTime / 10)
         print("Out of Order - Insert time: %f" % out_of_order_time)                      
         
         totalTime = 0
         for i in range(10):
+            cursor.execute("drop table if exists t2")
             cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)")
             startTime = time.time()
             cursor.execute("insert into t2 file 'ordered.csv'")
             totalTime += time.time() - startTime            
-            cursor.execute("drop table if exists t2")
 
         in_order_time = (float) (totalTime / 10)
         print("In order - Insert time: %f" % in_order_time)
-        cursor.execute("insert into %s values(now, %f, %f, '%s')" % (self.branchName, in_order_time, out_of_order_time, self.commitID))
+        cursor.close()
+
 
+        cursor2 = self.conn2.cursor()
+        cursor2.execute("create database if not exists %s" % self.dbName)
+        cursor2.execute("use %s" % self.dbName)
+        cursor2.execute("create table if not exists %s(ts timestamp, in_order_time float, out_of_order_time float, commit_id binary(50), branch binary(50), type binary(20))" % self.tbName)     
+        cursor2.execute("insert into %s values(now, %f, %f, '%s', '%s', '%s')" % (self.tbName, in_order_time, out_of_order_time, self.commitID, self.branchName, self.type))
+
+        cursor2.close()
         
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()    
@@ -111,7 +114,7 @@ if __name__ == '__main__':
         help='Database name to be created (default: perf)')
     parser.add_argument(
         '-t',
-        '--stable-name',
+        '--table-name',
         action='store',
         default='csv_insert',
         type=str,
@@ -123,9 +126,14 @@ if __name__ == '__main__':
         default='develop',
         type=str,
         help='branch name (default: develop)')
+    parser.add_argument(
+        '-T',
+        '--build-type',
+        action='store',
+        default='glibc',
+        type=str,
+        help='build type (default: glibc)')
     
     args = parser.parse_args()
-    perftest = insertFromCSVPerformace(args.commit_id, args.database_name, args.stable_name, args.branch_name)
-
-    perftest.createTables()
+    perftest = insertFromCSVPerformace(args.commit_id, args.database_name, args.table_name, args.branch_name, args.build_type)
     perftest.run()
\ No newline at end of file

tests/pytest/query/long_where_query.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import random
+import string
+import os
+import time
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+from util.dnodes import tdDnodes
+
+class TDTestCase:
+    updatecfgDict={'maxSQLLength':1048576}
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        self.ts = 1538548685000
+        self.num = 100
+
+    def get_random_string(self, length):
+        letters = string.ascii_lowercase
+        result_str = ''.join(random.choice(letters) for i in range(length))
+        return result_str
+
+    def run(self):
+        tdSql.prepare()
+        # test case for https://jira.taosdata.com:18080/browse/TD-5213
+
+        print("==============step1, regular table==============")
+        startTime = time.time() 
+        sql = "create table regular_table_1(ts timestamp, "
+        for i in range(4094):
+            sql += "col00000111112222233333444445555566666777778888899999000000l%d int, " % (i + 1)
+        sql += "col4095 binary(22))"  
+        tdLog.info(len(sql))      
+        tdSql.execute(sql)
+
+        tdLog.info("========== test1.1 : test regular table in ( ) ==========")
+        sql = '''insert into regular_table_1(ts,col00000111112222233333444445555566666777778888899999000000l1) values(now,1);'''
+        tdSql.execute(sql)
+        sql = ''' select * from regular_table_1  where  col00000111112222233333444445555566666777778888899999000000l1 in (1); '''
+        tdSql.query(sql)  
+        tdSql.checkData(0, 1, 1)
+
+        for i in range(self.num):
+            sql = "insert into regular_table_1 values(%d, "
+            for j in range(4094):
+                str = "'%s', " % random.randint(0,1000)                
+                sql += str
+            sql += "'%s')" % self.get_random_string(22)
+            tdSql.execute(sql % (self.ts + i))
+        time.sleep(1)
+        tdSql.query("select count(*) from regular_table_1")
+        tdSql.checkData(0, 0, self.num+1)
+        tdSql.query("select * from regular_table_1")
+        tdSql.checkRows(self.num+1)
+        tdSql.checkCols(4096)
+
+        #maxSQLLength 1048576
+        sql = "select * from regular_table_1 where col00000111112222233333444445555566666777778888899999000000l1 in ("
+        for i in range(2,128840):
+            sql += "%d , " % (i + 1)
+        sql += "1 ,12345) order by ts desc;"  
+        #tdLog.info(sql)
+        tdLog.info(len(sql)) 
+        tdSql.query(sql)  
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkCols(4096)
+
+        #maxSQLLength 1048577--error
+        sql = "select * from regular_table_1 where col00000111112222233333444445555566666777778888899999000000l1 in ("
+        for i in range(2,128840):
+            sql += "%d , " % (i + 1)
+        sql += "1 ,123456) order by ts desc;"  
+        #tdLog.info(sql)
+        tdLog.info(len(sql))  
+        tdSql.error(sql)  
+
+        tdLog.info("========== test1.2 : test regular table in (' ') ==========")
+        sql = '''insert into regular_table_1(ts,col4095) values(now,1);'''
+        tdSql.execute(sql)
+        sql = ''' select * from regular_table_1  where  col4095 in ('1',"1"); '''
+        tdSql.query(sql)  
+        tdSql.checkData(0, 4095, 1)
+
+        #maxSQLLength 1048576
+        sql = " select * from regular_table_1 where col4095 in ("
+        for i in range(96328):
+            sql += " '%d' , " % (i + 1)
+        sql += " '1' ) order by ts desc;" 
+        #tdLog.info(sql)
+        tdLog.info(len(sql)) 
+        tdSql.query(sql)  
+        tdSql.checkData(0, 4095, 1)
+        tdSql.checkCols(4096)
+        
+        #maxSQLLength 1048577--error
+        sql = " select * from regular_table_1 where col4095 in ("
+        for i in range(96328):
+            sql += " '%d' , " % (i + 1)
+        sql += " '123' ) order by ts desc;" 
+        #tdLog.info(sql)
+        tdLog.info(len(sql))   
+        tdSql.error(sql)  
+        
+        endTime = time.time()
+        print("total time %ds" % (endTime - startTime))
+ 
+
+
+
+        print("==============step2, super table ==============")
+        startTime = time.time() 
+        sql = "create stable stable_1(ts timestamp, "
+        for i in range(4090):
+            sql += "col00000111112222233333444445555566666777778888899999000000l%d int, " % (i + 1)
+        sql += "col4091 binary(22))"  
+        sql += " tags (loc nchar(10),tag_1 int,tag_2 int,tag_3 int) " 
+        tdLog.info(len(sql))      
+        tdSql.execute(sql)
+        sql = '''create table table_1 using stable_1 
+                    tags('table_1' , '1' , '2' , '3' );'''
+        tdSql.execute(sql)
+
+        tdLog.info("========== test2.1 : test super table in ( ) ==========")
+        sql = '''insert into table_1(ts,col00000111112222233333444445555566666777778888899999000000l1) values(now,1);'''
+        tdSql.execute(sql)
+        sql = ''' select * from stable_1  where  col00000111112222233333444445555566666777778888899999000000l1 in (1); '''
+        tdSql.query(sql)  
+        tdSql.checkData(0, 1, 1)
+        sql = ''' select * from table_1  where  col00000111112222233333444445555566666777778888899999000000l1 in (1); '''
+        tdSql.query(sql)  
+        tdSql.checkData(0, 1, 1)
+
+        for i in range(self.num):
+            sql = "insert into table_1 values(%d, "
+            for j in range(4090):
+                str = "'%s', " % random.randint(0,1000)                
+                sql += str
+            sql += "'%s')" % self.get_random_string(22)
+            tdSql.execute(sql % (self.ts + i))
+        time.sleep(1)
+
+        tdSql.query("select count(*) from table_1")
+        tdSql.checkData(0, 0, self.num+1)
+        tdSql.query("select * from table_1")
+        tdSql.checkRows(self.num+1)
+        tdSql.checkCols(4092)
+
+        tdSql.query("select count(*) from stable_1")
+        tdSql.checkData(0, 0, self.num+1)
+        tdSql.query("select * from stable_1")
+        tdSql.checkRows(self.num+1)
+        tdSql.checkCols(4096)
+
+        #maxSQLLength 1048576
+        sql = "select * from table_1 where col00000111112222233333444445555566666777778888899999000000l1 in ("
+        for i in range(128840):
+            sql += "%d , " % (i + 1)
+        sql += "1 ,12345) order by ts desc;"  
+        #tdLog.info(sql)
+        tdLog.info(len(sql)) 
+        tdSql.query(sql)  
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkCols(4092)  
+
+        sql = "select * from stable_1 where col00000111112222233333444445555566666777778888899999000000l1 in ("
+        for i in range(128840):
+            sql += "%d , " % (i + 1)
+        sql += "1 ,1234) order by ts desc;"  
+        #tdLog.info(sql)
+        tdLog.info(len(sql)) 
+        tdSql.query(sql)  
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkCols(4096)      
+
+        #TD-5640
+        sql = "select * from stable_1 where tag_1 in ("
+        for i in range(128847):
+            sql += "%d , " % (i + 1)
+        sql += "1)order by ts desc;"  
+        #tdLog.info(sql)
+        tdLog.info(len(sql)) 
+        tdSql.query(sql)  
+        tdSql.checkData(0, 1, 1)
+        tdSql.checkCols(4096)    
+
+        #maxSQLLength 1048577--error
+        sql = "select * from table_1 where col00000111112222233333444445555566666777778888899999000000l1 in ("
+        for i in range(128840):
+            sql += "%d , " % (i + 1)
+        sql += "1 ,123456) order by ts desc;"  
+        #tdLog.info(sql)
+        tdLog.info(len(sql)) 
+        tdSql.error(sql)    
+
+        sql = "select * from stable_1 where col00000111112222233333444445555566666777778888899999000000l1 in ("
+        for i in range(128840):
+            sql += "%d , " % (i + 1)
+        sql += "1 ,12345) order by ts desc;"  
+        #tdLog.info(sql)
+        tdLog.info(len(sql)) 
+        tdSql.error(sql)    
+
+        sql = "select * from stable_1 where tag_1 in ("
+        for i in range(128847):
+            sql += "%d , " % (i + 1)
+        sql += "1) order by ts desc;"  
+        #tdLog.info(sql)
+        tdLog.info(len(sql)) 
+        tdSql.error(sql)     
+
+
+        tdLog.info("========== tests2.2 : test super table in (' ') ==========")
+        sql = '''insert into table_1(ts,col4091) values(now,1);'''
+        tdSql.execute(sql)
+        sql = ''' select * from table_1  where  col4091 in ('1',"1"); '''
+        tdSql.query(sql)  
+        tdSql.checkData(0, 4091, 1)
+
+        #maxSQLLength 1048576
+        sql = " select * from table_1 where col4091 in ("
+        for i in range(96328):
+            sql += " '%d' , " % (i + 1)
+        sql += " '1','123456' ) order by ts desc;" 
+        #tdLog.info(sql)
+        tdLog.info(len(sql)) 
+        tdSql.query(sql)  
+        tdSql.checkData(0, 4091, 1)
+        tdSql.checkCols(4092)
+
+        sql = " select * from stable_1 where col4091 in ("
+        for i in range(96328):
+            sql += " '%d' , " % (i + 1)
+        sql += " '1','12345' ) order by ts desc;" 
+        #tdLog.info(sql)
+        tdLog.info(len(sql)) 
+        tdSql.query(sql)  
+        tdSql.checkData(0, 4091, 1)
+        tdSql.checkCols(4096)
+
+        #TD-5650
+        sql = " select * from stable_1 where loc in ("
+        for i in range(96328):
+            sql += " '%d' , " % (i + 1)
+        sql += " '123','table_1' ) order by ts desc;" 
+        #tdLog.info(sql)
+        tdLog.info(len(sql)) 
+        tdSql.query(sql)  
+        tdSql.checkData(0, 4092, 'table_1')
+        tdSql.checkCols(4096)
+
+        #maxSQLLength 1048577--error
+        sql = " select * from table_1 where col4091 in ("
+        for i in range(96328):
+            sql += " '%d' , " % (i + 1)
+        sql += " '1','1234567' ) order by ts desc;" 
+        #tdLog.info(sql)
+        tdLog.info(len(sql)) 
+        tdSql.error(sql) 
+
+        sql = " select * from stable_1 where col4091 in ("
+        for i in range(96328):
+            sql += " '%d' , " % (i + 1)
+        sql += " '1','123456' ) order by ts desc;" 
+        #tdLog.info(sql)
+        tdLog.info(len(sql)) 
+        tdSql.error(sql) 
+
+        sql = " select * from stable_1 where loc in ("
+        for i in range(96328):
+            sql += " '%d' , " % (i + 1)
+        sql += " '1','1234567890' ) order by ts desc;" 
+        #tdLog.info(sql)
+        tdLog.info(len(sql)) 
+        tdSql.error(sql) 
+
+
+        endTime = time.time()
+        print("total time %ds" % (endTime - startTime))
+
+
+
+        os.system("rm -rf query/long_where_query.py.sql") 
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/query/operator_cost.py

@@ -207,9 +207,9 @@ class TDTestCase:
         sql = '''select distinct(t_ts) from stable_1;'''
         tdSql.query(sql)
         tdSql.checkRows(3)
-        # sql = '''select distinct(tbname) from stable_1;'''
-        # tdSql.query(sql)
-        # tdSql.checkRows(6)
+        sql = '''select distinct(tbname) from stable_1;'''
+        tdSql.query(sql)
+        tdSql.checkRows(6)
 
         tdLog.info("========== operator=2(OP_DataBlocksOptScan) ==========")
         sql = '''select last(q_int),first(q_int) from stable_1;'''

tests/pytest/query/queryPerformance.py

@@ -20,12 +20,14 @@ import argparse
 
 
 class taosdemoQueryPerformace:
-    def __init__(self, clearCache, commitID, dbName, stbName, tbPerfix):
+    def __init__(self, clearCache, commitID, dbName, stbName, tbPerfix, branch, type):
         self.clearCache = clearCache
         self.commitID = commitID
         self.dbName = dbName
         self.stbName = stbName
         self.tbPerfix = tbPerfix
+        self.branch = branch
+        self.type = type
         self.host = "127.0.0.1"
         self.user = "root"
         self.password = "taosdata"
@@ -35,50 +37,56 @@ class taosdemoQueryPerformace:
             self.user,
             self.password,
             self.config)
+        self.host2 = "192.168.1.179"    
+        self.conn2 = taos.connect(
+            host = self.host2,
+            user = self.user,
+            password = self.password,
+            config = self.config)
 
     def createPerfTables(self):
-        cursor = self.conn.cursor()
-        cursor.execute("create database if not exists %s" % self.dbName)
-        cursor.execute("use %s" % self.dbName)
-        cursor.execute("create table if not exists %s(ts timestamp, query_time float, commit_id binary(50)) tags(query_id int, query_sql binary(300))" % self.stbName)
+        cursor2 = self.conn2.cursor()
+        cursor2.execute("create database if not exists %s" % self.dbName)
+        cursor2.execute("use %s" % self.dbName)
+        cursor2.execute("create table if not exists %s(ts timestamp, query_time float, commit_id binary(50), branch binary(50), type binary(20)) tags(query_id int, query_sql binary(300))" % self.stbName)
 
         sql = "select count(*) from test.meters"
         tableid = 1
-        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        cursor2.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
         
         sql = "select avg(f1), max(f2), min(f3) from test.meters"
         tableid = 2
-        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        cursor2.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
         
         sql = "select count(*) from test.meters where loc='beijing'"
         tableid = 3
-        cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        cursor2.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
         
         sql = "select avg(f1), max(f2), min(f3) from test.meters where areaid=10"
         tableid = 4
-        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        cursor2.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
         
         sql = "select avg(f1), max(f2), min(f3) from test.t10 interval(10s)"
         tableid = 5
-        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        cursor2.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
         
         sql = "select last_row(*) from meters"
         tableid = 6
-        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        cursor2.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
         
         sql = "select * from meters"
         tableid = 7
-        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        cursor2.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
         
         sql = "select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'"
         tableid = 8
-        cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        cursor2.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
 
         sql = "select last(*) from meters"
         tableid = 9
-        cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+        cursor2.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
 
-        cursor.close()
+        cursor2.close()
 
     def query(self): 
         cursor = self.conn.cursor() 
@@ -106,8 +114,8 @@ class taosdemoQueryPerformace:
             cursor2.close()
             print("query time for: %s %f seconds" % (sql, totalTime / 100))
                         
-            cursor3 = self.conn.cursor()
-            cursor3.execute("insert into %s.%s values(now, %f, '%s')" % (self.dbName, table_name, totalTime / 100, self.commitID))
+            cursor3 = self.conn2.cursor()
+            cursor3.execute("insert into %s.%s values(now, %f, '%s', '%s', '%s')" % (self.dbName, table_name, totalTime / 100, self.commitID, self.branch, self.type))
 
         cursor3.close()
         cursor.close()
@@ -148,8 +156,22 @@ if __name__ == '__main__':
         default='q',
         type=str,
         help='table name perfix (default: q)')
+    parser.add_argument(
+        '-b',
+        '--git-branch',
+        action='store',
+        default='master',
+        type=str,
+        help='git branch (default: master)')
+    parser.add_argument(
+        '-T',
+        '--build-type',
+        action='store',
+        default='glibc',
+        type=str,
+        help='build type (default: glibc)')
     
     args = parser.parse_args()
-    perftest = taosdemoQueryPerformace(args.remove_cache, args.commit_id, args.database_name, args.stable_name, args.table_perfix)
+    perftest = taosdemoQueryPerformace(args.remove_cache, args.commit_id, args.database_name, args.stable_name, args.table_perfix, args.git_branch, args.build_type)
     perftest.createPerfTables()
     perftest.query()

tests/pytest/tag_lite/TestModifyTag.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+        self.ts = 1625068800000000000   # this is timestamp  "2021-07-01 00:00:00"
+        self.numberOfTables = 10
+        self.numberOfRecords = 100
+
+    def checkCommunity(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+        if ("community" in selfPath):
+            return False
+        else:
+            return True
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosdump" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+    
+    
+
+    def run(self):
+
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosdump not found!")
+        else:
+            tdLog.info("taosdump found in %s" % buildPath)
+        binPath = buildPath + "/build/bin/"
+
+        # basic test for alter tags
+        tdSql.execute("create database tagdb ")
+        tdSql.execute(" use tagdb")
+        tdSql.execute("create table st (ts timestamp , a int) tags (tg1 binary(20), tg2 binary(20), tg3 binary(20))")
+        tdSql.execute("insert into t using st (tg3, tg2, tg1) tags ('tg3', 'tg2', 'tg1') values (now, 1)")
+        tdSql.execute("alter table t set tag tg1='newtg1'")
+        res = tdSql.getResult("select tg1,tg2,tg3 from t")
+
+        if res == [('newtg1', 'tg2', 'tg3')]:
+            tdLog.info(" alter tag check has pass!")
+        else:
+            tdLog.info(" alter tag failed , please check !")
+
+        tdSql.error("alter stable st modify tag tg2 binary(2)")
+        tdSql.execute("alter stable st modify tag tg2 binary(30) ")
+        tdSql.execute("alter table t set tag tg2 = 'abcdefghijklmnopqrstuvwxyz1234'")
+        res = tdSql.getResult("select tg1,tg2,tg3 from t")
+        if res == [('newtg1', 'abcdefghijklmnopqrstuvwxyz1234', 'tg3')]:
+            tdLog.info(" alter tag check has pass!")
+        else:
+            tdLog.info(" alter tag failed , please check !")
+
+        # test boundary about tags
+        tdSql.execute("create stable stb1 (ts timestamp , a int) tags (tg1 binary(16374))")
+        tdSql.error("create stable stb1 (ts timestamp , a int) tags (tg1 binary(16375))")
+        bound_sql = "create stable stb2 (ts timestamp , a int) tags (tg1 binary(10),"
+        for i in range(127):
+            bound_sql+="tag"+str(i)+" binary(10),"
+        sql1 = bound_sql[:-1]+")"
+        tdSql.execute(sql1)
+        sql2 = bound_sql[:-1]+"tag127 binary(10))"
+        tdSql.error(sql2)
+        tdSql.execute("create stable stb3 (ts timestamp , a int) tags (tg1 nchar(4093))")
+        tdSql.error("create stable stb3 (ts timestamp , a int) tags (tg1 nchar(4094))")
+        tdSql.execute("create stable stb4 (ts timestamp , a int) tags (tg1 nchar(4093),tag2 binary(8))")
+        tdSql.error("create stable stb4 (ts timestamp , a int) tags (tg1 nchar(4093),tag2 binary(9))")
+        tdSql.execute("create stable stb5 (ts timestamp , a int) tags (tg1 nchar(4093),tag2 binary(4),tag3 binary(2))")
+        tdSql.error("create stable stb5 (ts timestamp , a int) tags (tg1 nchar(4093),tag2 binary(4),tag3 binary(3))")
+
+        tdSql.execute("create table stt (ts timestamp , a binary(100)) tags (tg1 binary(20), tg2 binary(20), tg3 binary(20))")
+        tdSql.execute("insert into tt using stt (tg3, tg2, tg1) tags ('tg3', 'tg2', 'tg1') values (now, 1)")
+        tags = "t"*16337
+        sql3 = "alter table tt set tag tg1=" +"'"+tags+"'"
+        tdSql.error(sql3)
+        tdSql.execute("alter stable stt modify tag tg1 binary(16337)")
+        tdSql.execute(sql3)
+        res = tdSql.getResult("select tg1,tg2,tg3 from tt")
+        if res == [(tags, 'tg2', 'tg3')]:
+            tdLog.info(" alter tag check has pass!")
+        else:
+            tdLog.info(" alter tag failed , please check !")
+        
+        os.system("rm -rf ./tag_lite/TestModifyTag.py.sql")
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

tests/pytest/tools/taosdemoAllTest/taosdemoTestSupportNanosubscribe.py

@@ -79,7 +79,7 @@ class TDTestCase:
 
 
         # merge result files
-        sleep(10)
+        sleep(5)
         os.system("cat subscribe_res0.txt* > all_subscribe_res0.txt")
         os.system("cat subscribe_res1.txt* > all_subscribe_res1.txt")
         os.system("cat subscribe_res2.txt* > all_subscribe_res2.txt")
@@ -99,7 +99,7 @@ class TDTestCase:
         # insert extral data     
         tdSql.execute("use subnsdb")
         tdSql.execute("insert into tb0_0 values(now,100.1000,'subtest1',now-1s)")
-        sleep(1)   
+        sleep(15)   
 
         os.system("cat subscribe_res0.txt* > all_subscribe_res0.txt")
         subTimes0 = self.subTimes("all_subscribe_res0.txt")

tests/pytest/tools/taosdemoPerformance.py

@@ -20,9 +20,11 @@ from util.log import tdLog
 from util.sql import tdSql
 
 class taosdemoPerformace:
-    def __init__(self, commitID, dbName):
+    def __init__(self, commitID, dbName, branch, type):
         self.commitID = commitID
         self.dbName = dbName
+        self.branch = branch
+        self.type = type
         self.host = "127.0.0.1"
         self.user = "root"
         self.password = "taosdata"
@@ -33,6 +35,12 @@ class taosdemoPerformace:
             self.password,
             self.config)
         self.insertDB = "insertDB"
+        self.host2 = "192.168.1.179"    
+        self.conn2 = taos.connect(
+            host = self.host2,
+            user = self.user,
+            password = self.password,
+            config = self.config)
 
     def generateJson(self):
         db = {
@@ -122,12 +130,9 @@ class taosdemoPerformace:
         return buildPath
 
     def insertData(self):
-
         buildPath = self.getBuildPath()
         if (buildPath == ""):
             tdLog.exit("taosdemo not found!")
-        else:
-            tdLog.info("taosdemo found in %s" % buildPath)
         binPath = buildPath + "/build/bin/"
 
         os.system(
@@ -153,11 +158,11 @@ class taosdemoPerformace:
         os.system("[ -f taosdemoperf.txt ] && rm taosdemoperf.txt")
 
     def createTablesAndStoreData(self):
-        cursor = self.conn.cursor()
+        cursor = self.conn2.cursor()
 
         cursor.execute("create database if not exists %s" % self.dbName)
         cursor.execute("use %s" % self.dbName)
-        cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)")
+        cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float, branch binary(50), type binary(20))")
         print("==================== taosdemo performance ====================")
         print("create tables time: %f" % float(self.createTableTime))
         print("insert records time: %f" % float(self.insertRecordsTime))
@@ -165,19 +170,14 @@ class taosdemoPerformace:
         print("avg delay: %f" % float(self.avgDelay))
         print("max delay: %f" % float(self.maxDelay))
         print("min delay: %f" % float(self.minDelay))
-        cursor.execute(
-            "insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" %
-            (float(
-                self.createTableTime), float(
-                self.insertRecordsTime), float(
-                self.recordsPerSecond), self.commitID, float(
-                    self.avgDelay), float(
-                        self.maxDelay), float(
-                            self.minDelay)))
-        cursor.execute("drop database if exists %s" % self.insertDB)
-
+        cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f, '%s', '%s')" %
+            (float(self.createTableTime), float(self.insertRecordsTime), float(self.recordsPerSecond), 
+            self.commitID, float(self.avgDelay), float(self.maxDelay), float(self.minDelay), self.branch, self.type))
         cursor.close()
 
+        cursor1 = self.conn.cursor()
+        cursor1.execute("drop database if exists %s" % self.insertDB)
+        cursor1.close()
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
@@ -194,9 +194,22 @@ if __name__ == '__main__':
         default='perf',
         type=str,
         help='Database name to be created (default: perf)')
-
+    parser.add_argument(
+        '-b',
+        '--git-branch',
+        action='store',
+        default='master',
+        type=str,
+        help='git branch (default: master)')
+    parser.add_argument(
+        '-T',
+        '--build-type',
+        action='store',
+        default='glibc',
+        type=str,
+        help='build type (default: glibc)')
     args = parser.parse_args()
 
-    perftest = taosdemoPerformace(args.commit_id, args.database_name)
+    perftest = taosdemoPerformace(args.commit_id, args.database_name, args.git_branch, args.build_type)
     perftest.insertData()
     perftest.createTablesAndStoreData()

tests/script/general/field/binary.sim

@@ -30,7 +30,7 @@ while $i < 5
   $x = 0
   while $x < $rowNum
     $ms = $x . m 
-    sql insert into $tb values (now + $ms , '0' ) 
+    sql insert into $tb values (1626739200000 + $ms , '0' ) 
     $x = $x + 1
   endw  
   $i = $i + 1
@@ -41,7 +41,7 @@ while $i < 10
   $x = 0
   while $x < $rowNum
     $ms = $x . m 
-    sql insert into $tb values (now + $ms , '1' ) 
+    sql insert into $tb values (1626739200000 + $ms , '1' ) 
     $x = $x + 1
   endw  
   $i = $i + 1
@@ -55,14 +55,14 @@ if $rows != 100 then
   return -1
 endi
 
-sql select * from $mt where ts > now + 4m and tbcol = '1'
+sql select * from $mt where ts > 1626739440001 and tbcol = '1'
 if $rows != 75 then
   return -1
 endi
 
 print select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = '1'
 sql_error select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = '1' group by tgcol 
-sql_error select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m and tbcol = '1' group by tgcol 
+sql_error select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < 1626739440001 and tbcol = '1' group by tgcol 
 sql_error select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = '1' interval(1d) group by tgcol 
 
 #can't filter binary fields