Hotfix/sangshuduo/td 3607 taosdemo buffer overflow (#5659)

* [TD-3607] <fix>: fix taosdemo buffer overflow.

* [TD-3607] <fix>: taosdemo buffer overflow.

add tmp buffer.

* [TD-3607] <fix>: taosdemo buffer overflow.

fix data generation.

* [TD-3607] <fix>: taosdemo buffer overflow.

fix normal table writting.

* [TD-3607] <fix>: taosdemo buffer overflow.

remove tail spaces.

* [TD-3607] <fix>: taosdemo buffer overflow.

fix taosdemo alter table test case.

* [TD-3607] <fix>: taosdemo buffer overflow.

fix taosdemo alter table case.

* [TD-3607] <fix>: taosdemo buffer overflow.

adjust limit offset count warning.

* [TD-3607] <fix>: taosdemo buffer overflow.

add more logic for child tables exist.

* [TD-3607] <fix>: taosdemo buffer overflow.

create database if database be dropped only.

* [TD-3607] <fix>: fix taosdemo buffer overflow.

adjust limit and offset test cases.

* [TD-3607] <fix>: taosdemo buffer overflow.

adjust sample data test case.
Co-authored-by: NShuduo Sang <sdsang@taosdata.com>

src/kit/taosdemo/taosdemo.c

@@ -2433,82 +2433,82 @@ static int createDatabasesAndStables() {
         taos_close(taos);
         return -1;
       }
-    }
-
-    int dataLen = 0;
-    dataLen += snprintf(command + dataLen,
-        BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
 
-    if (g_Dbs.db[i].dbCfg.blocks > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
-    }
-    if (g_Dbs.db[i].dbCfg.cache > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
-    }
-    if (g_Dbs.db[i].dbCfg.days > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
-    }
-    if (g_Dbs.db[i].dbCfg.keep > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
-    }
-    if (g_Dbs.db[i].dbCfg.quorum > 1) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
-    }
-    if (g_Dbs.db[i].dbCfg.replica > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
-    }
-    if (g_Dbs.db[i].dbCfg.update > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
-    }
-    //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
-    //  dataLen += snprintf(command + dataLen,
-    //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
-    //}
-    if (g_Dbs.db[i].dbCfg.minRows > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
-    }
-    if (g_Dbs.db[i].dbCfg.maxRows > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
-    }
-    if (g_Dbs.db[i].dbCfg.comp > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
-    }
-    if (g_Dbs.db[i].dbCfg.walLevel > 0) {
-      dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
-    }
-    if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
+      int dataLen = 0;
       dataLen += snprintf(command + dataLen,
-          BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
-    }
-    if (g_Dbs.db[i].dbCfg.fsync > 0) {
-      dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
-              " fsync %d", g_Dbs.db[i].dbCfg.fsync);
-    }
-    if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
-            || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
-                    "us", strlen("us")))) {
-      dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
-              " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
-    }
+          BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName);
 
-    debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
-    if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
-      taos_close(taos);
-      errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
-      return -1;
+      if (g_Dbs.db[i].dbCfg.blocks > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks);
+      }
+      if (g_Dbs.db[i].dbCfg.cache > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache);
+      }
+      if (g_Dbs.db[i].dbCfg.days > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days);
+      }
+      if (g_Dbs.db[i].dbCfg.keep > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep);
+      }
+      if (g_Dbs.db[i].dbCfg.quorum > 1) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum);
+      }
+      if (g_Dbs.db[i].dbCfg.replica > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica);
+      }
+      if (g_Dbs.db[i].dbCfg.update > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update);
+      }
+      //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) {
+      //  dataLen += snprintf(command + dataLen,
+      //  BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode);
+      //}
+      if (g_Dbs.db[i].dbCfg.minRows > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows);
+      }
+      if (g_Dbs.db[i].dbCfg.maxRows > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows);
+      }
+      if (g_Dbs.db[i].dbCfg.comp > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp);
+      }
+      if (g_Dbs.db[i].dbCfg.walLevel > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel);
+      }
+      if (g_Dbs.db[i].dbCfg.cacheLast > 0) {
+        dataLen += snprintf(command + dataLen,
+            BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast);
+      }
+      if (g_Dbs.db[i].dbCfg.fsync > 0) {
+        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
+                " fsync %d", g_Dbs.db[i].dbCfg.fsync);
+      }
+      if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms")))
+              || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision,
+                      "us", strlen("us")))) {
+        dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen,
+                " precision \'%s\';", g_Dbs.db[i].dbCfg.precision);
+      }
+
+      debugPrint("%s() %d command: %s\n", __func__, __LINE__, command);
+      if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) {
+        taos_close(taos);
+        errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName);
+        return -1;
+      }
+      printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
     }
-    printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName);
 
     debugPrint("%s() %d supertbl count:%d\n",
             __func__, __LINE__, g_Dbs.db[i].superTblCount);
@@ -5132,16 +5132,14 @@ static void startMultiThreadInsertData(int threads, char* db_name,
   if (superTblInfo) {
     int limit, offset;
 
-    if (superTblInfo->childTblOffset >= superTblInfo->childTblCount) {
-      printf("WARNING: specified offset >= child table count! \n");
-      if (!g_args.answer_yes) {
-        printf("         Press enter key to continue or Ctrl-C to stop\n\n");
-        (void)getchar();
-      }
+    if ((superTblInfo->childTblExists == TBL_NO_EXISTS) &&
+            ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit != 0))) {
+      printf("WARNING: offset and limit will not be used since the child tables are not exists!\n");
     }
 
-    if (superTblInfo->childTblOffset >= 0) {
-      if (superTblInfo->childTblLimit <= 0) {
+    if ((superTblInfo->childTblExists == TBL_ALREADY_EXISTS)
+            && (superTblInfo->childTblOffset >= 0)) {
+      if (superTblInfo->childTblLimit < 0) {
         superTblInfo->childTblLimit =
             superTblInfo->childTblCount - superTblInfo->childTblOffset;
       }
@@ -5149,13 +5147,32 @@ static void startMultiThreadInsertData(int threads, char* db_name,
       offset = superTblInfo->childTblOffset;
       limit = superTblInfo->childTblLimit;
     } else {
-        limit = superTblInfo->childTblCount;
-        offset = 0;
+      limit = superTblInfo->childTblCount;
+      offset = 0;
     }
 
     ntables = limit;
     startFrom = offset;
 
+    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
+        && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit )
+            > superTblInfo->childTblCount)) {
+      printf("WARNING: specified offset + limit > child table count!\n");
+      if (!g_args.answer_yes) {
+        printf("         Press enter key to continue or Ctrl-C to stop\n\n");
+        (void)getchar();
+      }
+    }
+
+    if ((superTblInfo->childTblExists != TBL_NO_EXISTS)
+            && (0 == superTblInfo->childTblLimit)) {
+      printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n");
+      if (!g_args.answer_yes) {
+        printf("         Press enter key to continue or Ctrl-C to stop\n\n");
+        (void)getchar();
+      }
+    }
+
     superTblInfo->childTblName = (char*)calloc(1,
         limit * TSDB_TABLE_NAME_LEN);
     if (superTblInfo->childTblName == NULL) {

tests/pytest/tools/insert-tblimit-tboffset-createdb.json

+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,
+    "thread_count_create_tbl": 4,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "num_of_records_per_req": 100,
+    "max_sql_len": 1024000,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "yes",
+            "replica": 1,
+            "days": 10,
+            "cache": 16,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 365,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb",
+            "child_table_exists":"no",
+            "childtable_count": 100,
+            "childtable_prefix": "stb_",
+            "auto_create_table": "no",
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 0,
+            "multi_thread_write_one_tbl": "no",
+            "number_of_tbl_in_one_sql": 0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
+        }]
+    }]
+}

tests/pytest/tools/insert-tblimit-tboffset-insertrec.json

+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 4,
+    "thread_count_create_tbl": 4,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "num_of_records_per_req": 100,
+    "max_sql_len": 1024000,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "no",
+            "replica": 1,
+            "days": 10,
+            "cache": 16,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 365,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb",
+            "child_table_exists":"yes",
+            "childtable_count": 100,
+            "childtable_prefix": "stb_",
+            "auto_create_table": "no",
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 1000,
+            "childtable_limit": 33,
+            "childtable_offset": 33,
+            "multi_thread_write_one_tbl": "no",
+            "number_of_tbl_in_one_sql": 0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}]
+        }]
+    }]
+}

tests/pytest/tools/insert-tblimit-tboffset0.json

@@ -15,7 +15,7 @@
     "databases": [{
         "dbinfo": {
             "name": "db",
-            "drop": "yes",
+            "drop": "no",
             "replica": 1,
             "days": 10,
             "cache": 16,
@@ -33,7 +33,7 @@
         },
         "super_tables": [{
             "name": "stb",
-            "child_table_exists":"no",
+            "child_table_exists":"yes",
             "childtable_count": 100,
             "childtable_prefix": "stb_",
             "auto_create_table": "no",

tests/pytest/tools/insert-tblimit1-tboffset.json

@@ -15,7 +15,7 @@
     "databases": [{
         "dbinfo": {
             "name": "db",
-            "drop": "yes",
+            "drop": "no",
             "replica": 1,
             "days": 10,
             "cache": 16,
@@ -33,7 +33,7 @@
         },
         "super_tables": [{
             "name": "stb",
-            "child_table_exists":"no",
+            "child_table_exists":"yes",
             "childtable_count": 100,
             "childtable_prefix": "stb_",
             "auto_create_table": "no",

tests/pytest/tools/taosdemo-sampledata.json

@@ -16,8 +16,6 @@
 			"name": "stb",
 			"child_table_exists":"no",
 			"childtable_count": 20,
-			"childtable_limit": 10,
-			"childtable_offset": 0,
 			"childtable_prefix": "t_",
 			"auto_create_table": "no",
 			"data_source": "sample",

tests/pytest/tools/taosdemoTestLimitOffset.py

@@ -51,7 +51,8 @@ class TDTestCase:
         else:
             tdLog.info("taosd found in %s" % buildPath)
         binPath = buildPath+ "/build/bin/"
-        os.system("%staosdemo -f tools/insert-tblimit-tboffset.json" % binPath)
+        os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath)
+        os.system("%staosdemo -f tools/insert-tblimit-tboffset-insertrec.json" % binPath)
 
         tdSql.execute("use db")
         tdSql.query("select count(tbname) from db.stb")
@@ -59,6 +60,7 @@ class TDTestCase:
         tdSql.query("select count(*) from db.stb")
         tdSql.checkData(0, 0, 33000)
 
+        os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath)
         os.system("%staosdemo -f tools/insert-tblimit-tboffset0.json" % binPath)
 
         tdSql.execute("reset query cache")
@@ -68,6 +70,7 @@ class TDTestCase:
         tdSql.query("select count(*) from db.stb")
         tdSql.checkData(0, 0, 20000)
 
+        os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath)
         os.system("%staosdemo -f tools/insert-tblimit1-tboffset.json" % binPath)
 
         tdSql.execute("reset query cache")

tests/pytest/tools/taosdemoTestSampleData.py

@@ -57,7 +57,7 @@ class TDTestCase:
         tdSql.query("select count(tbname) from db.stb")
         tdSql.checkData(0, 0, 20)
         tdSql.query("select count(*) from db.stb")
-        tdSql.checkData(0, 0, 200)
+        tdSql.checkData(0, 0, 400)
 
     def stop(self):
         tdSql.close()