From e1739b86a90bc773e01ed1be112d0719b216e7ea Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 2 Apr 2021 19:22:04 +0800 Subject: [PATCH] Hotfix/sangshuduo/td 3607 taosdemo buffer overflow (#5659) * [TD-3607] : fix taosdemo buffer overflow. * [TD-3607] : taosdemo buffer overflow. add tmp buffer. * [TD-3607] : taosdemo buffer overflow. fix data generation. * [TD-3607] : taosdemo buffer overflow. fix normal table writting. * [TD-3607] : taosdemo buffer overflow. remove tail spaces. * [TD-3607] : taosdemo buffer overflow. fix taosdemo alter table test case. * [TD-3607] : taosdemo buffer overflow. fix taosdemo alter table case. * [TD-3607] : taosdemo buffer overflow. adjust limit offset count warning. * [TD-3607] : taosdemo buffer overflow. add more logic for child tables exist. * [TD-3607] : taosdemo buffer overflow. create database if database be dropped only. * [TD-3607] : fix taosdemo buffer overflow. adjust limit and offset test cases. * [TD-3607] : taosdemo buffer overflow. adjust sample data test case. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 181 ++++++++++-------- .../insert-tblimit-tboffset-createdb.json | 57 ++++++ .../insert-tblimit-tboffset-insertrec.json | 59 ++++++ .../tools/insert-tblimit-tboffset0.json | 4 +- .../tools/insert-tblimit1-tboffset.json | 4 +- tests/pytest/tools/taosdemo-sampledata.json | 2 - tests/pytest/tools/taosdemoTestLimitOffset.py | 5 +- tests/pytest/tools/taosdemoTestSampleData.py | 2 +- 8 files changed, 224 insertions(+), 90 deletions(-) create mode 100644 tests/pytest/tools/insert-tblimit-tboffset-createdb.json create mode 100644 tests/pytest/tools/insert-tblimit-tboffset-insertrec.json diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 13c8445e64..58d4a1820c 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -2433,82 +2433,82 @@ static int createDatabasesAndStables() { taos_close(taos); return -1; } - } - - int dataLen = 0; - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName); - if (g_Dbs.db[i].dbCfg.blocks > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks); - } - if (g_Dbs.db[i].dbCfg.cache > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache); - } - if (g_Dbs.db[i].dbCfg.days > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days); - } - if (g_Dbs.db[i].dbCfg.keep > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep); - } - if (g_Dbs.db[i].dbCfg.quorum > 1) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum); - } - if (g_Dbs.db[i].dbCfg.replica > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica); - } - if (g_Dbs.db[i].dbCfg.update > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update); - } - //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { - // dataLen += snprintf(command + dataLen, - // BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); - //} - if (g_Dbs.db[i].dbCfg.minRows > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows); - } - if (g_Dbs.db[i].dbCfg.maxRows > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows); - } - if (g_Dbs.db[i].dbCfg.comp > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp); - } - if (g_Dbs.db[i].dbCfg.walLevel > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel); - } - if (g_Dbs.db[i].dbCfg.cacheLast > 0) { + int dataLen = 0; dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast); - } - if (g_Dbs.db[i].dbCfg.fsync > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, - " fsync %d", g_Dbs.db[i].dbCfg.fsync); - } - if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms"))) - || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, - "us", strlen("us")))) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, - " precision \'%s\';", g_Dbs.db[i].dbCfg.precision); - } + BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName); - debugPrint("%s() %d command: %s\n", __func__, __LINE__, command); - if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { - taos_close(taos); - errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); - return -1; + if (g_Dbs.db[i].dbCfg.blocks > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.quorum > 1) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update); + } + //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { + // dataLen += snprintf(command + dataLen, + // BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); + //} + if (g_Dbs.db[i].dbCfg.minRows > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.cacheLast > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, + " fsync %d", g_Dbs.db[i].dbCfg.fsync); + } + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms"))) + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, + "us", strlen("us")))) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, + " precision \'%s\';", g_Dbs.db[i].dbCfg.precision); + } + + debugPrint("%s() %d command: %s\n", __func__, __LINE__, command); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { + taos_close(taos); + errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); + return -1; + } + printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); } - printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); debugPrint("%s() %d supertbl count:%d\n", __func__, __LINE__, g_Dbs.db[i].superTblCount); @@ -5132,16 +5132,14 @@ static void startMultiThreadInsertData(int threads, char* db_name, if (superTblInfo) { int limit, offset; - if (superTblInfo->childTblOffset >= superTblInfo->childTblCount) { - printf("WARNING: specified offset >= child table count! \n"); - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + if ((superTblInfo->childTblExists == TBL_NO_EXISTS) && + ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit != 0))) { + printf("WARNING: offset and limit will not be used since the child tables are not exists!\n"); } - if (superTblInfo->childTblOffset >= 0) { - if (superTblInfo->childTblLimit <= 0) { + if ((superTblInfo->childTblExists == TBL_ALREADY_EXISTS) + && (superTblInfo->childTblOffset >= 0)) { + if (superTblInfo->childTblLimit < 0) { superTblInfo->childTblLimit = superTblInfo->childTblCount - superTblInfo->childTblOffset; } @@ -5149,13 +5147,32 @@ static void startMultiThreadInsertData(int threads, char* db_name, offset = superTblInfo->childTblOffset; limit = superTblInfo->childTblLimit; } else { - limit = superTblInfo->childTblCount; - offset = 0; + limit = superTblInfo->childTblCount; + offset = 0; } ntables = limit; startFrom = offset; + if ((superTblInfo->childTblExists != TBL_NO_EXISTS) + && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit ) + > superTblInfo->childTblCount)) { + printf("WARNING: specified offset + limit > child table count!\n"); + if (!g_args.answer_yes) { + printf(" Press enter key to continue or Ctrl-C to stop\n\n"); + (void)getchar(); + } + } + + if ((superTblInfo->childTblExists != TBL_NO_EXISTS) + && (0 == superTblInfo->childTblLimit)) { + printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n"); + if (!g_args.answer_yes) { + printf(" Press enter key to continue or Ctrl-C to stop\n\n"); + (void)getchar(); + } + } + superTblInfo->childTblName = (char*)calloc(1, limit * TSDB_TABLE_NAME_LEN); if (superTblInfo->childTblName == NULL) { diff --git a/tests/pytest/tools/insert-tblimit-tboffset-createdb.json b/tests/pytest/tools/insert-tblimit-tboffset-createdb.json new file mode 100644 index 0000000000..8c3b39fb0b --- /dev/null +++ b/tests/pytest/tools/insert-tblimit-tboffset-createdb.json @@ -0,0 +1,57 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 0, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json b/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json new file mode 100644 index 0000000000..a9efa7c31c --- /dev/null +++ b/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json @@ -0,0 +1,59 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"yes", + "childtable_count": 100, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 1000, + "childtable_limit": 33, + "childtable_offset": 33, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/insert-tblimit-tboffset0.json b/tests/pytest/tools/insert-tblimit-tboffset0.json index 7dcb2e0527..6bf3783cb2 100644 --- a/tests/pytest/tools/insert-tblimit-tboffset0.json +++ b/tests/pytest/tools/insert-tblimit-tboffset0.json @@ -15,7 +15,7 @@ "databases": [{ "dbinfo": { "name": "db", - "drop": "yes", + "drop": "no", "replica": 1, "days": 10, "cache": 16, @@ -33,7 +33,7 @@ }, "super_tables": [{ "name": "stb", - "child_table_exists":"no", + "child_table_exists":"yes", "childtable_count": 100, "childtable_prefix": "stb_", "auto_create_table": "no", diff --git a/tests/pytest/tools/insert-tblimit1-tboffset.json b/tests/pytest/tools/insert-tblimit1-tboffset.json index a33dc22d5d..292902093d 100644 --- a/tests/pytest/tools/insert-tblimit1-tboffset.json +++ b/tests/pytest/tools/insert-tblimit1-tboffset.json @@ -15,7 +15,7 @@ "databases": [{ "dbinfo": { "name": "db", - "drop": "yes", + "drop": "no", "replica": 1, "days": 10, "cache": 16, @@ -33,7 +33,7 @@ }, "super_tables": [{ "name": "stb", - "child_table_exists":"no", + "child_table_exists":"yes", "childtable_count": 100, "childtable_prefix": "stb_", "auto_create_table": "no", diff --git a/tests/pytest/tools/taosdemo-sampledata.json b/tests/pytest/tools/taosdemo-sampledata.json index 473c977773..d543b7e086 100644 --- a/tests/pytest/tools/taosdemo-sampledata.json +++ b/tests/pytest/tools/taosdemo-sampledata.json @@ -16,8 +16,6 @@ "name": "stb", "child_table_exists":"no", "childtable_count": 20, - "childtable_limit": 10, - "childtable_offset": 0, "childtable_prefix": "t_", "auto_create_table": "no", "data_source": "sample", diff --git a/tests/pytest/tools/taosdemoTestLimitOffset.py b/tests/pytest/tools/taosdemoTestLimitOffset.py index bce41e1c75..dd8a1bee70 100644 --- a/tests/pytest/tools/taosdemoTestLimitOffset.py +++ b/tests/pytest/tools/taosdemoTestLimitOffset.py @@ -51,7 +51,8 @@ class TDTestCase: else: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" - os.system("%staosdemo -f tools/insert-tblimit-tboffset.json" % binPath) + os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath) + os.system("%staosdemo -f tools/insert-tblimit-tboffset-insertrec.json" % binPath) tdSql.execute("use db") tdSql.query("select count(tbname) from db.stb") @@ -59,6 +60,7 @@ class TDTestCase: tdSql.query("select count(*) from db.stb") tdSql.checkData(0, 0, 33000) + os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath) os.system("%staosdemo -f tools/insert-tblimit-tboffset0.json" % binPath) tdSql.execute("reset query cache") @@ -68,6 +70,7 @@ class TDTestCase: tdSql.query("select count(*) from db.stb") tdSql.checkData(0, 0, 20000) + os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath) os.system("%staosdemo -f tools/insert-tblimit1-tboffset.json" % binPath) tdSql.execute("reset query cache") diff --git a/tests/pytest/tools/taosdemoTestSampleData.py b/tests/pytest/tools/taosdemoTestSampleData.py index 893c53984d..a8710a9df3 100644 --- a/tests/pytest/tools/taosdemoTestSampleData.py +++ b/tests/pytest/tools/taosdemoTestSampleData.py @@ -57,7 +57,7 @@ class TDTestCase: tdSql.query("select count(tbname) from db.stb") tdSql.checkData(0, 0, 20) tdSql.query("select count(*) from db.stb") - tdSql.checkData(0, 0, 200) + tdSql.checkData(0, 0, 400) def stop(self): tdSql.close() -- GitLab