diff --git a/Jenkinsfile b/Jenkinsfile index 583fcc7efdacfc6aebe67b5c2233bb1b42efbb48..2ea765759059ecb6dfc8d8136c0f00c0cbb9ddff 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -79,14 +79,25 @@ pipeline { changeRequest() } parallel { - stage('python') { - agent{label 'pytest'} + stage('python_1') { + agent{label 'p1'} steps { pre_test() sh ''' cd ${WKC}/tests - ./test-all.sh pytestfq + ./test-all.sh p1 + date''' + } + } + stage('python_2') { + agent{label 'p2'} + steps { + + pre_test() + sh ''' + cd ${WKC}/tests + ./test-all.sh p2 date''' } } diff --git a/packaging/tools/set_core.sh b/packaging/tools/set_core.sh index 9f6c5ae9d12f15a72d2beafcfb2cd28f88a10dc6..f515fd3dc5ed4a26fc7f4120573d39303ea3d751 100755 --- a/packaging/tools/set_core.sh +++ b/packaging/tools/set_core.sh @@ -9,7 +9,7 @@ GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' -set -e +# set -e # set -x corePath=$1 diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index afdc84a7f55ea4bb24fe8f1e02ebbeebfe9ac9bf..f81364ff32757c735b18a9984d25b19f905f6e45 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -200,8 +200,8 @@ float tsTotalTmpDirGB = 0; float tsTotalDataDirGB = 0; float tsAvailTmpDirectorySpace = 0; float tsAvailDataDirGB = 0; -float tsReservedTmpDirectorySpace = 0.1f; -float tsMinimalDataDirGB = 0.5f; +float tsReservedTmpDirectorySpace = 1.0f; +float tsMinimalDataDirGB = 1.0f; int32_t tsTotalMemoryMB = 0; int32_t tsVersion = 0; diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 53e7d2398450fe22a11da1e7254e0c2ab5f02ea4..1d77a6bb6395d7b171a3d79ee75df27ec1826f34 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -95,7 +95,7 @@ typedef struct DemoArguments { {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 3}, #endif {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, - {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 3}, + {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 3}, {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 3}, {0, 's', "sql file", 0, "The select sql file.", 3}, {0, 'M', 0, 0, "Use metric flag.", 13}, @@ -205,10 +205,10 @@ typedef struct DemoArguments { arguments->tb_prefix = arg; break; case 'M': - arguments->use_metric = false; + arguments->use_metric = true; break; case 'x': - arguments->insert_only = false; + arguments->insert_only = true; break; case 'c': if (wordexp(arg, &full_path, 0) != 0) { @@ -406,9 +406,9 @@ typedef struct DemoArguments { } else if (strcmp(argv[i], "-m") == 0) { arguments->tb_prefix = argv[++i]; } else if (strcmp(argv[i], "-M") == 0) { - arguments->use_metric = false; + arguments->use_metric = true; } else if (strcmp(argv[i], "-x") == 0) { - arguments->insert_only = false; + arguments->insert_only = true; } else if (strcmp(argv[i], "-c") == 0) { strcpy(configDir, argv[++i]); } else if (strcmp(argv[i], "-O") == 0) { @@ -476,6 +476,14 @@ typedef struct { int notFinished; tsem_t lock_sem; int counter; + + // insert delay statitics + int64_t cntDelay; + int64_t totalDelay; + int64_t avgDelay; + int64_t maxDelay; + int64_t minDelay; + } info; typedef struct { @@ -575,7 +583,7 @@ int main(int argc, char *argv[]) { arguments.num_of_DPT = 100000; arguments.num_of_RPR = 1000; arguments.use_metric = true; - arguments.insert_only = true; + arguments.insert_only = false; // end change parse_args(argc, argv, &arguments); @@ -739,6 +747,9 @@ int main(int argc, char *argv[]) { printf("Inserting data......\n"); pthread_t *pids = malloc(threads * sizeof(pthread_t)); info *infos = malloc(threads * sizeof(info)); + + memset(pids, 0, threads * sizeof(pthread_t)); + memset(infos, 0, threads * sizeof(info)); int a = ntables / threads; if (a < 1) { @@ -768,6 +779,7 @@ int main(int argc, char *argv[]) { t_info->end_table_id = i < b ? last + a : last + a - 1; last = t_info->end_table_id + 1; t_info->counter = 0; + t_info->minDelay = INT16_MAX; tsem_init(&(t_info->mutex_sem), 0, 1); t_info->notFinished = t_info->end_table_id - t_info->start_table_id + 1; @@ -799,12 +811,29 @@ int main(int argc, char *argv[]) { t, (int64_t)ntables * nrecords_per_table, nrecords_per_request, (int64_t)ntables * nrecords_per_table / t); + int64_t totalDelay = 0; + int64_t maxDelay = 0; + int64_t minDelay = INT16_MAX; + int64_t cntDelay = 0; + double avgDelay = 0; for (int i = 0; i < threads; i++) { info *t_info = infos + i; taos_close(t_info->taos); tsem_destroy(&(t_info->mutex_sem)); tsem_destroy(&(t_info->lock_sem)); + + totalDelay += t_info->totalDelay; + cntDelay += t_info->cntDelay; + if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay; + if (t_info->minDelay < minDelay) minDelay = t_info->minDelay; } + avgDelay = (double)totalDelay / cntDelay; + + fprintf(fp, "insert delay, avg:%10.6fms, max: %10.6fms, min: %10.6fms\n\n", + avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); + + printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n", + avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); free(pids); free(infos); @@ -859,7 +888,7 @@ int main(int argc, char *argv[]) { } - if (!insert_only) { + if (false == insert_only) { // query data pthread_t read_id; info *rInfo = malloc(sizeof(info)); @@ -998,7 +1027,7 @@ void * createTable(void *sarg) /* Create all the tables; */ printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s;", winfo->db_name, winfo->tb_prefix, i, winfo->cols); + snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s);", winfo->db_name, winfo->tb_prefix, i, winfo->cols); queryDB(winfo->taos, command); } } else { @@ -1204,6 +1233,41 @@ void *readMetric(void *sarg) { return NULL; } +static int queryDbExec(TAOS *taos, char *command, int type) { + int i; + TAOS_RES *res = NULL; + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != res) { + taos_free_result(res); + res = NULL; + } + + res = taos_query(taos, command); + code = taos_errno(res); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(res)); + taos_free_result(res); + //taos_close(taos); + return -1; + } + + if (1 == type) { + int affectedRows = taos_affected_rows(res); + taos_free_result(res); + return affectedRows; + } + + taos_free_result(res); + return 0; +} + void queryDB(TAOS *taos, char *command) { int i; TAOS_RES *pSql = NULL; @@ -1273,7 +1337,21 @@ void *syncWrite(void *sarg) { } /* puts(buffer); */ - queryDB(winfo->taos, buffer); + int64_t startTs; + int64_t endTs; + startTs = taosGetTimestampUs(); + //queryDB(winfo->taos, buffer); + int affectedRows = queryDbExec(winfo->taos, buffer, 1); + + if (0 <= affectedRows){ + endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + if (delay > winfo->maxDelay) winfo->maxDelay = delay; + if (delay < winfo->minDelay) winfo->minDelay = delay; + winfo->cntDelay++; + winfo->totalDelay += delay; + //winfo->avgDelay = (double)winfo->totalDelay / winfo->cntDelay; + } if (tID == winfo->end_table_id) { i = inserted; diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index c0f89e8465c35611cdf6437d8f8a64572ddc2d6b..76ea6fa1e55ff3749e72a6903d61befbd9cc1e64 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -67,7 +67,7 @@ int32_t tsLogKeepDays = 0; int32_t tsAsyncLog = 1; float tsTotalLogDirGB = 0; float tsAvailLogDirGB = 0; -float tsMinimalLogDirGB = 0.1f; +float tsMinimalLogDirGB = 1.0f; #ifdef _TD_POWER_ char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power"; #else diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh index 05d6ec1cecdf59737f9bf5a8b2417b9adde0cc87..645d89899e3fb88bf68eb13c96b01699f1382a31 100755 --- a/tests/pytest/pytest_1.sh +++ b/tests/pytest/pytest_1.sh @@ -19,7 +19,8 @@ python3 ./test.py -f insert/randomNullCommit.py python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py -python3 ./test.py -f query/isNullTest.py +#python3 ./test.py -f insert/before_1970.py +python3 bug2265.py #table python3 ./test.py -f table/alter_wal0.py @@ -33,7 +34,7 @@ python3 ./test.py -f table/alter_column.py python3 ./test.py -f table/boundary.py python3 ./test.py -f table/create.py python3 ./test.py -f table/del_stable.py -python3 ./test.py -f table/queryWithTaosdKilled.py + # tag python3 ./test.py -f tag_lite/filter.py @@ -163,10 +164,16 @@ python3 ./test.py -f query/bug1471.py python3 ./test.py -f query/bug1874.py python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py -python3 ./test.py -f query/bug2218.py +python3 ./test.py -f query/bug2218.py +python3 ./test.py -f query/bug2117.py +python3 ./test.py -f query/bug2118.py +python3 ./test.py -f query/bug2143.py +python3 ./test.py -f query/sliding.py +python3 ./test.py -f query/unionAllTest.py python3 ./test.py -f query/bug2281.py python3 ./test.py -f query/bug2119.py -python3 bug2265.py +python3 ./test.py -f query/isNullTest.py +python3 ./test.py -f query/queryWithTaosdKilled.py python3 ./test.py -f query/floatCompare.py #stream @@ -184,6 +191,7 @@ python3 ./test.py -f alter/alter_table_crash.py python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py python3 ./test.py -f client/alterDatabase.py +python3 ./test.py -f client/noConnectionErrorTest.py # Misc python3 testCompress.py @@ -207,7 +215,7 @@ python3 ./test.py -f functions/function_spread.py -r 1 python3 ./test.py -f functions/function_stddev.py -r 1 python3 ./test.py -f functions/function_sum.py -r 1 python3 ./test.py -f functions/function_top.py -r 1 -#python3 ./test.py -f functions/function_twa.py -r 1 +python3 ./test.py -f functions/function_twa.py -r 1 python3 ./test.py -f functions/function_twa_test2.py python3 queryCount.py python3 ./test.py -f query/queryGroupbyWithInterval.py @@ -219,10 +227,10 @@ python3 test.py -f query/queryFillTest.py python3 test.py -f tools/taosdemoTest.py python3 test.py -f tools/taosdumpTest.py python3 test.py -f tools/lowaTest.py +python3 test.py -f tools/taosdemoTest2.py # subscribe python3 test.py -f subscribe/singlemeter.py #python3 test.py -f subscribe/stability.py python3 test.py -f subscribe/supertable.py - diff --git a/tests/pytest/pytest_2.sh b/tests/pytest/pytest_2.sh index fededea3bb34481aa34c37d4a802ba183494e491..28c0a38351a14268594497431a19ec7f83bc05cb 100755 --- a/tests/pytest/pytest_2.sh +++ b/tests/pytest/pytest_2.sh @@ -12,6 +12,7 @@ python3 ./test.py -f update/merge_commit_data2.py python3 ./test.py -f update/merge_commit_data2_update0.py python3 ./test.py -f update/merge_commit_last-0.py python3 ./test.py -f update/merge_commit_last.py +python3 ./test.py -f update/bug_td2279.py # wal python3 ./test.py -f wal/addOldWalTest.py \ No newline at end of file diff --git a/tests/test-all.sh b/tests/test-all.sh index 30d687de981608f38d51f372a21ca0e2352f3017..a85ac39b44cc40a8db4c889a8cd793ab2b785aa2 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -202,10 +202,10 @@ if [ "$2" != "sim" ]; then runPyCaseOneByOnefq fulltest.sh elif [ "$1" == "p1" ]; then echo "### run Python_1 test ###" - runPyCaseOneByOne pytest_1.sh + runPyCaseOneByOnefq pytest_1.sh elif [ "$1" == "p2" ]; then echo "### run Python_2 test ###" - runPyCaseOneByOne pytest_2.sh + runPyCaseOneByOnefq pytest_2.sh elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then exit $(($totalFailed + $totalPyFailed)) elif [ "$1" == "smoke" ] || [ -z "$1" ]; then