提交 cc97d60f 编写于 作者: H Hongze Cheng

Merge branch 'develop' into feature/TD-2354

...@@ -79,14 +79,25 @@ pipeline { ...@@ -79,14 +79,25 @@ pipeline {
changeRequest() changeRequest()
} }
parallel { parallel {
stage('python') { stage('python_1') {
agent{label 'pytest'} agent{label 'p1'}
steps { steps {
pre_test() pre_test()
sh ''' sh '''
cd ${WKC}/tests cd ${WKC}/tests
./test-all.sh pytestfq ./test-all.sh p1
date'''
}
}
stage('python_2') {
agent{label 'p2'}
steps {
pre_test()
sh '''
cd ${WKC}/tests
./test-all.sh p2
date''' date'''
} }
} }
......
...@@ -9,7 +9,7 @@ GREEN_DARK='\033[0;32m' ...@@ -9,7 +9,7 @@ GREEN_DARK='\033[0;32m'
GREEN_UNDERLINE='\033[4;32m' GREEN_UNDERLINE='\033[4;32m'
NC='\033[0m' NC='\033[0m'
set -e # set -e
# set -x # set -x
corePath=$1 corePath=$1
......
...@@ -200,8 +200,8 @@ float tsTotalTmpDirGB = 0; ...@@ -200,8 +200,8 @@ float tsTotalTmpDirGB = 0;
float tsTotalDataDirGB = 0; float tsTotalDataDirGB = 0;
float tsAvailTmpDirectorySpace = 0; float tsAvailTmpDirectorySpace = 0;
float tsAvailDataDirGB = 0; float tsAvailDataDirGB = 0;
float tsReservedTmpDirectorySpace = 0.1f; float tsReservedTmpDirectorySpace = 1.0f;
float tsMinimalDataDirGB = 0.5f; float tsMinimalDataDirGB = 1.0f;
int32_t tsTotalMemoryMB = 0; int32_t tsTotalMemoryMB = 0;
int32_t tsVersion = 0; int32_t tsVersion = 0;
......
...@@ -95,7 +95,7 @@ typedef struct DemoArguments { ...@@ -95,7 +95,7 @@ typedef struct DemoArguments {
{0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 3}, {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 3},
#endif #endif
{0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3},
{0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 3}, {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 3},
{0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 3}, {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 3},
{0, 's', "sql file", 0, "The select sql file.", 3}, {0, 's', "sql file", 0, "The select sql file.", 3},
{0, 'M', 0, 0, "Use metric flag.", 13}, {0, 'M', 0, 0, "Use metric flag.", 13},
...@@ -205,10 +205,10 @@ typedef struct DemoArguments { ...@@ -205,10 +205,10 @@ typedef struct DemoArguments {
arguments->tb_prefix = arg; arguments->tb_prefix = arg;
break; break;
case 'M': case 'M':
arguments->use_metric = false; arguments->use_metric = true;
break; break;
case 'x': case 'x':
arguments->insert_only = false; arguments->insert_only = true;
break; break;
case 'c': case 'c':
if (wordexp(arg, &full_path, 0) != 0) { if (wordexp(arg, &full_path, 0) != 0) {
...@@ -406,9 +406,9 @@ typedef struct DemoArguments { ...@@ -406,9 +406,9 @@ typedef struct DemoArguments {
} else if (strcmp(argv[i], "-m") == 0) { } else if (strcmp(argv[i], "-m") == 0) {
arguments->tb_prefix = argv[++i]; arguments->tb_prefix = argv[++i];
} else if (strcmp(argv[i], "-M") == 0) { } else if (strcmp(argv[i], "-M") == 0) {
arguments->use_metric = false; arguments->use_metric = true;
} else if (strcmp(argv[i], "-x") == 0) { } else if (strcmp(argv[i], "-x") == 0) {
arguments->insert_only = false; arguments->insert_only = true;
} else if (strcmp(argv[i], "-c") == 0) { } else if (strcmp(argv[i], "-c") == 0) {
strcpy(configDir, argv[++i]); strcpy(configDir, argv[++i]);
} else if (strcmp(argv[i], "-O") == 0) { } else if (strcmp(argv[i], "-O") == 0) {
...@@ -476,6 +476,14 @@ typedef struct { ...@@ -476,6 +476,14 @@ typedef struct {
int notFinished; int notFinished;
tsem_t lock_sem; tsem_t lock_sem;
int counter; int counter;
// insert delay statitics
int64_t cntDelay;
int64_t totalDelay;
int64_t avgDelay;
int64_t maxDelay;
int64_t minDelay;
} info; } info;
typedef struct { typedef struct {
...@@ -575,7 +583,7 @@ int main(int argc, char *argv[]) { ...@@ -575,7 +583,7 @@ int main(int argc, char *argv[]) {
arguments.num_of_DPT = 100000; arguments.num_of_DPT = 100000;
arguments.num_of_RPR = 1000; arguments.num_of_RPR = 1000;
arguments.use_metric = true; arguments.use_metric = true;
arguments.insert_only = true; arguments.insert_only = false;
// end change // end change
parse_args(argc, argv, &arguments); parse_args(argc, argv, &arguments);
...@@ -739,6 +747,9 @@ int main(int argc, char *argv[]) { ...@@ -739,6 +747,9 @@ int main(int argc, char *argv[]) {
printf("Inserting data......\n"); printf("Inserting data......\n");
pthread_t *pids = malloc(threads * sizeof(pthread_t)); pthread_t *pids = malloc(threads * sizeof(pthread_t));
info *infos = malloc(threads * sizeof(info)); info *infos = malloc(threads * sizeof(info));
memset(pids, 0, threads * sizeof(pthread_t));
memset(infos, 0, threads * sizeof(info));
int a = ntables / threads; int a = ntables / threads;
if (a < 1) { if (a < 1) {
...@@ -768,6 +779,7 @@ int main(int argc, char *argv[]) { ...@@ -768,6 +779,7 @@ int main(int argc, char *argv[]) {
t_info->end_table_id = i < b ? last + a : last + a - 1; t_info->end_table_id = i < b ? last + a : last + a - 1;
last = t_info->end_table_id + 1; last = t_info->end_table_id + 1;
t_info->counter = 0; t_info->counter = 0;
t_info->minDelay = INT16_MAX;
tsem_init(&(t_info->mutex_sem), 0, 1); tsem_init(&(t_info->mutex_sem), 0, 1);
t_info->notFinished = t_info->end_table_id - t_info->start_table_id + 1; t_info->notFinished = t_info->end_table_id - t_info->start_table_id + 1;
...@@ -799,12 +811,29 @@ int main(int argc, char *argv[]) { ...@@ -799,12 +811,29 @@ int main(int argc, char *argv[]) {
t, (int64_t)ntables * nrecords_per_table, nrecords_per_request, t, (int64_t)ntables * nrecords_per_table, nrecords_per_request,
(int64_t)ntables * nrecords_per_table / t); (int64_t)ntables * nrecords_per_table / t);
int64_t totalDelay = 0;
int64_t maxDelay = 0;
int64_t minDelay = INT16_MAX;
int64_t cntDelay = 0;
double avgDelay = 0;
for (int i = 0; i < threads; i++) { for (int i = 0; i < threads; i++) {
info *t_info = infos + i; info *t_info = infos + i;
taos_close(t_info->taos); taos_close(t_info->taos);
tsem_destroy(&(t_info->mutex_sem)); tsem_destroy(&(t_info->mutex_sem));
tsem_destroy(&(t_info->lock_sem)); tsem_destroy(&(t_info->lock_sem));
totalDelay += t_info->totalDelay;
cntDelay += t_info->cntDelay;
if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay;
if (t_info->minDelay < minDelay) minDelay = t_info->minDelay;
} }
avgDelay = (double)totalDelay / cntDelay;
fprintf(fp, "insert delay, avg:%10.6fms, max: %10.6fms, min: %10.6fms\n\n",
avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0);
printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n",
avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0);
free(pids); free(pids);
free(infos); free(infos);
...@@ -859,7 +888,7 @@ int main(int argc, char *argv[]) { ...@@ -859,7 +888,7 @@ int main(int argc, char *argv[]) {
} }
if (!insert_only) { if (false == insert_only) {
// query data // query data
pthread_t read_id; pthread_t read_id;
info *rInfo = malloc(sizeof(info)); info *rInfo = malloc(sizeof(info));
...@@ -998,7 +1027,7 @@ void * createTable(void *sarg) ...@@ -998,7 +1027,7 @@ void * createTable(void *sarg)
/* Create all the tables; */ /* Create all the tables; */
printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id);
for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) {
snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s;", winfo->db_name, winfo->tb_prefix, i, winfo->cols); snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s);", winfo->db_name, winfo->tb_prefix, i, winfo->cols);
queryDB(winfo->taos, command); queryDB(winfo->taos, command);
} }
} else { } else {
...@@ -1204,6 +1233,41 @@ void *readMetric(void *sarg) { ...@@ -1204,6 +1233,41 @@ void *readMetric(void *sarg) {
return NULL; return NULL;
} }
static int queryDbExec(TAOS *taos, char *command, int type) {
int i;
TAOS_RES *res = NULL;
int32_t code = -1;
for (i = 0; i < 5; i++) {
if (NULL != res) {
taos_free_result(res);
res = NULL;
}
res = taos_query(taos, command);
code = taos_errno(res);
if (0 == code) {
break;
}
}
if (code != 0) {
fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(res));
taos_free_result(res);
//taos_close(taos);
return -1;
}
if (1 == type) {
int affectedRows = taos_affected_rows(res);
taos_free_result(res);
return affectedRows;
}
taos_free_result(res);
return 0;
}
void queryDB(TAOS *taos, char *command) { void queryDB(TAOS *taos, char *command) {
int i; int i;
TAOS_RES *pSql = NULL; TAOS_RES *pSql = NULL;
...@@ -1273,7 +1337,21 @@ void *syncWrite(void *sarg) { ...@@ -1273,7 +1337,21 @@ void *syncWrite(void *sarg) {
} }
/* puts(buffer); */ /* puts(buffer); */
queryDB(winfo->taos, buffer); int64_t startTs;
int64_t endTs;
startTs = taosGetTimestampUs();
//queryDB(winfo->taos, buffer);
int affectedRows = queryDbExec(winfo->taos, buffer, 1);
if (0 <= affectedRows){
endTs = taosGetTimestampUs();
int64_t delay = endTs - startTs;
if (delay > winfo->maxDelay) winfo->maxDelay = delay;
if (delay < winfo->minDelay) winfo->minDelay = delay;
winfo->cntDelay++;
winfo->totalDelay += delay;
//winfo->avgDelay = (double)winfo->totalDelay / winfo->cntDelay;
}
if (tID == winfo->end_table_id) { if (tID == winfo->end_table_id) {
i = inserted; i = inserted;
......
...@@ -67,7 +67,7 @@ int32_t tsLogKeepDays = 0; ...@@ -67,7 +67,7 @@ int32_t tsLogKeepDays = 0;
int32_t tsAsyncLog = 1; int32_t tsAsyncLog = 1;
float tsTotalLogDirGB = 0; float tsTotalLogDirGB = 0;
float tsAvailLogDirGB = 0; float tsAvailLogDirGB = 0;
float tsMinimalLogDirGB = 0.1f; float tsMinimalLogDirGB = 1.0f;
#ifdef _TD_POWER_ #ifdef _TD_POWER_
char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power"; char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power";
#else #else
......
...@@ -19,7 +19,8 @@ python3 ./test.py -f insert/randomNullCommit.py ...@@ -19,7 +19,8 @@ python3 ./test.py -f insert/randomNullCommit.py
python3 insert/retentionpolicy.py python3 insert/retentionpolicy.py
python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/alterTableAndInsert.py
python3 ./test.py -f insert/insertIntoTwoTables.py python3 ./test.py -f insert/insertIntoTwoTables.py
python3 ./test.py -f query/isNullTest.py #python3 ./test.py -f insert/before_1970.py
python3 bug2265.py
#table #table
python3 ./test.py -f table/alter_wal0.py python3 ./test.py -f table/alter_wal0.py
...@@ -33,7 +34,7 @@ python3 ./test.py -f table/alter_column.py ...@@ -33,7 +34,7 @@ python3 ./test.py -f table/alter_column.py
python3 ./test.py -f table/boundary.py python3 ./test.py -f table/boundary.py
python3 ./test.py -f table/create.py python3 ./test.py -f table/create.py
python3 ./test.py -f table/del_stable.py python3 ./test.py -f table/del_stable.py
python3 ./test.py -f table/queryWithTaosdKilled.py
# tag # tag
python3 ./test.py -f tag_lite/filter.py python3 ./test.py -f tag_lite/filter.py
...@@ -163,10 +164,16 @@ python3 ./test.py -f query/bug1471.py ...@@ -163,10 +164,16 @@ python3 ./test.py -f query/bug1471.py
python3 ./test.py -f query/bug1874.py python3 ./test.py -f query/bug1874.py
python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1875.py
python3 ./test.py -f query/bug1876.py python3 ./test.py -f query/bug1876.py
python3 ./test.py -f query/bug2218.py python3 ./test.py -f query/bug2218.py
python3 ./test.py -f query/bug2117.py
python3 ./test.py -f query/bug2118.py
python3 ./test.py -f query/bug2143.py
python3 ./test.py -f query/sliding.py
python3 ./test.py -f query/unionAllTest.py
python3 ./test.py -f query/bug2281.py python3 ./test.py -f query/bug2281.py
python3 ./test.py -f query/bug2119.py python3 ./test.py -f query/bug2119.py
python3 bug2265.py python3 ./test.py -f query/isNullTest.py
python3 ./test.py -f query/queryWithTaosdKilled.py
python3 ./test.py -f query/floatCompare.py python3 ./test.py -f query/floatCompare.py
#stream #stream
...@@ -184,6 +191,7 @@ python3 ./test.py -f alter/alter_table_crash.py ...@@ -184,6 +191,7 @@ python3 ./test.py -f alter/alter_table_crash.py
python3 ./test.py -f client/client.py python3 ./test.py -f client/client.py
python3 ./test.py -f client/version.py python3 ./test.py -f client/version.py
python3 ./test.py -f client/alterDatabase.py python3 ./test.py -f client/alterDatabase.py
python3 ./test.py -f client/noConnectionErrorTest.py
# Misc # Misc
python3 testCompress.py python3 testCompress.py
...@@ -207,7 +215,7 @@ python3 ./test.py -f functions/function_spread.py -r 1 ...@@ -207,7 +215,7 @@ python3 ./test.py -f functions/function_spread.py -r 1
python3 ./test.py -f functions/function_stddev.py -r 1 python3 ./test.py -f functions/function_stddev.py -r 1
python3 ./test.py -f functions/function_sum.py -r 1 python3 ./test.py -f functions/function_sum.py -r 1
python3 ./test.py -f functions/function_top.py -r 1 python3 ./test.py -f functions/function_top.py -r 1
#python3 ./test.py -f functions/function_twa.py -r 1 python3 ./test.py -f functions/function_twa.py -r 1
python3 ./test.py -f functions/function_twa_test2.py python3 ./test.py -f functions/function_twa_test2.py
python3 queryCount.py python3 queryCount.py
python3 ./test.py -f query/queryGroupbyWithInterval.py python3 ./test.py -f query/queryGroupbyWithInterval.py
...@@ -219,10 +227,10 @@ python3 test.py -f query/queryFillTest.py ...@@ -219,10 +227,10 @@ python3 test.py -f query/queryFillTest.py
python3 test.py -f tools/taosdemoTest.py python3 test.py -f tools/taosdemoTest.py
python3 test.py -f tools/taosdumpTest.py python3 test.py -f tools/taosdumpTest.py
python3 test.py -f tools/lowaTest.py python3 test.py -f tools/lowaTest.py
python3 test.py -f tools/taosdemoTest2.py
# subscribe # subscribe
python3 test.py -f subscribe/singlemeter.py python3 test.py -f subscribe/singlemeter.py
#python3 test.py -f subscribe/stability.py #python3 test.py -f subscribe/stability.py
python3 test.py -f subscribe/supertable.py python3 test.py -f subscribe/supertable.py
...@@ -12,6 +12,7 @@ python3 ./test.py -f update/merge_commit_data2.py ...@@ -12,6 +12,7 @@ python3 ./test.py -f update/merge_commit_data2.py
python3 ./test.py -f update/merge_commit_data2_update0.py python3 ./test.py -f update/merge_commit_data2_update0.py
python3 ./test.py -f update/merge_commit_last-0.py python3 ./test.py -f update/merge_commit_last-0.py
python3 ./test.py -f update/merge_commit_last.py python3 ./test.py -f update/merge_commit_last.py
python3 ./test.py -f update/bug_td2279.py
# wal # wal
python3 ./test.py -f wal/addOldWalTest.py python3 ./test.py -f wal/addOldWalTest.py
\ No newline at end of file
...@@ -202,10 +202,10 @@ if [ "$2" != "sim" ]; then ...@@ -202,10 +202,10 @@ if [ "$2" != "sim" ]; then
runPyCaseOneByOnefq fulltest.sh runPyCaseOneByOnefq fulltest.sh
elif [ "$1" == "p1" ]; then elif [ "$1" == "p1" ]; then
echo "### run Python_1 test ###" echo "### run Python_1 test ###"
runPyCaseOneByOne pytest_1.sh runPyCaseOneByOnefq pytest_1.sh
elif [ "$1" == "p2" ]; then elif [ "$1" == "p2" ]; then
echo "### run Python_2 test ###" echo "### run Python_2 test ###"
runPyCaseOneByOne pytest_2.sh runPyCaseOneByOnefq pytest_2.sh
elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then
exit $(($totalFailed + $totalPyFailed)) exit $(($totalFailed + $totalPyFailed))
elif [ "$1" == "smoke" ] || [ -z "$1" ]; then elif [ "$1" == "smoke" ] || [ -z "$1" ]; then
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册