From cb802a0c42261c7f53a49f3327220f9b327633a2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 8 Jul 2021 07:01:49 +0800 Subject: [PATCH] Hotfix/sangshuduo/m2d 20210707 (#6795) * [TD-4132]: add taosdump-testcase of boundary value testing * [TD-3952]: add taodemo-testcase that query with restful connector * merge master * modify taosdemo-testcase * add testcase of async subscirbe and max sql num * testcase-subscribe of super table * testcase-modify sub * modify subscribe testcase * [TD-4238]: add taodemo-testcase that records of number in request sql less than 32767 * add taosdemo-subscribe-testcase * update taosdemo testcase * temp commit * temp commit * temp commit * temp1 * temp * [TD-4835] add lua lib * [TD-4369]: add testcase of resubscibe * [TD-4826]: new error code 0x0406 (Dnode is exiting) for tsc * [TD-4369]: add testcase of resubscibe and json file * temp * update master to develop's version * Update boundary2.py * [TD-4912]: fix coredump when drop vnode * Hotfix/sangshuduo/td 3801 taosdump coverity scan issue (#6642) * [TD-3801]: taosdump coverity scan issue. * merge from commit: 069169e7c1 and fix coverity scan issue. Co-authored-by: Shuduo Sang * [TD-4932]fix jenkins errors on master branch (#6645) * [TD-4932]fix jenkins errors on master branch * fix * fix * fix * fix * [TD-4132]:modify taosdump-testcase of column length * [TD-4918]: taosdemo subscribe endAfterResume. (#6653) * [TD-4915] fix show-create-database except * Hotfix/sangshuduo/td 4892 taosdemo sub fetch (#6669) * [TD-4892]: taosdemo subscribe fetch result. * fix stbname length. * restrict prefix length. * submit empty * fix minor code. * fix crash if no result file. * decrease few debug print level. * fix offset issue * [TD-4987]: issues memory sanitizier detected. (#6675) * [TD-4826]: no extra sleep if no message in vwqueue * [TD-4593]: fix vnode cannnot close while syncing * Hotfix/sangshuduo/td 4918 taosdemo sub afterconsume (#6685) * [TD-4918]: taosdemo subscribe endAfterResume. * fix mask length. * fix heap use-after-free. * Hotfix/sangshuduo/td 4987 detected by mem sanitizer (#6684) * [TD-4987]: issues memory sanitizier detected. * fix mask width issue. * [TD-4828] handle dnode exit code * Hotfix/sangshuduo/td 3973 use jemalloc for master (#6702) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo Co-authored-by: Shuduo Sang * TD-4835 * only compile in lua * [td-4748]:fix the temp file generate buf on windows platform. * change cmake options * [TD-5007]: use build directory's taosdump to test. (#6710) btw, solve two taosdemo test cases too. * Hotfix/sangshuduo/td 4838 taosdump binary len bug (#6713) * [TD-4838]: taosdump binary length bug. * taosdump code refactoring. * fix arm32 compile issue. * Hotfix/sangshuduo/td 4918 taosdemo sub afterconsume (#6723) * [TD-4918]: taosdemo subscribe endAfterResume. * fix mask length. * fix heap use-after-free. * fix illegal input value of consume. * Hotfix/sangshuduo/td 3197 taosdemo coverity scan for master (#6725) * [TD-3197]: taosdemo and taosdump coverity scan issues. * exit if read sample file failed. Co-authored-by: Shuduo Sang * [TD-4985]:test query limit and offset function * [TD-4985]:test query limit and offset function * [TD-4985]:test query limit and offset function * [TD-4985]:test query limit and offset function * [TD-5018]: taosdemo prompt if the batch is too large to insert data. (#6736) * fix dead loop issue * fix taosdemoPerformance.py runtime error * Hotfix/sangshuduo/td 3973 use jemalloc for master (#6753) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo * [TD-3937]: support jemalloc add install funtion to all scripts. Co-authored-by: Shuduo Sang * Hotfix/sangshuduo/td 3973 use jemalloc for master (#6763) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for check compatibility. Co-authored-by: Shuduo Sang * Hotfix/sangshuduo/td 3973 use jemalloc for master (#6765) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for check compatibility. * fix position bug. Co-authored-by: Shuduo Sang * [TD-5067]: taosdemo stmt use sample data (#6760) * Hotfix/sangshuduo/td 3973 use jemalloc for master (#6768) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for check compatibility. * fix position bug. * add ldconfig for jemalloc library cache refresh. Co-authored-by: Shuduo Sang * [TD-5063]:test insert locking bug * [TD-5063]:test insert locking bug * [TD-4705]: disallow file sync starting when vnode closing * Hotfix/sangshuduo/td 5053 taosdump support nanosecond for master (#6778) * cherrypick from develop branch * fix arm32 compile issue. * [TD-5053]: taosdump supports nanosecond. * fix precision parsing order issue. * [TD-5053]: taosdump support nanosecond. pre-cherry-pick. * Hotfix/sangshuduo/td 5100 gtest centos for master (#6781) * cherry pick from develop branch. * detect both shared library and static library. * Hotfix/sangshuduo/td 3973 use jemalloc for master (#6789) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for check compatibility. * fix position bug. * add ldconfig for jemalloc library cache refresh. Co-authored-by: Shuduo Sang * Hotfix/sangshuduo/td 3973 use jemalloc for master (#6794) * [TD-3973]: add jemalloc as submodule. * [TD-3973]: add jemalloc as submodule. * [TD-3973]: use jemalloc. build works as following instructions: cmake .. -DJEMALLOC_ENABLED=true make * fix jemalloc at tag 5.2.1 * fix conflicts * make install works. * fix conflicts. * release script works. * fix typo * [TD-3937]: support jemalloc add install funtion to all scripts. * adjust install_jemalloc() position for check compatibility. * fix position bug. * add ldconfig for jemalloc library cache refresh. * cherry pick from develop branch. * cherry pick from develop branch. Co-authored-by: Shuduo Sang * fix taosdumpTest case as -N will lead no property dumped. Co-authored-by: tomchon Co-authored-by: Shengliang Guan Co-authored-by: Minglei Jin Co-authored-by: Baosheng Chang Co-authored-by: Hongze Cheng Co-authored-by: Shuduo Sang Co-authored-by: Yiqing Liu Co-authored-by: Hui Li <52318143+plum-lihui@users.noreply.github.com> Co-authored-by: yihaoDeng Co-authored-by: wpan Co-authored-by: Haojun Liao Co-authored-by: Haojun Liao Co-authored-by: happyguoxy Co-authored-by: Ping Xiao --- src/client/src/tscParseInsert.c | 4 + src/kit/taosdemo/taosdemo.c | 159 +++++++++++---- src/vnode/src/vnodeSync.c | 4 +- tests/pytest/fulltest.sh | 2 + tests/pytest/insert/insert_locking.py | 178 ++++++++++++++++ .../TD-4985/query-limit-offset.csv | 10 + .../TD-4985/query-limit-offset.json | 62 ++++++ .../TD-4985/query-limit-offset.py | 191 ++++++++++++++++++ tests/pytest/tools/taosdemoPerformance.py | 2 + tests/pytest/tools/taosdumpTest.py | 6 +- 10 files changed, 575 insertions(+), 43 deletions(-) create mode 100644 tests/pytest/insert/insert_locking.py create mode 100644 tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.csv create mode 100644 tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json create mode 100644 tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 137a7be7c7..cd56153bc4 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -961,6 +961,10 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC break; } + if (sToken.n == 0 || sToken.type == TK_SEMI || index == 0) { + return tscSQLSyntaxErrMsg(pCmd->payload, "unexpected token", sql); + } + sql += index; ++numOfColsAfterTags; } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 9f300102df..e5945f4eac 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -3216,13 +3216,6 @@ static int readTagFromCsvFileToMem(SSuperTable * superTblInfo) { return 0; } -#if 0 -int readSampleFromJsonFileToMem(SSuperTable * superTblInfo) { - // TODO - return 0; -} -#endif - /* Read 10000 lines at most. If more than 10000 lines, continue to read after using */ @@ -5338,7 +5331,7 @@ static int64_t generateInterlaceDataWithoutStb( #if STMT_IFACE_ENABLED == 1 static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, - char *dataType, int32_t dataLen, char **ptr) + char *dataType, int32_t dataLen, char **ptr, char *value) { if (0 == strncasecmp(dataType, "BINARY", strlen("BINARY"))) { @@ -5348,12 +5341,18 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, return -1; } char *bind_binary = (char *)*ptr; - rand_string(bind_binary, dataLen); bind->buffer_type = TSDB_DATA_TYPE_BINARY; - bind->buffer_length = dataLen; - bind->buffer = bind_binary; + if (value) { + strncpy(bind_binary, value, strlen(value)); + bind->buffer_length = strlen(bind_binary); + } else { + rand_string(bind_binary, dataLen); + bind->buffer_length = dataLen; + } + bind->length = &bind->buffer_length; + bind->buffer = bind_binary; bind->is_null = NULL; *ptr += bind->buffer_length; @@ -5365,9 +5364,14 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, return -1; } char *bind_nchar = (char *)*ptr; - rand_string(bind_nchar, dataLen); bind->buffer_type = TSDB_DATA_TYPE_NCHAR; + if (value) { + strncpy(bind_nchar, value, strlen(value)); + } else { + rand_string(bind_nchar, dataLen); + } + bind->buffer_length = strlen(bind_nchar); bind->buffer = bind_nchar; bind->length = &bind->buffer_length; @@ -5378,7 +5382,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "INT", strlen("INT"))) { int32_t *bind_int = (int32_t *)*ptr; - *bind_int = rand_int(); + if (value) { + *bind_int = atoi(value); + } else { + *bind_int = rand_int(); + } bind->buffer_type = TSDB_DATA_TYPE_INT; bind->buffer_length = sizeof(int32_t); bind->buffer = bind_int; @@ -5390,7 +5398,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "BIGINT", strlen("BIGINT"))) { int64_t *bind_bigint = (int64_t *)*ptr; - *bind_bigint = rand_bigint(); + if (value) { + *bind_bigint = atoll(value); + } else { + *bind_bigint = rand_bigint(); + } bind->buffer_type = TSDB_DATA_TYPE_BIGINT; bind->buffer_length = sizeof(int64_t); bind->buffer = bind_bigint; @@ -5402,7 +5414,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "FLOAT", strlen("FLOAT"))) { float *bind_float = (float *) *ptr; - *bind_float = rand_float(); + if (value) { + *bind_float = (float)atof(value); + } else { + *bind_float = rand_float(); + } bind->buffer_type = TSDB_DATA_TYPE_FLOAT; bind->buffer_length = sizeof(float); bind->buffer = bind_float; @@ -5414,7 +5430,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "DOUBLE", strlen("DOUBLE"))) { double *bind_double = (double *)*ptr; - *bind_double = rand_double(); + if (value) { + *bind_double = atof(value); + } else { + *bind_double = rand_double(); + } bind->buffer_type = TSDB_DATA_TYPE_DOUBLE; bind->buffer_length = sizeof(double); bind->buffer = bind_double; @@ -5426,7 +5446,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "SMALLINT", strlen("SMALLINT"))) { int16_t *bind_smallint = (int16_t *)*ptr; - *bind_smallint = rand_smallint(); + if (value) { + *bind_smallint = (int16_t)atoi(value); + } else { + *bind_smallint = rand_smallint(); + } bind->buffer_type = TSDB_DATA_TYPE_SMALLINT; bind->buffer_length = sizeof(int16_t); bind->buffer = bind_smallint; @@ -5438,7 +5462,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "TINYINT", strlen("TINYINT"))) { int8_t *bind_tinyint = (int8_t *)*ptr; - *bind_tinyint = rand_tinyint(); + if (value) { + *bind_tinyint = (int8_t)atoi(value); + } else { + *bind_tinyint = rand_tinyint(); + } bind->buffer_type = TSDB_DATA_TYPE_TINYINT; bind->buffer_length = sizeof(int8_t); bind->buffer = bind_tinyint; @@ -5461,7 +5489,11 @@ static int32_t prepareStmtBindArrayByType(TAOS_BIND *bind, "TIMESTAMP", strlen("TIMESTAMP"))) { int64_t *bind_ts2 = (int64_t *) *ptr; - *bind_ts2 = rand_bigint(); + if (value) { + *bind_ts2 = atoll(value); + } else { + *bind_ts2 = rand_bigint(); + } bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; bind->buffer_length = sizeof(int64_t); bind->buffer = bind_ts2; @@ -5527,12 +5559,13 @@ static int32_t prepareStmtWithoutStb( ptr += bind->buffer_length; for (int i = 0; i < g_args.num_of_CPR; i ++) { - bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * (i + 1))); + bind = (TAOS_BIND *)((char *)bindArray + + (sizeof(TAOS_BIND) * (i + 1))); if ( -1 == prepareStmtBindArrayByType( bind, data_type[i], g_args.len_of_binary, - &ptr)) { + &ptr, NULL)) { return -1; } } @@ -5551,12 +5584,14 @@ static int32_t prepareStmtWithoutStb( return k; } -static int32_t prepareStbStmt(SSuperTable *stbInfo, +static int32_t prepareStbStmt( + SSuperTable *stbInfo, TAOS_STMT *stmt, char *tableName, uint32_t batch, uint64_t insertRows, uint64_t recordFrom, - int64_t startTime, char *buffer) + int64_t startTime, + int64_t *pSamplePos) { int ret = taos_stmt_set_tbname(stmt, tableName); if (ret != 0) { @@ -5567,16 +5602,24 @@ static int32_t prepareStbStmt(SSuperTable *stbInfo, char *bindArray = malloc(sizeof(TAOS_BIND) * (stbInfo->columnCount + 1)); if (bindArray == NULL) { - errorPrint("Failed to allocate %d bind params\n", - (stbInfo->columnCount + 1)); + errorPrint("%s() LN%d, Failed to allocate %d bind params\n", + __func__, __LINE__, (stbInfo->columnCount + 1)); return -1; } - bool tsRand; + bool sourceRand; if (0 == strncasecmp(stbInfo->dataSource, "rand", strlen("rand"))) { - tsRand = true; + sourceRand = true; } else { - tsRand = false; + sourceRand = false; // from sample data file + } + + char *bindBuffer = malloc(g_args.len_of_binary); + if (bindBuffer == NULL) { + errorPrint("%s() LN%d, Failed to allocate %d bind buffer\n", + __func__, __LINE__, g_args.len_of_binary); + free(bindArray); + return -1; } uint32_t k; @@ -5592,7 +5635,7 @@ static int32_t prepareStbStmt(SSuperTable *stbInfo, bind_ts = (int64_t *)ptr; bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - if (tsRand) { + if (sourceRand) { *bind_ts = startTime + getTSRandTail( stbInfo->timeStampStep, k, stbInfo->disorderRatio, @@ -5607,14 +5650,46 @@ static int32_t prepareStbStmt(SSuperTable *stbInfo, ptr += bind->buffer_length; + int cursor = 0; for (int i = 0; i < stbInfo->columnCount; i ++) { bind = (TAOS_BIND *)((char *)bindArray + (sizeof(TAOS_BIND) * (i + 1))); - if ( -1 == prepareStmtBindArrayByType( - bind, - stbInfo->columns[i].dataType, - stbInfo->columns[i].dataLen, - &ptr)) { - return -1; + + if (sourceRand) { + if ( -1 == prepareStmtBindArrayByType( + bind, + stbInfo->columns[i].dataType, + stbInfo->columns[i].dataLen, + &ptr, + NULL)) { + free(bindArray); + free(bindBuffer); + return -1; + } + } else { + char *restStr = stbInfo->sampleDataBuf + cursor; + int lengthOfRest = strlen(restStr); + + int index = 0; + for (index = 0; index < lengthOfRest; index ++) { + if (restStr[index] == ',') { + break; + } + } + + memset(bindBuffer, 0, g_args.len_of_binary); + strncpy(bindBuffer, restStr, index); + cursor += index + 1; // skip ',' too + + if ( -1 == prepareStmtBindArrayByType( + bind, + stbInfo->columns[i].dataType, + stbInfo->columns[i].dataLen, + &ptr, + bindBuffer)) { + free(bindArray); + free(bindBuffer); + return -1; + } } } taos_stmt_bind_param(stmt, (TAOS_BIND *)bindArray); @@ -5623,11 +5698,16 @@ static int32_t prepareStbStmt(SSuperTable *stbInfo, k++; recordFrom ++; + + if (!sourceRand) { + (*pSamplePos) ++; + } if (recordFrom >= insertRows) { break; } } + free(bindBuffer); free(bindArray); return k; } @@ -5820,13 +5900,14 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (superTblInfo) { if (superTblInfo->iface == STMT_IFACE) { #if STMT_IFACE_ENABLED == 1 - generated = prepareStbStmt(superTblInfo, + generated = prepareStbStmt( + superTblInfo, pThreadInfo->stmt, tableName, batchPerTbl, insertRows, i, startTime, - pThreadInfo->buffer); + &(pThreadInfo->samplePos)); #else generated = -1; #endif @@ -6051,7 +6132,8 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pThreadInfo->stmt, tableName, g_args.num_of_RPR, - insertRows, i, start_time, pstr); + insertRows, i, start_time, + &(pThreadInfo->samplePos)); #else generated = -1; #endif @@ -7332,6 +7414,7 @@ static void *superSubscribe(void *sarg) { TAOS_RES* res = NULL; uint64_t st = 0, et = 0; + while ((g_queryInfo.superQueryInfo.endAfterConsume == -1) || (g_queryInfo.superQueryInfo.endAfterConsume > consumed[pThreadInfo->end_table_to diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c index 4197428fec..2bdfd2ead3 100644 --- a/src/vnode/src/vnodeSync.c +++ b/src/vnode/src/vnodeSync.c @@ -95,7 +95,7 @@ void vnodeCtrlFlow(int32_t vgId, int32_t level) { } void vnodeStartSyncFile(int32_t vgId) { - SVnodeObj *pVnode = vnodeAcquire(vgId); + SVnodeObj *pVnode = vnodeAcquireNotClose(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while start filesync", vgId); return; @@ -155,7 +155,7 @@ int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rpara } int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) { - SVnodeObj *pVnode = vnodeAcquire(vgId); + SVnodeObj *pVnode = vnodeAcquireNotClose(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while write to cache", vgId); return -1; diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 2cbc3747f6..c9a91e1688 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -345,12 +345,14 @@ python3 ./test.py -f tag_lite/unsignedTinyint.py python3 ./test.py -f functions/function_percentile2.py python3 ./test.py -f insert/boundary2.py +python3 ./test.py -f insert/insert_locking.py python3 ./test.py -f alter/alter_debugFlag.py python3 ./test.py -f query/queryBetweenAnd.py python3 ./test.py -f tag_lite/alter_tag.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py +python3 test.py -f tools/taosdemoAllTest/TD-4985/query-limit-offset.py python3 ./test.py -f tag_lite/drop_auto_create.py python3 test.py -f insert/insert_before_use_db.py python3 test.py -f alter/alter_keep.py diff --git a/tests/pytest/insert/insert_locking.py b/tests/pytest/insert/insert_locking.py new file mode 100644 index 0000000000..0d780a7132 --- /dev/null +++ b/tests/pytest/insert/insert_locking.py @@ -0,0 +1,178 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +import random + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + + def run(self): + tdSql.prepare() + # test case for https://jira.taosdata.com:18080/browse/TD-5021 + + tdLog.info("\n\n----------step1 : drop db and create db----------\n") + tdSql.execute('''drop database if exists db ;''') + tdSql.execute('''create database db ;''') + sql = '''show databases;''' + tdSql.query(sql) + tdSql.checkRows(1) + + tdLog.info("\n\n----------step2 : create stable----------\n") + tdSql.execute('''create stable + db.stable_1 (ts timestamp, payload binary(256)) + tags(t1 binary(16),t2 int);''') + sql = '''show db.stables;''' + tdSql.query(sql) + tdSql.checkRows(1) + + tdLog.info("\n\n----------step3 : create table and insert----------\n") + sql = '''insert into db.table1 using db.stable_1 (t1 , t2) tags ("table_1" , 111) ( values (now, ;''' + tdLog.info(sql) + tdSql.error(sql) + try: + tdSql.execute(sql) + tdLog.exit(" unexpected token") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("DB error: syntax error near ', ;' (unexpected token)") + + sql = '''insert into db.table1(ts , payload) using db.stable_1 (t1 , t2) tags ("table_1" , 111) ( values (now, ;''' + tdLog.info(sql) + tdSql.error(sql) + try: + tdSql.execute(sql) + tdLog.exit(" bind columns again") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("DB error: syntax error near ', ;' (bind columns again)") + + sql = '''insert into db.table1 using db.stable_1 (t1 , t2) tags ("table_1",111) (ts , payload) ( values (now, ;''' + tdLog.info(sql) + tdSql.error(sql) + try: + tdSql.execute(sql) + tdLog.exit(" keyword VALUES or FILE required ") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("DB error: invalid SQL: (keyword VALUES or FILE required)") + + tdSql.execute('''insert into db.table1 using db.stable_1 (t1 , t2) + tags ("table_1" , 111) values ( now , 1) ''') + sql = '''select * from db.stable_1;''' + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0,1,1) + tdSql.checkData(0,2,'table_1') + + tdLog.info("\n\n----------step4 : create table and insert again----------\n") + sql = '''insert into db.table2 using db.stable_1 (t1) tags ("table_2") ( values (now, ;''' + tdLog.info(sql) + tdSql.error(sql) + try: + tdSql.execute(sql) + tdLog.exit(" unexpected token") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("DB error: syntax error near ', ;' (unexpected token)") + + tdSql.execute('''insert into db.table2 using db.stable_1 (t1) + tags ("table_2") values ( now , 2) ''') + sql = '''select * from db.stable_1;''' + tdSql.query(sql) + tdSql.checkRows(2) + tdSql.checkData(1,1,2) + tdSql.checkData(1,2,'table_2') + + tdLog.info("\n\n----------step5 : create table and insert without db----------\n") + tdSql.execute('''use db''') + sql = '''insert into table3 using stable_1 (t1) tags ("table_3") ( values (now, ;''' + tdLog.info(sql) + tdSql.error(sql) + try: + tdSql.execute(sql) + tdLog.exit(" unexpected token") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("DB error: syntax error near ', ;' (unexpected token)") + + tdSql.execute('''insert into table3 using stable_1 (t1 , t2) + tags ("table_3" , 333) values ( now , 3) ''') + sql = '''select * from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(3) + tdSql.checkData(2,1,3) + tdSql.checkData(2,2,'table_3') + + tdLog.info("\n\n----------step6 : create tables in one sql ----------\n") + sql = '''insert into table4 using stable_1 (t1) tags ("table_4") values (now, 4) + table5 using stable_1 (t1) tags ("table_5") ( values (now, ;''' + tdLog.info(sql) + tdSql.error(sql) + try: + tdSql.execute(sql) + tdLog.exit(" unexpected token") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("DB error: syntax error near ', ;' (unexpected token)") + + tdSql.execute('''insert into table4 using stable_1 (t1) tags ("table_4") values (now, 4) + table5 using stable_1 (t1) tags ("table_5") values (now, 5) ''') + sql = '''select * from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(5) + tdSql.checkData(3,1,4) + tdSql.checkData(3,2,'table_4') + tdSql.checkData(4,1,5) + tdSql.checkData(4,2,'table_5') + + + sql = '''insert into table6 using stable_1 (t1) tags ("table_6") ( values (now, + table7 using stable_1 (t1) tags ("table_7") values (now, 7);''' + tdLog.info(sql) + tdSql.error(sql) + try: + tdSql.execute(sql) + tdLog.exit(" invalid SQL") + except Exception as e: + tdLog.info(repr(e)) + tdLog.info("invalid SQL") + + tdSql.execute('''insert into table6 using stable_1 (t1 , t2) tags ("table_6" , 666) values (now, 6) + table7 using stable_1 (t1) tags ("table_7") values (now, 7) ''') + sql = '''select * from stable_1;''' + tdSql.query(sql) + tdSql.checkRows(7) + tdSql.checkData(5,1,6) + tdSql.checkData(5,2,'table_6') + tdSql.checkData(6,1,7) + tdSql.checkData(6,2,'table_7') + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.csv b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.csv new file mode 100644 index 0000000000..d4138798e3 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.csv @@ -0,0 +1,10 @@ +0,0,'TAOSdata-0' +1,1,'TAOSdata-1' +2,22,'TAOSdata-2' +3,333,'TAOSdata-3' +4,4444,'TAOSdata-4' +5,55555,'TAOSdata-5' +6,666666,'TAOSdata-6' +7,7777777,'TAOSdata-7' +8,88888888,'TAOSdata-8' +9,999999999,'TAOSdata-9' \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json new file mode 100644 index 0000000000..265f42036b --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json @@ -0,0 +1,62 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 10000, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 1, + "data_source": "sample", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": 0, + "childtable_offset":0, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./tools/taosdemoAllTest/TD-4985/query-limit-offset.csv", + "tags_file": "./tools/taosdemoAllTest/TD-4985/query-limit-offset.csv", + "columns": [{"type": "INT","count":2}, {"type": "BINARY", "len": 16, "count":1}], + "tags": [{"type": "INT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} diff --git a/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py new file mode 100644 index 0000000000..081057f180 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py @@ -0,0 +1,191 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + + # insert: create one or mutiple tables per sql and insert multiple rows per sql + # test case for https://jira.taosdata.com:18080/browse/TD-4985 + os.system("%staosdemo -f tools/taosdemoAllTest/TD-4985/query-limit-offset.json -y " % binPath) + tdSql.execute("use db") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 10000) + + for i in range(1000): + tdSql.execute('''insert into stb00_9999 values(%d, %d, %d,'test99.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_8888 values(%d, %d, %d,'test98.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_7777 values(%d, %d, %d,'test97.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_6666 values(%d, %d, %d,'test96.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_5555 values(%d, %d, %d,'test95.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_4444 values(%d, %d, %d,'test94.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_3333 values(%d, %d, %d,'test93.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_2222 values(%d, %d, %d,'test92.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_1111 values(%d, %d, %d,'test91.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.execute('''insert into stb00_100 values(%d, %d, %d,'test90.%s')''' + % (1600000000000 + i, i, -10000+i, i)) + tdSql.query("select * from stb0 where col2 like 'test99%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_9999' limit 10" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_9999' limit 10 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test98%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_8888' limit 10" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_8888' limit 10 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test97%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_7777' limit 10" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_7777' limit 10 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test96%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_6666' limit 10" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_6666' limit 10 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test95%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_5555' limit 10" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_5555' limit 10 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test94%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_4444' limit 10" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_4444' limit 10 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test93%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_3333' limit 100" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_3333' limit 100 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test92%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_2222' limit 100" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_2222' limit 100 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test91%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_1111' limit 100" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_1111' limit 100 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + tdSql.query("select * from stb0 where col2 like 'test90%' ") + tdSql.checkRows(1000) + tdSql.query("select * from stb0 where tbname like 'stb00_100' limit 100" ) + tdSql.checkData(0, 1, 0) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 1, 2) + tdSql.query("select * from stb0 where tbname like 'stb00_100' limit 100 offset 5" ) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 6) + tdSql.checkData(2, 1, 7) + + + os.system("rm -rf tools/taosdemoAllTest/TD-4985/query-limit-offset.py.sql") + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index c4b125969a..90d2c52c15 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -16,6 +16,8 @@ import pandas as pd import argparse import os.path import json +from util.log import tdLog +from util.sql import tdSql class taosdemoPerformace: diff --git a/tests/pytest/tools/taosdumpTest.py b/tests/pytest/tools/taosdumpTest.py index beac22a2af..b2c9eb3ec1 100644 --- a/tests/pytest/tools/taosdumpTest.py +++ b/tests/pytest/tools/taosdumpTest.py @@ -63,7 +63,6 @@ class TDTestCase: tdSql.execute("create database db days 11 keep 3649 blocks 8 ") tdSql.execute("create database db1 days 12 keep 3640 blocks 7 ") tdSql.execute("use db") - tdSql.execute( "create table st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))") tdSql.execute("create table t1 using st tags(1, 'beijing')") @@ -86,9 +85,10 @@ class TDTestCase: tdLog.info("taosdump found in %s" % buildPath) binPath = buildPath + "/build/bin/" - os.system("rm ./taosdumptest/tmp1/*.sql") os.system("%staosdump --databases db -o ./taosdumptest/tmp1" % binPath) - os.system("%staosdump --databases db1 -o ./taosdumptest/tmp2" % binPath) + os.system( + "%staosdump --databases db1 -o ./taosdumptest/tmp2" % + binPath) tdSql.execute("drop database db") tdSql.execute("drop database db1") -- GitLab