diff --git a/Jenkinsfile b/Jenkinsfile index 6bff8ec1ce005d42c1a6caa21dfb010c8f445abf..e3aa1c7a29214fbbf60871ffa07164c5b2678476 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -70,7 +70,6 @@ def pre_test(){ git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD git clean -dfx - git ls-files --stage | grep 160000 | awk '{print $4}' | xargs git rm --cached git submodule update --init --recursive cd ${WK} git reset --hard HEAD~10 @@ -144,7 +143,6 @@ def pre_test_noinstall(){ git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD git clean -dfx - git ls-files --stage | grep 160000 | awk '{print $4}' | xargs git rm --cached git submodule update --init --recursive cd ${WK} git reset --hard HEAD~10 @@ -215,7 +213,6 @@ def pre_test_ningsi(){ git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD git clean -dfx - git ls-files --stage | grep 160000 | awk '{print $4}' | xargs git rm --cached git submodule update --init --recursive cd ${WK} git reset --hard HEAD~10 diff --git a/cmake/install.inc b/cmake/install.inc index 641099d146a149bcdd9fd3ab739f50b0f5a8a0e1..48d594e8898b5a278dc46fbec1e61ebe90ead905 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -24,19 +24,19 @@ ELSEIF (TD_WINDOWS) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) IF (TD_POWER) - INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) + INSTALL(FILES ${TD_COMMUNITY_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/power.exe DESTINATION .) ELSEIF (TD_TQ) - INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) + INSTALL(FILES ${TD_COMMUNITY_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/tq.exe DESTINATION .) ELSEIF (TD_PRO) - INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) + INSTALL(FILES ${TD_COMMUNITY_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/prodbc.exe DESTINATION .) ELSEIF (TD_KH) INSTALL(FILES ${TD_COMMUNITY_DIR}/packaging/cfg/kinghistorian.cfg DESTINATION cfg) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/khclient.exe DESTINATION .) ELSEIF (TD_JH) - INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) + INSTALL(FILES ${TD_COMMUNITY_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/jh_taos.exe DESTINATION .) ELSE () INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector) @@ -44,7 +44,8 @@ ELSEIF (TD_WINDOWS) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/C\# DESTINATION connector) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .) - + + INSTALL(FILES ${TD_COMMUNITY_DIR}/packaging/cfg/taos.cfg DESTINATION cfg) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosdemo.exe DESTINATION .) ENDIF () diff --git a/deps/avro b/deps/avro deleted file mode 160000 index a1fce29d9675b4dd95dfee9db32cc505d0b2227c..0000000000000000000000000000000000000000 --- a/deps/avro +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a1fce29d9675b4dd95dfee9db32cc505d0b2227c diff --git a/deps/googletest b/deps/googletest deleted file mode 160000 index e2239ee6043f73722e7aa812a459f54a28552929..0000000000000000000000000000000000000000 --- a/deps/googletest +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e2239ee6043f73722e7aa812a459f54a28552929 diff --git a/deps/rocksdb b/deps/rocksdb deleted file mode 160000 index c3034fce329017036c807e01261729bfc11a5d62..0000000000000000000000000000000000000000 --- a/deps/rocksdb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c3034fce329017036c807e01261729bfc11a5d62 diff --git a/deps/zlib b/deps/zlib deleted file mode 160000 index cacf7f1d4e3d44d871b605da3b647f07d718623f..0000000000000000000000000000000000000000 --- a/deps/zlib +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cacf7f1d4e3d44d871b605da3b647f07d718623f diff --git a/packaging/deb/nginxd b/packaging/deb/nginxd new file mode 100644 index 0000000000000000000000000000000000000000..bdc50fb0464df504977249182857f06c2ec2ccbf --- /dev/null +++ b/packaging/deb/nginxd @@ -0,0 +1,46 @@ +#!/bin/bash +#Startup script for the nginx Web Server +# chkconfig: 2345 99 01 +# description: Nginx For TDengine Service. +# +# +### BEGIN INIT INFO +# Provides: nginx +# Required-Start: $local_fs $network $syslog +# Required-Stop: $local_fs $network $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts nginx +# Description: Starts nginx. +### END INIT INFO + +nginx=/usr/local/nginxd/sbin/nginx +case $1 in + start) + echo -n "Starting Nginx" + $nginx + echo " done." + ;; + stop) + echo -n "Stopping Nginx" + $nginx -s stop + echo " done." + ;; + test) + $nginx -t + echo "Success." + ;; + reload) + echo -n "Reloading Nginx" + $nginx -s reload + echo " done." + ;; + restart) + $nginx -s reload + echo "reload done." + ;; + *) + echo "Usage: $0 {start|restart|reload|stop|test|show}" + ;; +esac + diff --git a/packaging/rpm/nginxd b/packaging/rpm/nginxd new file mode 100644 index 0000000000000000000000000000000000000000..fc6593992b297f7fcad03d04051a030dc1898d5e --- /dev/null +++ b/packaging/rpm/nginxd @@ -0,0 +1,44 @@ +#!/bin/bash +#Startup script for the nginx Web Server +# chkconfig: 2345 99 01 +# +# +### BEGIN INIT INFO +# Provides: nginx +# Provides: nginx +# Required-Start: $local_fs $network $syslog +# Required-Stop: $local_fs $network $syslog +# Short-Description: Starts nginx +# Description: Starts nginx. +### END INIT INFO + +nginx=/usr/local/nginxd/sbin/nginx +case $1 in + start) + echo -n "Starting Nginx" + $nginx + echo " done." + ;; + stop) + echo -n "Stopping Nginx" + $nginx -s stop + echo " done." + ;; + test) + $nginx -t + echo "Success." + ;; + reload) + echo -n "Reloading Nginx" + $nginx -s reload + echo " done." + ;; + restart) + $nginx -s reload + echo "reload done." + ;; + *) + echo "Usage: $0 {start|restart|reload|stop|test|show}" + ;; +esac + diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 0d7a7dd9442fd635fce56792dc37c3276d1b955d..3930433b96166c8beb0bddca17aa73ab11c0ebc5 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -582,6 +582,26 @@ function clean_service_on_sysvinit() { ${csudo} rm -f ${service_config_dir}/taosd || : ${csudo} rm -f ${service_config_dir}/tarbitratord || : + if [ "$verMode" == "cluster" ]; then\ + if pidof nginxd ; then + ${csudo} service nginxd stop || : + fi + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/nginxd ]; then + ${csudo} chkconfig --del nginxd || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/nginxd ]; then + ${csudo} insserv -r nginxd || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/nginxd ]; then + ${csudo} update-rc.d -f nginxd remove || : + fi + fi + ${csudo} rm -f ${service_config_dir}/nginxd || : + fi + if $(which init &> /dev/null); then ${csudo} init q || : fi @@ -598,11 +618,19 @@ function install_service_on_sysvinit() { ${csudo} cp ${script_dir}/init.d/taosd.deb ${service_config_dir}/taosd && ${csudo} chmod a+x ${service_config_dir}/taosd ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + if [ "$verMode" == "cluster" ]; then + ${csudo} cp -f ${script_dir}/init.d/nginxd.deb ${install_main_dir}/init.d/nginxd + ${csudo} cp ${script_dir}/init.d/nginxd.deb ${service_config_dir}/nginxd && ${csudo} chmod a+x ${service_config_dir}/nginxd + fi elif ((${os_type}==2)); then ${csudo} cp -f ${script_dir}/init.d/taosd.rpm ${install_main_dir}/init.d/taosd ${csudo} cp ${script_dir}/init.d/taosd.rpm ${service_config_dir}/taosd && ${csudo} chmod a+x ${service_config_dir}/taosd ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + if [ "$verMode" == "cluster" ]; then + ${csudo} cp -f ${script_dir}/init.d/nginxd.rpm ${install_main_dir}/init.d/nginxd + ${csudo} cp ${script_dir}/init.d/nginxd.rpm ${service_config_dir}/nginxd && ${csudo} chmod a+x ${service_config_dir}/nginxd + fi fi #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" @@ -613,14 +641,26 @@ function install_service_on_sysvinit() { ${csudo} chkconfig --level 2345 taosd on || : ${csudo} chkconfig --add tarbitratord || : ${csudo} chkconfig --level 2345 tarbitratord on || : + if [ "$verMode" == "cluster" ]; then + ${csudo} chkconfig --add nginxd || : + ${csudo} chkconfig --level 0123456 nginxd on || : + ${csudo} service nginxd start + fi elif ((${initd_mod}==2)); then ${csudo} insserv taosd || : ${csudo} insserv -d taosd || : ${csudo} insserv tarbitratord || : ${csudo} insserv -d tarbitratord || : + if [ "$verMode" == "cluster" ]; then + ${csudo} insserv nginxd || : + ${csudo} insserv -d nginxd || : + fi elif ((${initd_mod}==3)); then ${csudo} update-rc.d taosd defaults || : ${csudo} update-rc.d tarbitratord defaults || : + if [ "$verMode" == "cluster" ]; then + ${csudo} update-rc.d nginxd defaults || : + fi fi } @@ -779,7 +819,7 @@ vercomp () { function is_version_compatible() { - curr_version=`ls ${script_dir}/driver/libtaos.so* |cut -d '.' -f 3-6` + curr_version=`ls ${script_dir}/driver/libtaos.so* | awk -F 'libtaos.so.' '{print $2}'` if [ -f ${script_dir}/driver/vercomp.txt ]; then min_compatible_version=`cat ${script_dir}/driver/vercomp.txt` diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 39918325992ba140db6db656c6946ec40dd84435..1b11973e3c232f3b00cd0f1d1db8c903e8046da6 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -63,6 +63,8 @@ init_file_deb=${script_dir}/../deb/taosd init_file_rpm=${script_dir}/../rpm/taosd init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord +init_file_nginx_deb=${script_dir}/../deb/nginxd +init_file_nginx_rpm=${script_dir}/../rpm/nginxd # make directories. mkdir -p ${install_dir} @@ -73,6 +75,8 @@ mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/taos mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/taosd.rpm mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : +mkdir -p ${install_dir}/init.d && cp ${init_file_nginx_deb} ${install_dir}/init.d/nginxd.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_nginx_rpm} ${install_dir}/init.d/nginxd.rpm || : if [ -f ${build_dir}/bin/jemalloc-config ]; then mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3} diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index ae279a8b8b84d54bd1084f668f5594addfbdf11c..b198070525cc1c8f80f611768583996bd8bd4314 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -358,19 +358,28 @@ static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) { // validate the out put field type for "UNION ALL" subclause static int32_t normalizeVarDataTypeLength(SSqlCmd* pCmd) { const char* msg1 = "columns in select clause not identical"; + const char* msg2 = "too many select clause siblings, at most 100 allowed"; + int32_t siblings = 0; int32_t diffSize = 0; // if there is only one element, the limit of clause is the limit of global result. SQueryInfo* pQueryInfo1 = pCmd->pQueryInfo; SQueryInfo* pSibling = pQueryInfo1->sibling; + // pQueryInfo1 itself + ++siblings; + while(pSibling != NULL) { int32_t ret = tscFieldInfoCompare(&pQueryInfo1->fieldsInfo, &pSibling->fieldsInfo, &diffSize); if (ret != 0) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); } + if (++siblings > 100) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + pSibling = pSibling->sibling; } @@ -6550,17 +6559,15 @@ int32_t validateLimitNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlN // todo refactor if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - if (!tscQueryTags(pQueryInfo)) { // local handle the super table tag query - if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { - if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); - } + if (tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { + if (pQueryInfo->slimit.limit > 0 || pQueryInfo->slimit.offset > 0) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); + } - // for projection query on super table, all queries are subqueries - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY)) { - pQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; - } + // for projection query on super table, all queries are subqueries + if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && + !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY)) { + pQueryInfo->type |= TSDB_QUERY_TYPE_SUBQUERY; } } @@ -7535,6 +7542,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { const char* msg3 = "tag value too long"; const char* msg4 = "illegal value or data overflow"; const char* msg5 = "tags number not matched"; + const char* msg6 = "create table only from super table is allowed"; SSqlCmd* pCmd = &pSql->cmd; @@ -7575,6 +7583,10 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return code; } + if (!UTIL_TABLE_IS_SUPER_TABLE(pStableMetaInfo)) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); + } + size_t valSize = taosArrayGetSize(pValList); // too long tag values will return invalid sql, not be truncated automatically @@ -8685,7 +8697,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf const char* msg4 = "interval query not supported, since the result of sub query not include valid timestamp column"; const char* msg5 = "only tag query not compatible with normal column filter"; const char* msg6 = "not support stddev/percentile/interp in the outer query yet"; - const char* msg7 = "derivative/twa/irate requires timestamp column exists in subquery"; + const char* msg7 = "derivative/twa/rate/irate/diff requires timestamp column exists in subquery"; const char* msg8 = "condition missing for join query"; const char* msg9 = "not support 3 level select"; @@ -8769,7 +8781,8 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf SExprInfo* pExpr = tscExprGet(pQueryInfo, i); int32_t f = pExpr->base.functionId; - if (f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE) { + if (f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE || + f == TSDB_FUNC_RATE || f == TSDB_FUNC_DIFF) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 4d4667e135d04b28a3b94952f1c165b5dd16a784..dd0975de78dfdc231908130c6a2193d0cbc9f2bf 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -980,7 +980,11 @@ static int32_t serializeSqlExpr(SSqlExpr* pExpr, STableMetaInfo* pTableMetaInfo, return TSDB_CODE_TSC_INVALID_OPERATION; } - assert(pExpr->resColId < 0); + if (pExpr->resColId >= 0) { + tscError("result column id underflowed: %d", pExpr->resColId); + return TSDB_CODE_TSC_RES_TOO_MANY; + } + SSqlExpr* pSqlExpr = (SSqlExpr *)(*pMsg); SColIndex* pIndex = &pSqlExpr->colInfo; @@ -3142,15 +3146,19 @@ int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { SSqlCmd* pCmd2 = &pSql->rootObj->cmd; pCmd2->pTableMetaMap = tscCleanupTableMetaMap(pCmd2->pTableMetaMap); pCmd2->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - + pSql->rootObj->retryReason = pSql->retryReason; + SSqlObj *tmpSql = pSql->rootObj; + tscFreeSubobj(pSql->rootObj); + tfree(tmpSql->pSubs); + SArray* pNameList = taosArrayInit(1, POINTER_BYTES); SArray* vgroupList = taosArrayInit(1, POINTER_BYTES); char* n = strdup(name); taosArrayPush(pNameList, &n); - code = getMultiTableMetaFromMnode(pSql, pNameList, vgroupList, NULL, tscTableMetaCallBack, true); + code = getMultiTableMetaFromMnode(tmpSql, pNameList, vgroupList, NULL, tscTableMetaCallBack, true); taosArrayDestroyEx(pNameList, freeElem); taosArrayDestroyEx(vgroupList, freeElem); diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index c18597c7db21ed3f36553d5ba531347afbb639ee..d93c0ce89582295557d90128e6484f6a5fe9ac2e 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -448,13 +448,14 @@ typedef struct { #define kvRowSetNCols(r, n) kvRowNCols(r) = (n) #define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE) #define kvRowValues(r) POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * kvRowNCols(r)) +#define kvRowKeys(r) POINTER_SHIFT(r, *(uint16_t *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(int16_t))) #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) #define kvRowFree(r) tfree(r) #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) #define kvRowValLen(r) (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r)) -#define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) +#define kvRowTKey(r) (*(TKEY *)(kvRowKeys(r))) #define kvRowKey(r) tdGetKey(kvRowTKey(r)) #define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r)) @@ -652,7 +653,7 @@ static FORCE_INLINE char *memRowEnd(SMemRow row) { #define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) #define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r)) // schema version #define memRowSetKvVersion(r, v) (memRowKvVersion(r) = (v)) -#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r))) +#define memRowKeys(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowKeys(memRowKvBody(r))) #define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r))) #define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r))) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 9775a3898fb759873ffe48673062dd8402873041..ffa89b8c39bea339e7916cc027232e88bfbaa44b 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -223,6 +223,8 @@ extern uint32_t curRange; extern char Compressor[]; #endif +// long query +extern int8_t tsDeadLockKillQuery; typedef struct { char dir[TSDB_FILENAME_LEN]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index cd8d6e21d1ac52579a6e23bc953d5efdd5696a1f..50d3fcd1a914bcb6af2629334a3f94daf7d26101 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -271,6 +271,9 @@ uint32_t curRange = 100; // range char Compressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR #endif +// long query death-lock +int8_t tsDeadLockKillQuery = 1; + int32_t (*monStartSystemFp)() = NULL; void (*monStopSystemFp)() = NULL; void (*monExecuteSQLFp)(char *sql) = NULL; @@ -1606,6 +1609,17 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + // enable kill long query + cfg.option = "deadLockKillQuery"; + cfg.ptr = &tsDeadLockKillQuery; + cfg.valType = TAOS_CFG_VTYPE_INT8; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 1; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + #ifdef TD_TSZ // lossy compress cfg.option = "lossyColumns"; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index 21c76f73b287e55ef14f5d70cf6a911a9cb543db..eba92741dd8bb3c20f114e35c9a8190193139988 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -66,7 +66,11 @@ public class RestfulStatement extends AbstractStatement { boolean result = true; if (SqlSyntaxValidator.isUseSql(sql)) { - HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken()); + String ret = HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken()); + JSONObject resultJson = JSON.parseObject(ret); + if (resultJson.getString("status").equals("error")) { + throw TSDBError.createSQLException(resultJson.getInteger("code"), "sql: " + sql + ", desc: " + resultJson.getString("desc")); + } this.database = sql.trim().replace("use", "").trim(); this.conn.setCatalog(this.database); result = false; @@ -115,7 +119,7 @@ public class RestfulStatement extends AbstractStatement { String result = HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken()); JSONObject resultJson = JSON.parseObject(result); if (resultJson.getString("status").equals("error")) { - throw TSDBError.createSQLException(resultJson.getInteger("code"), resultJson.getString("desc")); + throw TSDBError.createSQLException(resultJson.getInteger("code"), "sql: " + sql + ", desc: " + resultJson.getString("desc")); } this.resultSet = new RestfulResultSet(database, this, resultJson); this.affectedRows = 0; @@ -126,7 +130,7 @@ public class RestfulStatement extends AbstractStatement { String result = HttpClientPoolUtil.execute(getUrl(), sql, this.conn.getToken()); JSONObject jsonObject = JSON.parseObject(result); if (jsonObject.getString("status").equals("error")) { - throw TSDBError.createSQLException(jsonObject.getInteger("code"), jsonObject.getString("desc")); + throw TSDBError.createSQLException(jsonObject.getInteger("code"), "sql: " + sql + ", desc: " + jsonObject.getString("desc")); } this.resultSet = null; this.affectedRows = getAffectedRows(jsonObject); @@ -134,16 +138,13 @@ public class RestfulStatement extends AbstractStatement { } private int getAffectedRows(JSONObject jsonObject) throws SQLException { - // create ... SQLs should return 0 , and Restful result like this: - // {"status": "succ", "head": ["affected_rows"], "data": [[0]], "rows": 1} JSONArray head = jsonObject.getJSONArray("head"); if (head.size() != 1 || !"affected_rows".equals(head.getString(0))) - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE, "invalid variable: [" + head.toJSONString() + "]"); JSONArray data = jsonObject.getJSONArray("data"); if (data != null) return data.getJSONArray(0).getInteger(0); - - throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE, "invalid variable: [" + jsonObject.toJSONString() + "]"); } @Override diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java index 0ea46dade29316b99447a6ea4e372bc8057670e8..5ff4b1a41232f57eb21d8394d62cc2dcd27c9cf3 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AuthenticationTest.java @@ -57,53 +57,33 @@ public class AuthenticationTest { @Ignore @Test - public void test() { + public void test() throws SQLException { // change password - try { - conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=" + user + "&password=taosdata"); - Statement stmt = conn.createStatement(); - stmt.execute("alter user " + user + " pass '" + password + "'"); - stmt.close(); - conn.close(); - } catch (SQLException e) { - e.printStackTrace(); - } + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=" + user + "&password=taosdata"); + Statement stmt = conn.createStatement(); + stmt.execute("alter user " + user + " pass '" + password + "'"); + stmt.close(); + conn.close(); + // use new to login and execute query - try { - conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=" + user + "&password=" + password); - Statement stmt = conn.createStatement(); - stmt.execute("show databases"); - ResultSet rs = stmt.getResultSet(); - ResultSetMetaData meta = rs.getMetaData(); - while (rs.next()) { - for (int i = 1; i <= meta.getColumnCount(); i++) { - System.out.print(meta.getColumnLabel(i) + ":" + rs.getString(i) + "\t"); - } - System.out.println(); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=" + user + "&password=" + password); + stmt = conn.createStatement(); + stmt.execute("show databases"); + ResultSet rs = stmt.getResultSet(); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ":" + rs.getString(i) + "\t"); } - } catch (SQLException e) { - e.printStackTrace(); + System.out.println(); } - // change password back - try { - conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=" + user + "&password=" + password); - Statement stmt = conn.createStatement(); - stmt.execute("alter user " + user + " pass 'taosdata'"); - stmt.close(); - conn.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - - } - @Before - public void before() { - try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } + // change password back + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=" + user + "&password=" + password); + stmt = conn.createStatement(); + stmt.execute("alter user " + user + " pass 'taosdata'"); + stmt.close(); + conn.close(); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java index d8d8f1eae42f555b2bc0be12e8a3a943071da3c6..5f821c5cc34dde0050d8e62afb6fc8ab17534a17 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java @@ -1,7 +1,5 @@ package com.taosdata.jdbc.cases; -import com.taosdata.jdbc.TSDBErrorNumbers; -import org.junit.Assert; import org.junit.Test; import java.sql.DriverManager; @@ -9,16 +7,14 @@ import java.sql.SQLException; public class ConnectWrongDatabaseTest { - @Test - public void connect() { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - DriverManager.getConnection("jdbc:TAOS://localhost:6030/wrong_db?user=root&password=taosdata"); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } catch (SQLException e) { - Assert.assertEquals(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL, e.getErrorCode()); - } + @Test(expected = SQLException.class) + public void connectByJni() throws SQLException { + DriverManager.getConnection("jdbc:TAOS://localhost:6030/wrong_db?user=root&password=taosdata"); + } + + @Test(expected = SQLException.class) + public void connectByRestful() throws SQLException { + DriverManager.getConnection("jdbc:TAOS-RS://localhost:6041/wrong_db?user=root&password=taosdata"); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java index beea990456ec98c2ab51fc2086034e0b31b570b6..05c7b0feca21f3f5b9062f9cbc26921aa607732a 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java @@ -18,9 +18,8 @@ public class InsertDbwithoutUseDbTest { private static final Random random = new Random(System.currentTimeMillis()); @Test - public void case001() throws ClassNotFoundException, SQLException { + public void case001() throws SQLException { // prepare schema - Class.forName("com.taosdata.jdbc.TSDBDriver"); String url = "jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata"; Connection conn = DriverManager.getConnection(url, properties); try (Statement stmt = conn.createStatement()) { @@ -51,9 +50,8 @@ public class InsertDbwithoutUseDbTest { } @Test - public void case002() throws ClassNotFoundException, SQLException { + public void case002() throws SQLException { // prepare the schema - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); final String url = "jdbc:TAOS-RS://" + host + ":6041/inWithoutDb?user=root&password=taosdata"; Connection conn = DriverManager.getConnection(url, properties); try (Statement stmt = conn.createStatement()) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java index 1c5c03aacb5e7ed5683c75414975224a67d49e21..d50b56b434fba5795750ec2d92af9f73ae0a02d3 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java @@ -374,22 +374,17 @@ public class RestfulConnectionTest { } @BeforeClass - public static void beforeClass() { - try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/log?user=root&password=taosdata", properties); - // create test database for test cases - try (Statement stmt = conn.createStatement()) { - stmt.execute("create database if not exists test"); - } - - } catch (ClassNotFoundException | SQLException e) { - e.printStackTrace(); + public static void beforeClass() throws SQLException { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata", properties); + // create test database for test cases + try (Statement stmt = conn.createStatement()) { + stmt.execute("create database if not exists test"); } + } @AfterClass diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java index b07dae8003d6e2fea073c0d240f59fb6db0c593f..4f727d43a01ce3c4c459fcd2d7328e1c4b132976 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java @@ -10,11 +10,11 @@ import java.util.Random; public class RestfulJDBCTest { private static final String host = "127.0.0.1"; - private final Random random = new Random(System.currentTimeMillis()); - private Connection connection; + private static final Random random = new Random(System.currentTimeMillis()); + private static Connection connection; @Test - public void testCase001() { + public void testCase001() throws SQLException { // given String sql = "drop database if exists restful_test"; // when @@ -38,7 +38,7 @@ public class RestfulJDBCTest { } @Test - public void testCase002() { + public void testCase002() throws SQLException { // given String sql = "create table weather(ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)"; // when @@ -48,7 +48,7 @@ public class RestfulJDBCTest { } @Test - public void testCase004() { + public void testCase004() throws SQLException { for (int i = 1; i <= 100; i++) { // given String sql = "create table t" + i + " using weather tags('beijing', '" + i + "')"; @@ -60,7 +60,7 @@ public class RestfulJDBCTest { } @Test - public void testCase005() { + public void testCase005() throws SQLException { int rows = 0; for (int i = 0; i < 10; i++) { for (int j = 1; j <= 100; j++) { @@ -99,7 +99,7 @@ public class RestfulJDBCTest { } @Test - public void testCase007() { + public void testCase007() throws SQLException { // given String sql = "drop database restful_test"; @@ -110,50 +110,41 @@ public class RestfulJDBCTest { Assert.assertFalse(execute); } - private int executeUpdate(Connection connection, String sql) { + private int executeUpdate(Connection connection, String sql) throws SQLException { try (Statement stmt = connection.createStatement()) { return stmt.executeUpdate(sql); - } catch (SQLException e) { - e.printStackTrace(); } - return 0; } - private boolean execute(Connection connection, String sql) { + private boolean execute(Connection connection, String sql) throws SQLException { try (Statement stmt = connection.createStatement()) { return stmt.execute(sql); - } catch (SQLException e) { - e.printStackTrace(); } - return false; } - private ResultSet executeQuery(Connection connection, String sql) { + private ResultSet executeQuery(Connection connection, String sql) throws SQLException { try (Statement statement = connection.createStatement()) { return statement.executeQuery(sql); - } catch (SQLException e) { - e.printStackTrace(); } - return null; } - @Before - public void before() { + @BeforeClass + public static void beforeClass() { try { - connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); + connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); } catch (SQLException e) { e.printStackTrace(); } } - @After - public void after() { - try { - if (connection != null) - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); + @AfterClass + public static void afterClass() throws SQLException { + if (connection != null) { + Statement stmt = connection.createStatement(); + stmt.execute("drop database if exists restful_test"); + stmt.close(); + connection.close(); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java index c7fc81297264f3cf38795d9d5a3b7eccc51574c9..f3011af799c987ed399920875ae512fd8533ec77 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java @@ -186,22 +186,17 @@ public class RestfulResultSetMetaDataTest { } @BeforeClass - public static void beforeClass() { - try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); - stmt = conn.createStatement(); - stmt.execute("create database if not exists restful_test"); - stmt.execute("use restful_test"); - stmt.execute("drop table if exists weather"); - stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); - stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); - rs = stmt.executeQuery("select * from restful_test.weather"); - rs.next(); - meta = rs.getMetaData(); - } catch (ClassNotFoundException | SQLException e) { - e.printStackTrace(); - } + public static void beforeClass() throws SQLException { + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + stmt.execute("drop table if exists weather"); + stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); + stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); + rs = stmt.executeQuery("select * from restful_test.weather"); + rs.next(); + meta = rs.getMetaData(); } @AfterClass diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java index 21a91669b270df4dc2e8f7b4885fb9e8eedbfdf7..8ffa1de21c604c2235c23e63e93b008f5a1d146c 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java @@ -658,36 +658,29 @@ public class RestfulResultSetTest { } @BeforeClass - public static void beforeClass() { - try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); - stmt = conn.createStatement(); - stmt.execute("create database if not exists restful_test"); - stmt.execute("use restful_test"); - stmt.execute("drop table if exists weather"); - stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); - stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); - rs = stmt.executeQuery("select * from restful_test.weather"); - rs.next(); - } catch (ClassNotFoundException | SQLException e) { - e.printStackTrace(); - } - + public static void beforeClass() throws SQLException { + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("drop database if exists restful_test"); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + stmt.execute("drop table if exists weather"); + stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); + stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); + rs = stmt.executeQuery("select * from restful_test.weather"); + rs.next(); } @AfterClass - public static void afterClass() { - try { - if (rs != null) - rs.close(); - if (stmt != null) - stmt.close(); - if (conn != null) - conn.close(); - } catch (SQLException e) { - e.printStackTrace(); + public static void afterClass() throws SQLException { + if (rs != null) + rs.close(); + if (stmt != null) { + stmt.execute("drop database if exists restful_test"); + stmt.close(); } + if (conn != null) + conn.close(); } } \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java index 98482ade80656f2e48bc6927953439cfe4b010c1..7dff90a77556f029049a4216e789dc64e42a2e96 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java @@ -581,11 +581,14 @@ public class SQLTest { @BeforeClass public static void before() throws SQLException { - connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); + connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); } @AfterClass public static void after() throws SQLException { + Statement stmt = connection.createStatement(); + stmt.execute("drop database if exists restful_test"); + stmt.close(); connection.close(); } diff --git a/src/connector/python/taos/cinterface.py b/src/connector/python/taos/cinterface.py index a1b6fe312b5725b8bf030701608d93c3e0c85706..a9bdba33e8212b7a32ef2c6b518aef03aed3f50a 100644 --- a/src/connector/python/taos/cinterface.py +++ b/src/connector/python/taos/cinterface.py @@ -103,7 +103,7 @@ _libtaos.taos_get_client_info.restype = c_char_p def taos_get_client_info(): # type: () -> str """Get client version info.""" - return _libtaos.taos_get_client_info().decode() + return _libtaos.taos_get_client_info().decode("utf-8") _libtaos.taos_get_server_info.restype = c_char_p @@ -113,7 +113,7 @@ _libtaos.taos_get_server_info.argtypes = (c_void_p,) def taos_get_server_info(connection): # type: (c_void_p) -> str """Get server version as string.""" - return _libtaos.taos_get_server_info(connection).decode() + return _libtaos.taos_get_server_info(connection).decode("utf-8") _libtaos.taos_close.restype = None diff --git a/src/connector/python/taos/field.py b/src/connector/python/taos/field.py index b0bec58b932f2136b868739bb28fca04de759e3f..f6fa28e8336fa53a137ce7bd9a25b99a32263f1b 100644 --- a/src/connector/python/taos/field.py +++ b/src/connector/python/taos/field.py @@ -144,7 +144,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, precision=FieldType.C_ try: if num_of_rows >= 0: tmpstr = ctypes.c_char_p(data) - res.append(tmpstr.value.decode()) + res.append(tmpstr.value.decode("utf-8")) else: res.append( ( @@ -172,7 +172,7 @@ def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, precision=Field if rbyte == 1 and buffer[0] == b'\xff': res.append(None) else: - res.append(cast(buffer, c_char_p).value.decode()) + res.append(cast(buffer, c_char_p).value.decode("utf-8")) return res @@ -188,7 +188,7 @@ def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, precision=FieldT if rbyte == 4 and buffer[:4] == b'\xff'*4: res.append(None) else: - res.append(cast(buffer, c_char_p).value.decode()) + res.append(cast(buffer, c_char_p).value.decode("utf-8")) return res diff --git a/src/connector/python/taos/result.py b/src/connector/python/taos/result.py index 81151733615d1b7fdc3318b6e53888ae39d32b14..8881c26ca94edfd4ed3a5042bebbc327f06b82ac 100644 --- a/src/connector/python/taos/result.py +++ b/src/connector/python/taos/result.py @@ -3,6 +3,8 @@ from .cinterface import * # from .connection import TaosConnection from .error import * +from ctypes import c_void_p + class TaosResult(object): """TDengine result interface""" @@ -12,7 +14,11 @@ class TaosResult(object): # to make the __del__ order right self._conn = conn self._close_after = close_after - self._result = result + if isinstance(result, c_void_p): + self._result = result + else: + self._result = c_void_p(result) + self._fields = None self._field_count = None self._precision = None diff --git a/src/connector/python/tests/test_lines.py b/src/connector/python/tests/test_lines.py index bd9d2cdb39d6f4f2612581ce7284c057c456ef91..c9b581e8bcb0d300ade8cb991ba648efc2550f8a 100644 --- a/src/connector/python/tests/test_lines.py +++ b/src/connector/python/tests/test_lines.py @@ -36,7 +36,6 @@ def test_insert_lines(conn): conn.insert_lines(lines) print("inserted") result = conn.query("select * from st") - print(*result.fields) all = result.rows_iter() for row in all: print(row) diff --git a/src/connector/python/tests/test_stmt.py b/src/connector/python/tests/test_stmt.py index 938ba10eb3d2377a63f7972deb99dbd47f7de1b2..3368ecb6a9336a4295790f2cd55314ac9bb6290e 100644 --- a/src/connector/python/tests/test_stmt.py +++ b/src/connector/python/tests/test_stmt.py @@ -1,3 +1,4 @@ +# encoding:UTF-8 from taos import * from ctypes import * diff --git a/src/connector/python/tests/test_stream.py b/src/connector/python/tests/test_stream.py index de6e20928b176e51bc6d350fb01268459f4e7f95..32ec4c5999c975be907cf69a42a04b5f4dd5d54c 100644 --- a/src/connector/python/tests/test_stream.py +++ b/src/connector/python/tests/test_stream.py @@ -20,7 +20,8 @@ def stream_callback(p_param, p_result, p_row): result = TaosResult(p_result) row = TaosRow(result, p_row) try: - ts, count = row() + ts, count = row.as_tuple() + print(ts, count) p = cast(p_param, POINTER(Counter)) p.contents.count += count print("[%s] inserted %d in 5s, total count: %d" % (ts.strftime("%Y-%m-%d %H:%M:%S"), count, p.contents.count)) diff --git a/src/inc/query.h b/src/inc/query.h index fb9cbff8584892b4a6bc6e4a6ce046a7500aef39..0872e3dbaa517ded77dd758b30e69f273c13a580 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -76,6 +76,11 @@ void* qGetResultRetrieveMsg(qinfo_t qinfo); */ int32_t qKillQuery(qinfo_t qinfo); +//kill by qid +int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount); + +bool qSolveCommitNoBlock(void* pRepo, void* pMgmt); + int32_t qQueryCompleted(qinfo_t qinfo); /** diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 000703464cfb8c687e473b7559e1048c42f8a6de..43ed20fe14af7c5f60c580dc08ff8456e7903e65 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -35,6 +35,7 @@ int32_t* taosGetErrno(); #define terrno (*taosGetErrno()) #define TSDB_CODE_SUCCESS 0 +#define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error // rpc #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //"Action in progress") @@ -106,6 +107,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names") #define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length") #define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid column length") +#define TSDB_CODE_TSC_RES_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0227) //"Result set too large to be output") // mnode #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 7abe3e99c720af1682fc103beec9a5d4caeb09eb..089e30ac3728761c68fe155f960c8650a32c2f7a 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -39,6 +39,7 @@ extern "C" { #define TSDB_STATUS_COMMIT_START 1 #define TSDB_STATUS_COMMIT_OVER 2 +#define TSDB_STATUS_COMMIT_NOBLOCK 3 //commit no block, need to be solved // TSDB STATE DEFINITION #define TSDB_STATE_OK 0x0 @@ -413,6 +414,11 @@ int tsdbSyncRecv(void *pRepo, SOCKET socketFd); // For TSDB Compact int tsdbCompact(STsdbRepo *pRepo); +// For TSDB Health Monitor + +// no problem return true +bool tsdbNoProblem(STsdbRepo* pRepo); + #ifdef __cplusplus } #endif diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index d5ddc9d32a8dcd741865823b534c39e7248ae199..c426653ab529405509a4fe7d2c1003a1bd5679b3 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -17,7 +17,7 @@ #include "taos.h" #include "shellCommand.h" -#define SHELL_INPUT_MAX_COMMAND_SIZE 500000 +#define SHELL_INPUT_MAX_COMMAND_SIZE 10000 extern char configDir[]; diff --git a/src/plugins/blm3 b/src/plugins/blm3 deleted file mode 160000 index f56aa0f485d7bb6aebbcefc2007eeecdccb767c8..0000000000000000000000000000000000000000 --- a/src/plugins/blm3 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f56aa0f485d7bb6aebbcefc2007eeecdccb767c8 diff --git a/src/plugins/taosadapter b/src/plugins/taosadapter deleted file mode 160000 index 6397bf5963f62f0aa5c4b9b961b16ed5c62579f1..0000000000000000000000000000000000000000 --- a/src/plugins/taosadapter +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6397bf5963f62f0aa5c4b9b961b16ed5c62579f1 diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index e849ca6726309277511220906083a8214efe29fa..6dda83eaba97e3db926963ae9374f9192b370fbf 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -1005,11 +1005,11 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if ((*retVal < pData[i]) ^ isMin) { *retVal = pData[i]; - TSKEY k = tsList[i]; - - DO_UPDATE_TAG_COLUMNS(pCtx, k); + if(tsList) { + TSKEY k = tsList[i]; + DO_UPDATE_TAG_COLUMNS(pCtx, k); + } } - *notNullElems += 1; } #if defined(_DEBUG_VIEW) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 4d38cf149d61e877485448fbd482d303afa69832..eb21a8f5c304d922489f90b319c515e4b6144d7c 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -8416,6 +8416,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S } pQInfo->qId = qId; + pQInfo->startExecTs = 0; pQInfo->runtimeEnv.pUdfInfo = pUdfInfo; diff --git a/src/query/src/queryMain.c b/src/query/src/queryMain.c index 28c5df32516ae83fbccecd40208b2735e484a6c4..0ba29ab9047d1b526ed29fd7ec3eb87170f2ae17 100644 --- a/src/query/src/queryMain.c +++ b/src/query/src/queryMain.c @@ -35,7 +35,7 @@ typedef struct SQueryMgmt { bool closed; } SQueryMgmt; -static void queryMgmtKillQueryFn(void* handle) { +static void queryMgmtKillQueryFn(void* handle, void* param1) { void** fp = (void**)handle; qKillQuery(*fp); } @@ -452,7 +452,7 @@ void qQueryMgmtNotifyClosed(void* pQMgmt) { pQueryMgmt->closed = true; pthread_mutex_unlock(&pQueryMgmt->lock); - taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn); + taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn, NULL); } void qQueryMgmtReOpen(void *pQMgmt) { @@ -547,3 +547,148 @@ void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); return 0; } + +//kill by qid +int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCount) { + int32_t err = TSDB_CODE_SUCCESS; + void** handle = qAcquireQInfo(pMgmt, qId); + if(handle == NULL) return terrno; + + SQInfo* pQInfo = (SQInfo*)(*handle); + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + qWarn("QId:0x%"PRIx64" be killed(no memory commit).", pQInfo->qId); + setQueryKilled(pQInfo); + + // wait query stop + int32_t loop = 0; + while (pQInfo->owner != 0) { + taosMsleep(waitMs); + if(loop++ > waitCount){ + err = TSDB_CODE_FAILED; + break; + } + } + + qReleaseQInfo(pMgmt, (void **)&handle, true); + return err; +} + +// local struct +typedef struct { + int64_t qId; + int64_t startExecTs; +} SLongQuery; + +// callbark for sort compare +static int compareLongQuery(const void* p1, const void* p2) { + // sort desc + SLongQuery* plq1 = *(SLongQuery**)p1; + SLongQuery* plq2 = *(SLongQuery**)p2; + if(plq1->startExecTs == plq2->startExecTs) { + return 0; + } else if(plq1->startExecTs > plq2->startExecTs) { + return 1; + } else { + return -1; + } +} + +// callback for taosCacheRefresh +static void cbFoundItem(void* handle, void* param1) { + SQInfo * qInfo = *(SQInfo**) handle; + if(qInfo == NULL) return ; + SArray* qids = (SArray*) param1; + if(qids == NULL) return ; + + bool usedMem = true; + bool usedIMem = true; + SMemTable* mem = qInfo->query.memRef.snapshot.omem; + SMemTable* imem = qInfo->query.memRef.snapshot.imem; + if(mem == NULL || T_REF_VAL_GET(mem) == 0) + usedMem = false; + if(imem == NULL || T_REF_VAL_GET(mem) == 0) + usedIMem = false ; + + if(!usedMem && !usedIMem) + return ; + + // push to qids + SLongQuery* plq = (SLongQuery*)malloc(sizeof(SLongQuery)); + plq->qId = qInfo->qId; + plq->startExecTs = qInfo->startExecTs; + taosArrayPush(qids, &plq); +} + +// longquery +void* qObtainLongQuery(void* param){ + SQueryMgmt* qMgmt = (SQueryMgmt*)param; + if(qMgmt == NULL || qMgmt->qinfoPool == NULL) + return NULL; + SArray* qids = taosArrayInit(4, sizeof(int64_t*)); + if(qids == NULL) return NULL; + // Get each item + taosCacheRefresh(qMgmt->qinfoPool, cbFoundItem, qids); + + size_t cnt = taosArrayGetSize(qids); + if(cnt == 0) { + taosArrayDestroy(qids); + return NULL; + } + if(cnt > 1) + taosArraySort(qids, compareLongQuery); + + return qids; +} + +//solve tsdb no block to commit +bool qFixedNoBlock(void* pRepo, void* pMgmt, int32_t longQueryMs) { + SQueryMgmt *pQueryMgmt = pMgmt; + bool fixed = false; + + // qid top list + SArray *qids = (SArray*)qObtainLongQuery(pQueryMgmt); + if(qids == NULL) return false; + + // kill Query + int64_t now = taosGetTimestampMs(); + size_t cnt = taosArrayGetSize(qids); + size_t i; + SLongQuery* plq; + for(i=0; i < cnt; i++) { + plq = (SLongQuery* )taosArrayGetP(qids, i); + if(plq->startExecTs > now) continue; + if(now - plq->startExecTs >= longQueryMs) { + qKillQueryByQId(pMgmt, plq->qId, 500, 10); // wait 50*100 ms + if(tsdbNoProblem(pRepo)) { + fixed = true; + qWarn("QId:0x%"PRIx64" fixed problem after kill this query.", plq->qId); + break; + } + } + } + + // free qids + for(i=0; i < cnt; i++) { + free(taosArrayGetP(qids, i)); + } + taosArrayDestroy(qids); + return fixed; +} + +//solve tsdb no block to commit +bool qSolveCommitNoBlock(void* pRepo, void* pMgmt) { + qWarn("pRepo=%p start solve problem.", pRepo); + if(qFixedNoBlock(pRepo, pMgmt, 10*60*1000)) { + return true; + } + if(qFixedNoBlock(pRepo, pMgmt, 2*60*1000)){ + return true; + } + if(qFixedNoBlock(pRepo, pMgmt, 30*1000)){ + return true; + } + qWarn("pRepo=%p solve problem failed.", pRepo); + return false; +} \ No newline at end of file diff --git a/src/tsdb/inc/tsdbBuffer.h b/src/tsdb/inc/tsdbBuffer.h index ec6b057aef142fb938993b3a27717c5e64937258..4b650d3993a54f6a98caf00a3605feb37e972ebd 100644 --- a/src/tsdb/inc/tsdbBuffer.h +++ b/src/tsdb/inc/tsdbBuffer.h @@ -29,6 +29,7 @@ typedef struct { int tBufBlocks; int nBufBlocks; int nRecycleBlocks; + int nElasticBlocks; int64_t index; SList* bufBlockList; } STsdbBufPool; @@ -41,6 +42,10 @@ int tsdbOpenBufPool(STsdbRepo* pRepo); void tsdbCloseBufPool(STsdbRepo* pRepo); SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); int tsdbExpandPool(STsdbRepo* pRepo, int32_t oldTotalBlocks); -void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode); +void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode, bool bELastic); + +// health cite +STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize); +void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock); #endif /* _TD_TSDB_BUFFER_H_ */ diff --git a/src/tsdb/inc/tsdbHealth.h b/src/tsdb/inc/tsdbHealth.h new file mode 100644 index 0000000000000000000000000000000000000000..324f4312e05fc0ca0200c319728bf692bf476bf6 --- /dev/null +++ b/src/tsdb/inc/tsdbHealth.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_HEALTH_H_ +#define _TD_TSDB_HEALTH_H_ + +bool tsdbUrgeQueryFree(STsdbRepo* pRepo); +int32_t tsdbInsertNewBlock(STsdbRepo* pRepo); + +bool tsdbIdleMemEnough(); +bool tsdbAllowNewBlock(STsdbRepo* pRepo); + +#endif /* _TD_TSDB_BUFFER_H_ */ diff --git a/src/tsdb/inc/tsdbint.h b/src/tsdb/inc/tsdbint.h index 3b0937861d6d256f28818157b4517efecd2f11c7..a7f08e61ed1ac576c1ca70c0da20e14f4b3a306f 100644 --- a/src/tsdb/inc/tsdbint.h +++ b/src/tsdb/inc/tsdbint.h @@ -97,6 +97,7 @@ struct STsdbRepo { SMergeBuf mergeBuf; //used when update=2 int8_t compactState; // compact state: inCompact/noCompact/waitingCompact? + pthread_t* pthread; }; #define REPO_ID(r) (r)->config.tsdbId diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index e675bf6f9de04021112d43a1db70cf56cf430f08..1faec90bf6fcb0fa48ff0b539e2c6da7cbe6df91 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -14,11 +14,10 @@ */ #include "tsdbint.h" +#include "tsdbHealth.h" #define POOL_IS_EMPTY(b) (listNEles((b)->bufBlockList) == 0) -static STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize); -static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock); // ---------------- INTERNAL FUNCTIONS ---------------- STsdbBufPool *tsdbNewBufPool() { @@ -69,6 +68,7 @@ int tsdbOpenBufPool(STsdbRepo *pRepo) { pPool->bufBlockSize = pCfg->cacheBlockSize * 1024 * 1024; // MB pPool->tBufBlocks = pCfg->totalBlocks; pPool->nBufBlocks = 0; + pPool->nElasticBlocks = 0; pPool->index = 0; pPool->nRecycleBlocks = 0; @@ -120,6 +120,18 @@ SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) { STsdbBufPool *pBufPool = pRepo->pPool; while (POOL_IS_EMPTY(pBufPool)) { + if(tsDeadLockKillQuery) { + // supply new Block + if(tsdbInsertNewBlock(pRepo) > 0) { + tsdbWarn("vgId:%d add new elastic block . elasticBlocks=%d cur free Blocks=%d", REPO_ID(pRepo), pBufPool->nElasticBlocks, pBufPool->bufBlockList->numOfEles); + break; + } else { + // no newBlock, kill query free + if(!tsdbUrgeQueryFree(pRepo)) + tsdbWarn("vgId:%d Urge query free thread start failed.", REPO_ID(pRepo)); + } + } + pRepo->repoLocked = false; pthread_cond_wait(&(pBufPool->poolNotEmpty), &(pRepo->mutex)); pRepo->repoLocked = true; @@ -139,11 +151,11 @@ SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) { } // ---------------- LOCAL FUNCTIONS ---------------- -static STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize) { +STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize) { STsdbBufBlock *pBufBlock = (STsdbBufBlock *)malloc(sizeof(*pBufBlock) + bufBlockSize); if (pBufBlock == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + return NULL; } pBufBlock->blockId = 0; @@ -151,13 +163,9 @@ static STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize) { pBufBlock->remain = bufBlockSize; return pBufBlock; - -_err: - tsdbFreeBufBlock(pBufBlock); - return NULL; } -static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); } +void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); } int tsdbExpandPool(STsdbRepo* pRepo, int32_t oldTotalBlocks) { if (oldTotalBlocks == pRepo->config.totalBlocks) { @@ -193,10 +201,15 @@ err: return err; } -void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode) { +void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode, bool bELastic) { STsdbBufBlock *pBufBlock = NULL; tdListNodeGetData(pPool->bufBlockList, pNode, (void *)(&pBufBlock)); tsdbFreeBufBlock(pBufBlock); free(pNode); - pPool->nBufBlocks--; + if(bELastic) { + pPool->nElasticBlocks--; + tsdbWarn("pPool=%p elastic block reduce one . nElasticBlocks=%d cur free Blocks=%d", pPool, pPool->nElasticBlocks, pPool->bufBlockList->numOfEles); + } + else + pPool->nBufBlocks--; } diff --git a/src/tsdb/src/tsdbHealth.c b/src/tsdb/src/tsdbHealth.c new file mode 100644 index 0000000000000000000000000000000000000000..8198c480334912b1ce373ceca7b82409f5a644f2 --- /dev/null +++ b/src/tsdb/src/tsdbHealth.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taosmsg.h" +#include "tarray.h" +#include "query.h" +#include "tglobal.h" +#include "tlist.h" +#include "tsdbint.h" +#include "tsdbBuffer.h" +#include "tsdbLog.h" +#include "tsdbHealth.h" +#include "ttimer.h" +#include "tthread.h" + + +// return malloc new block count +int32_t tsdbInsertNewBlock(STsdbRepo * pRepo) { + STsdbBufPool *pPool = pRepo->pPool; + int32_t cnt = 0; + + if(tsdbAllowNewBlock(pRepo)) { + STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); + if (pBufBlock) { + if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { + // append error + tsdbFreeBufBlock(pBufBlock); + } else { + pPool->nElasticBlocks ++; + cnt ++ ; + } + } + } + return cnt; +} + +// switch anther thread to run +void* cbKillQueryFree(void* param) { + STsdbRepo* pRepo = (STsdbRepo*)param; + // vnode + if(pRepo->appH.notifyStatus) { + pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_NOBLOCK, TSDB_CODE_SUCCESS); + } + + // free + if(pRepo->pthread){ + void* p = pRepo->pthread; + pRepo->pthread = NULL; + free(p); + } + + return NULL; +} + +// return true do free , false do nothing +bool tsdbUrgeQueryFree(STsdbRepo * pRepo) { + // check previous running + if(pRepo->pthread && taosThreadRunning(pRepo->pthread)) { + tsdbWarn("vgId:%d pre urge thread is runing. nBlocks=%d nElasticBlocks=%d", REPO_ID(pRepo), pRepo->pPool->nBufBlocks, pRepo->pPool->nElasticBlocks); + return false; + } + // create new + pRepo->pthread = taosCreateThread(cbKillQueryFree, pRepo); + if(pRepo->pthread == NULL) { + tsdbError("vgId:%d create urge thread error.", REPO_ID(pRepo)); + return false; + } + return true; +} + +bool tsdbAllowNewBlock(STsdbRepo* pRepo) { + int32_t nMaxElastic = pRepo->config.totalBlocks/3; + STsdbBufPool* pPool = pRepo->pPool; + if(pPool->nElasticBlocks >= nMaxElastic) { + tsdbWarn("vgId:%d tsdbAllowNewBlock return fasle. nElasticBlock(%d) >= MaxElasticBlocks(%d)", REPO_ID(pRepo), pPool->nElasticBlocks, nMaxElastic); + return false; + } + return true; +} + +bool tsdbNoProblem(STsdbRepo* pRepo) { + if(listNEles(pRepo->pPool->bufBlockList) == 0) + return false; + return true; +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index b09f6ed4b5a107502f1ee87ba065e05a83d80f1c..39a0543b23d4333dca6f941324976d6bb156907a 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -16,6 +16,8 @@ // no test file errors here #include "taosdef.h" #include "tsdbint.h" +#include "ttimer.h" +#include "tthread.h" #define IS_VALID_PRECISION(precision) \ (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO)) @@ -126,6 +128,10 @@ int tsdbCloseRepo(STsdbRepo *repo, int toCommit) { terrno = TSDB_CODE_SUCCESS; tsdbStopStream(pRepo); + if(pRepo->pthread){ + taosDestoryThread(pRepo->pthread); + pRepo->pthread = NULL; + } if (toCommit) { tsdbSyncCommit(repo); @@ -547,6 +553,7 @@ static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { pRepo->appH = *pAppH; } pRepo->repoLocked = false; + pRepo->pthread = NULL; int code = pthread_mutex_init(&(pRepo->mutex), NULL); if (code != 0) { diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index c5ee4b0b6f78e3a489c640954722fe6b79facbd4..be074bff83ec1c3fb5708fad6aac0afa33684a0e 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -99,17 +99,22 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { STsdbBufPool *pBufPool = pRepo->pPool; SListNode *pNode = NULL; - bool recycleBlocks = pBufPool->nRecycleBlocks > 0; + bool addNew = false; if (tsdbLockRepo(pRepo) < 0) return -1; while ((pNode = tdListPopHead(pMemTable->bufBlockList)) != NULL) { if (pBufPool->nRecycleBlocks > 0) { - tsdbRecycleBufferBlock(pBufPool, pNode); + tsdbRecycleBufferBlock(pBufPool, pNode, false); pBufPool->nRecycleBlocks -= 1; } else { - tdListAppendNode(pBufPool->bufBlockList, pNode); + if(pBufPool->nElasticBlocks > 0 && listNEles(pBufPool->bufBlockList) > 2) { + tsdbRecycleBufferBlock(pBufPool, pNode, true); + } else { + tdListAppendNode(pBufPool->bufBlockList, pNode); + addNew = true; + } } } - if (!recycleBlocks) { + if (addNew) { int code = pthread_cond_signal(&pBufPool->poolNotEmpty); if (code != 0) { if (tsdbUnlockRepo(pRepo) < 0) return -1; @@ -555,7 +560,7 @@ static void tsdbFreeTableData(STableData *pTableData) { } } -static char *tsdbGetTsTupleKey(const void *data) { return memRowTuple((SMemRow)data); } +static char *tsdbGetTsTupleKey(const void *data) { return memRowKeys((SMemRow)data); } static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { ASSERT(pMemTable->maxTables < maxTables); diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index e41b544d00e55f7eece904c5957ef9c06063e6c3..40069d7d273caa14ce3b80467b25d68ea476fb75 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -33,6 +33,7 @@ extern "C" { #endif typedef void (*__cache_free_fn_t)(void*); +typedef void (*__cache_trav_fn_t)(void*, void*); typedef struct SCacheStatis { int64_t missCount; @@ -176,7 +177,7 @@ void taosCacheCleanup(SCacheObj *pCacheObj); * @param fp * @return */ -void taosCacheRefresh(SCacheObj *pCacheObj, __cache_free_fn_t fp); +void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void* param1); /** * stop background refresh worker thread diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index 2a2ac7dacf378d90bb925e97c32887c487fc3313..e2165ec1a32c3b47a47a4b22243d1ac178d58876 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -20,7 +20,7 @@ extern "C" { #endif -#define TSDB_CFG_MAX_NUM 121 +#define TSDB_CFG_MAX_NUM 122 #define TSDB_CFG_PRINT_LEN 23 #define TSDB_CFG_OPTION_LEN 24 #define TSDB_CFG_VALUE_LEN 41 diff --git a/src/util/inc/tthread.h b/src/util/inc/tthread.h new file mode 100644 index 0000000000000000000000000000000000000000..7443ad706dcbef529d857fe823cddd0cc1efbdd3 --- /dev/null +++ b/src/util/inc/tthread.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TTHREAD_H +#define TDENGINE_TTHREAD_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" +#include "taosdef.h" + +// create new thread +pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param); +// destory thread +bool taosDestoryThread(pthread_t* pthread); +// thread running return true +bool taosThreadRunning(pthread_t* pthread); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TTHREAD_H diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 1bb2c7070dafa875afcf836174f337147a93ae7c..f49dd16c3943040bcc9b58000a87845ea1c87b13 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -505,7 +505,8 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { typedef struct SHashTravSupp { SCacheObj* pCacheObj; int64_t time; - __cache_free_fn_t fp; + __cache_trav_fn_t fp; + void* param1; } SHashTravSupp; static bool travHashTableEmptyFn(void* param, void* data) { @@ -667,17 +668,17 @@ bool travHashTableFn(void* param, void* data) { } if (ps->fp) { - (ps->fp)(pNode->data); + (ps->fp)(pNode->data, ps->param1); } // do not remove element in hash table return true; } -static void doCacheRefresh(SCacheObj* pCacheObj, int64_t utl_time, __cache_free_fn_t fp) { +static void doCacheRefresh(SCacheObj* pCacheObj, int64_t utl_time, __cache_trav_fn_t fp, void* param1) { assert(pCacheObj != NULL); - SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = fp, .time = utl_time}; + SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = fp, .time = utl_time, .param1 = param1}; taosHashCondTraverse(pCacheObj->pHashTable, travHashTableFn, &sup); } @@ -748,7 +749,7 @@ void* taosCacheTimedRefresh(void *handle) { // refresh data in hash table if (elemInHash > 0) { int64_t now = taosGetTimestampMs(); - doCacheRefresh(pCacheObj, now, NULL); + doCacheRefresh(pCacheObj, now, NULL, NULL); } taosTrashcanEmpty(pCacheObj, false); @@ -766,13 +767,13 @@ void* taosCacheTimedRefresh(void *handle) { return NULL; } -void taosCacheRefresh(SCacheObj *pCacheObj, __cache_free_fn_t fp) { +void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void* param1) { if (pCacheObj == NULL) { return; } int64_t now = taosGetTimestampMs(); - doCacheRefresh(pCacheObj, now, fp); + doCacheRefresh(pCacheObj, now, fp, param1); } void taosStopCacheRefreshWorker(void) { diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 0b9a76004b3e445abe1bbffe593d8061803d1b1b..d1ecc2188bec5c4a5b1af36a67cf177dd6d935ec 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -112,9 +112,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, "SQL statement too lon TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FILE_EMPTY, "File is empty") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_LINE_SYNTAX_ERROR, "Syntax error in Line") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_META_CACHED, "No table meta cached") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_COL_NAMES, "duplicated column names") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TAG_LENGTH, "Invalid tag length") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid column length") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_COL_NAMES, "duplicated column names") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TAG_LENGTH, "Invalid tag length") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid column length") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_RES_TOO_MANY, "Result set too large to be output") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") diff --git a/src/util/src/tthread.c b/src/util/src/tthread.c new file mode 100644 index 0000000000000000000000000000000000000000..043b2de2f241297d209041294428dde2c55e974e --- /dev/null +++ b/src/util/src/tthread.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "tthread.h" +#include "tglobal.h" +#include "taosdef.h" +#include "tutil.h" +#include "tulog.h" +#include "taoserror.h" + +// create new thread +pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param) { + pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t)); + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + int32_t ret = pthread_create(pthread, &thattr, __start_routine, param); + pthread_attr_destroy(&thattr); + + if (ret != 0) { + free(pthread); + return NULL; + } + return pthread; +} + +// destory thread +bool taosDestoryThread(pthread_t* pthread) { + if(pthread == NULL) return false; + if(taosThreadRunning(pthread)) { + pthread_cancel(*pthread); + pthread_join(*pthread, NULL); + } + + free(pthread); + return true; +} + +// thread running return true +bool taosThreadRunning(pthread_t* pthread) { + if(pthread == NULL) return false; + int ret = pthread_kill(*pthread, 0); + if(ret == ESRCH) + return false; + if(ret == EINVAL) + return false; + // alive + return true; +} diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index fd61cbef77b03bae7d76be9b74e37a059c64078e..60c60cf531c815023fb18eafcc79d7c41b5cae4f 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -560,5 +560,10 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { return vnodeSaveVersion(pVnode); } + // timer thread callback + if(status == TSDB_STATUS_COMMIT_NOBLOCK) { + qSolveCommitNoBlock(pVnode->tsdb, pVnode->qMgmt); + } + return 0; } diff --git a/tests/examples/python/taosdemo/taosdemo.py b/tests/examples/python/taosdemo/taosdemo.py index d55023bdbf119544a788aa6246c9d63dbf024872..4aaf00157c5fe5bbeec27b001f663a94c1d89439 100755 --- a/tests/examples/python/taosdemo/taosdemo.py +++ b/tests/examples/python/taosdemo/taosdemo.py @@ -21,78 +21,91 @@ import json import random import time import datetime +import multiprocessing from multiprocessing import Manager, Pool, Lock from multipledispatch import dispatch from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED @dispatch(str, str) -def v_print(msg: str, arg: str): +def v_print(msg, arg): + # type: (str, str) -> None if verbose: print(msg % arg) @dispatch(str, str, str) -def v_print(msg: str, arg1: str, arg2: str): +def v_print(msg, arg1, arg2): + # type: (str, str, str) -> None if verbose: print(msg % (arg1, arg2)) @dispatch(str, str, str, str) -def v_print(msg: str, arg1: str, arg2: str, arg3: str): +def v_print(msg, arg1, arg2, arg3): + # type: (str, str, str, str) -> None if verbose: print(msg % (arg1, arg2, arg3)) @dispatch(str, str, str, str, str) -def v_print(msg: str, arg1: str, arg2: str, arg3: str, arg4: str): +def v_print(msg, arg1, arg2, arg3, arg4): + # type: (str, str, str, str, str) -> None if verbose: print(msg % (arg1, arg2, arg3, arg4)) @dispatch(str, int) -def v_print(msg: str, arg: int): +def v_print(msg, arg): + # type: (str, int) -> None if verbose: print(msg % int(arg)) @dispatch(str, int, str) -def v_print(msg: str, arg1: int, arg2: str): +def v_print(msg, arg1, arg2): + # type: (str, int, str) -> None if verbose: print(msg % (int(arg1), str(arg2))) @dispatch(str, str, int) -def v_print(msg: str, arg1: str, arg2: int): +def v_print(msg, arg1, arg2): + # type: (str, str, int) -> None if verbose: print(msg % (arg1, int(arg2))) @dispatch(str, int, int) -def v_print(msg: str, arg1: int, arg2: int): +def v_print(msg, arg1, arg2): + # type: (str, int, int) -> None if verbose: print(msg % (int(arg1), int(arg2))) @dispatch(str, int, int, str) -def v_print(msg: str, arg1: int, arg2: int, arg3: str): +def v_print(msg, arg1, arg2, arg3): + # type: (str, int, int, str) -> None if verbose: print(msg % (int(arg1), int(arg2), str(arg3))) @dispatch(str, int, int, int) -def v_print(msg: str, arg1: int, arg2: int, arg3: int): +def v_print(msg, arg1, arg2, arg3): + # type: (str, int, int, int) -> None if verbose: print(msg % (int(arg1), int(arg2), int(arg3))) @dispatch(str, int, int, int, int) -def v_print(msg: str, arg1: int, arg2: int, arg3: int, arg4: int): +def v_print(msg, arg1, arg2, arg3, arg4): + # type: (str, int, int, int, int) -> None if verbose: print(msg % (int(arg1), int(arg2), int(arg3), int(arg4))) -def restful_execute(host: str, port: int, user: str, password: str, cmd: str): +def restful_execute(host, port, user, password, cmd): + # type: (str, int, str, str, str) -> None url = "http://%s:%d/rest/sql" % (host, restPort) v_print("restful_execute - cmd: %s", cmd) @@ -112,7 +125,8 @@ def restful_execute(host: str, port: int, user: str, password: str, cmd: str): print("resp: %s" % json.dumps(resp.json())) -def query_func(process: int, thread: int, cmd: str): +def query_func(process, thread, cmd): + # type: (int, int, str) -> None v_print("%d process %d thread cmd: %s", process, thread, cmd) if oneMoreHost != "NotSupported" and random.randint( @@ -133,7 +147,8 @@ def query_func(process: int, thread: int, cmd: str): host, port, user, password, cmd) -def query_data_process(cmd: str): +def query_data_process(cmd): + # type: (str) -> None # establish connection if native if native: v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir) @@ -256,7 +271,8 @@ def drop_databases(): (dbName, i)) -def insert_func(process: int, thread: int): +def insert_func(process, thread): + # type: (int, int) -> None v_print("%d process %d thread, insert_func ", process, thread) # generate uuid @@ -374,7 +390,8 @@ def create_tb(): (tbName, j)) -def insert_data_process(lock, i: int, begin: int, end: int): +def insert_data_process(lock, i, begin, end): + # type: (multiprocessing._LockType, int, int, int) -> None lock.acquire() tasks = end - begin v_print("insert_data_process:%d table from %d to %d, tasks %d", i, begin, end, tasks) @@ -675,7 +692,10 @@ if __name__ == "__main__": printConfig() if not skipPrompt: - input("Press any key to continue..") + try: + input("Press any key to continue..") + except SyntaxError: + pass # establish connection first if native if native: diff --git a/tests/pytest/fulltest.bat b/tests/pytest/fulltest.bat index 6eac17ad2e53a31d48acc37f856a607f78304d6f..535aafe2d4692f4dc85756256390502c5fd96efd 100644 --- a/tests/pytest/fulltest.bat +++ b/tests/pytest/fulltest.bat @@ -19,4 +19,3 @@ python .\test.py -f query\filterFloatAndDouble.py python .\test.py -f query\filterOtherTypes.py python .\test.py -f query\querySort.py python .\test.py -f query\queryJoin.py -python .\test.py -f tools\windows_input.py \ No newline at end of file diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index da2f2c48dfd9d925600cf8a12f2f7e2e051d04ed..35cb0eeed1067b0fe1b252a8decb3bb3026ee501 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -42,6 +42,7 @@ python3 ./test.py -f table/alter_column.py python3 ./test.py -f table/boundary.py python3 ./test.py -f table/create.py python3 ./test.py -f table/del_stable.py +python3 ./test.py -f table/create_db_from_normal_db.py #stable python3 ./test.py -f stable/insert.py diff --git a/tests/pytest/query/unionAllTest.py b/tests/pytest/query/unionAllTest.py index d1e1bf4d3e11191be2875c464150793936acd065..9a7aedc763db05ebb11da33dc65fa74b2ba8daa3 100644 --- a/tests/pytest/query/unionAllTest.py +++ b/tests/pytest/query/unionAllTest.py @@ -103,10 +103,27 @@ class TDTestCase: select count(*) as count, loc from st where ts between 1600000000000 and 1600000000010 group by loc''') tdSql.checkRows(6) + # https://jira.taosdata.com:18080/browse/TS-715 + tdLog.info("test case for TS-715") + sql = "" + + tdSql.execute("create table st2(ts timestamp, c1 int, c2 int, c3 int) tags(loc nchar(20))") + + for i in range(101): + if i == 0: + sql = "select last(*) from sub0 " + else: + sql += f"union all select last(*) from sub{i} " + + tdSql.execute("create table sub%d using st2 tags('nchar%d')" % (i, i)) + tdSql.execute("insert into sub%d values(%d, %d, %d, %d)" % (i, self.ts + i, i, i, i)) + + tdSql.error(sql) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/table/create_db_from_normal_db.py b/tests/pytest/table/create_db_from_normal_db.py new file mode 100644 index 0000000000000000000000000000000000000000..8b5182c3b16ca31b2bbf966df294e2c4e4c12ff3 --- /dev/null +++ b/tests/pytest/table/create_db_from_normal_db.py @@ -0,0 +1,45 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("test case for TS-783") + tdSql.execute("drop table if exists db.state1;") + tdSql.execute("create table db.state1 (ts timestamp, c1 int);") + tdSql.error("create table db.test1 using db.state1 tags('tt');") + + tdSql.execute("drop table if exists db.state2;") + tdSql.execute("create table db.state2 (ts timestamp, c1 int) tags (t binary(20));") + tdSql.query("create table db.test2 using db.state2 tags('tt');") + tdSql.error("create table db.test22 using db.test2 tags('tt');") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/windows_input.py b/tests/pytest/tools/windows_input.py index 72f50a5acfa8b65c99b29a3beb55d384ad5c5dd6..99e30fa311673903850149c1916c6466c6a29ee5 100644 --- a/tests/pytest/tools/windows_input.py +++ b/tests/pytest/tools/windows_input.py @@ -15,6 +15,7 @@ import os from uiautomation import WindowControl from util.cases import * from util.sql import * +import clipboard class TDTestCase: @@ -55,16 +56,22 @@ class TDTestCase: sql = "insert into db.tb values(now,'%s');" % temp window.SendKeys(sql) window.SendKeys('{Enter}') + window.SendKeys('{Ctrl}A') + window.SendKeys('{Ctrl}C') + # 获取剪切板里面的复制内容 + result = clipboard.paste() window.SendKeys('{Ctrl}C') window.SendKeys('exit') window.SendKeys('{Enter}') + return result def run(self): tdSql.prepare() ret = tdSql.execute('create table tb (ts timestamp, i binary(300))') - self.win_input_test() + result = self.win_input_test() + tdLog.info(result) tdSql.query("select * from tb") tdSql.checkRows(1) diff --git a/tests/script/general/parser/nestquery.sim b/tests/script/general/parser/nestquery.sim index ed1e9d47909af13647114d0e45e34b60f66d303e..5bbdace4da0cb65b4573e2f9cd93b53ac28f6c2f 100644 --- a/tests/script/general/parser/nestquery.sim +++ b/tests/script/general/parser/nestquery.sim @@ -330,9 +330,9 @@ if $data12 != 71680.000000000 then return -1 endi -sql select top(x, 20) from (select c1 x from nest_tb0); +sql select top(x, 20) from (select ts,c1 x from nest_tb0); -sql select bottom(x, 20) from (select c1 x from nest_tb0) +sql select bottom(x, 20) from (select ts,c1 x from nest_tb0) print ===================> group by + having @@ -407,12 +407,14 @@ if $data03 != @20-09-15 00:00:00.000@ then return -1 endi -sql select diff(val) from (select c1 val from nest_tb0); +sql_error select diff(val) from (select c1 val from nest_tb0); + +sql select diff(val) from (select ts,c1 val from nest_tb0); if $rows != 9999 then return -1 endi -if $data00 != @70-01-01 08:00:00.000@ then +if $data00 != @20-09-15 00:01:00.000@ then return -1 endi diff --git a/tests/script/general/parser/slimit_query.sim b/tests/script/general/parser/slimit_query.sim index 0a793f0611a5875f53ac75644f3156ad9aa6cc65..0dbcffa0a43b596d542c8799a9a94170e6f71f5a 100644 --- a/tests/script/general/parser/slimit_query.sim +++ b/tests/script/general/parser/slimit_query.sim @@ -31,6 +31,8 @@ $tsu = $tsu + $ts0 #sql_error select top(c1, 1) from $stb where ts >= $ts0 and ts <= $tsu slimit 5 offset 1 #sql_error select bottom(c1, 1) from $stb where ts >= $ts0 and ts <= $tsu slimit 5 offset 1 +sql_error select t1 from $stb slimit 5 offset 1; + ### select from stb + group by + slimit offset sql select max(c1), min(c2), avg(c3), sum(c4), spread(c5), sum(c6), count(c7), first(c8), last(c9) from $stb group by t1 slimit 5 soffset 0 if $rows != 5 then diff --git a/tests/script/unique/dnode/alternativeRole.sim b/tests/script/unique/dnode/alternativeRole.sim index 955b757f06df22c884565d0fac350456e79cb73a..7e647925d1d3d66d21f279ace852e3fc12496510 100644 --- a/tests/script/unique/dnode/alternativeRole.sim +++ b/tests/script/unique/dnode/alternativeRole.sim @@ -32,33 +32,48 @@ sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start sleep 3000 +$x = 0 +show1: + $x = $x + 1 + sleep 1000 + if $x == 30 then + return -1 + endi + sql show dnodes print dnode1 $data5_1 -print dnode1 $data5_2 -print dnode1 $data5_3 +print dnode2 $data5_2 +print dnode3 $data5_3 if $data5_1 != mnode then - return -1 + goto show1 endi if $data5_2 != vnode then - return -1 + goto show1 endi if $data5_3 != any then - return -1 + goto show1 endi +show2: + $x = $x + 1 + sleep 1000 + if $x == 30 then + return -1 + endi + sql show mnodes print dnode1 ==> $data2_1 print dnode2 ==> $data2_2 print dnode3 ==> $data2_3 if $data2_1 != master then - return -1 + goto show2 endi if $data2_2 != null then - return -1 + goto show2 endi if $data2_3 != slave then - return -1 + goto show2 endi print ========== step2 @@ -72,26 +87,28 @@ sql create table d1.t6 (ts timestamp, i int) sql create table d1.t7 (ts timestamp, i int) sql create table d1.t8 (ts timestamp, i int) +show3: + $x = $x + 1 + sleep 1000 + if $x == 30 then + return -1 + endi + sql show dnodes print dnode1 $data2_1 print dnode2 $data2_2 print dnode3 $data2_3 if $data2_1 != 0 then - return -1 + goto show3 endi if $data2_2 != 1 then - return -1 + goto show3 endi if $data2_3 != 1 then - return -1 + goto show3 endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -system sh/exec.sh -n dnode5 -s stop -x SIGINT -system sh/exec.sh -n dnode6 -s stop -x SIGINT -system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file