diff --git a/Jenkinsfile b/Jenkinsfile index 3cbcad14233a9de6bd78651a1c0a96d53643c3cd..9d986e742f654ca3b71a1af0cc7a629cb3eacc72 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,5 @@ import hudson.model.Result +import hudson.model.*; import jenkins.model.CauseOfInterruption properties([pipelineTriggers([githubPush()])]) node { @@ -6,6 +7,7 @@ node { } def skipbuild=0 +def win_stop=0 def abortPreviousBuilds() { def currentJobName = env.JOB_NAME @@ -110,7 +112,83 @@ def pre_test(){ ''' return 1 } +def pre_test_win(){ + bat ''' + cd C:\\ + rd /s /Q C:\\TDengine + cd C:\\workspace\\TDinternal + rd /s /Q C:\\workspace\\TDinternal\\debug + cd C:\\workspace\\TDinternal\\community + git reset --hard HEAD~10 + ''' + script { + if (env.CHANGE_TARGET == 'master') { + bat ''' + cd C:\\workspace\\TDinternal\\community + git checkout master + ''' + } + else if(env.CHANGE_TARGET == '2.0'){ + bat ''' + cd C:\\workspace\\TDinternal\\community + git checkout 2.0 + ''' + } + else{ + bat ''' + cd C:\\workspace\\TDinternal\\community + git checkout develop + ''' + } + } + bat''' + cd C:\\workspace\\TDinternal\\community + git pull + git fetch origin +refs/pull/%CHANGE_ID%/merge + git checkout -qf FETCH_HEAD + git clean -dfx + cd C:\\workspace\\TDinternal + git reset --hard HEAD~10 + ''' + script { + if (env.CHANGE_TARGET == 'master') { + bat ''' + cd C:\\workspace\\TDinternal + git checkout master + ''' + } + else if(env.CHANGE_TARGET == '2.0'){ + bat ''' + cd C:\\workspace\\TDinternal + git checkout 2.0 + ''' + } + else{ + bat ''' + cd C:\\workspace\\TDinternal + git checkout develop + ''' + } + } + bat ''' + cd C:\\workspace\\TDinternal + git pull + date + git clean -dfx + mkdir debug + cd debug + call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall.bat" amd64 + cmake ../ -G "NMake Makefiles" + nmake + nmake install + xcopy /e/y/i/f C:\\workspace\\TDinternal\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 + cd C:\\workspace\\TDinternal\\community\\src\\connector\\python + python -m pip install . + + ''' + return 1 +} pipeline { agent none environment{ @@ -236,9 +314,9 @@ pipeline { node nodejsChecker.js host=localhost ''' sh ''' - cd ${WKC}/tests/examples/C#/taosdemo - mcs -out:taosdemo *.cs > /dev/null 2>&1 - echo '' |./taosdemo -c /etc/taos + cd ${WKC}/tests/examples/C#/taosdemo + mcs -out:taosdemo *.cs > /dev/null 2>&1 + ./taosdemo -c /etc/taos -y ''' sh ''' cd ${WKC}/tests/gotest @@ -369,7 +447,37 @@ pipeline { date''' } } - } + } + + stage('build'){ + agent{label " wintest "} + steps { + pre_test() + script{ + while(win_stop == 0){ + sleep(1) + } + } + } + } + stage('test'){ + agent{label "win"} + steps{ + + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + pre_test_win() + bat''' + cd C:\\workspace\\TDinternal\\community\\tests\\pytest + .\\test-all.bat Wintest + ''' + } + script{ + win_stop=1 + } + } + } + + } } } @@ -451,4 +559,4 @@ pipeline { ) } } -} \ No newline at end of file +} diff --git a/cmake/install.inc b/cmake/install.inc index e9ad240a793b9736edbe5769c6af12276e13a1a6..5a39173697e596cb9fa96a883000d2f3ea996c46 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -6,6 +6,8 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) IF (TD_POWER) SET(CMAKE_INSTALL_PREFIX C:/PowerDB) + ELSEIF (TD_PRO) + SET(CMAKE_INSTALL_PREFIX C:/ProDB) ELSE () SET(CMAKE_INSTALL_PREFIX C:/TDengine) ENDIF () @@ -24,6 +26,8 @@ ELSEIF (TD_WINDOWS) IF (TD_POWER) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/power.exe DESTINATION .) + ELSEIF (TD_PRO) + INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/prodbc.exe DESTINATION .) ELSE () INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taos.exe DESTINATION .) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/taosdemo.exe DESTINATION .) diff --git a/cmake/version.inc b/cmake/version.inc index baba08d748ea59df3ed3a4eb27343ba159c074eb..8d56ff1de6a39c1a1ecdbcf4e681c0d380ec1ba1 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.2.0.2") + SET(TD_VER_NUMBER "2.2.0.5") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh old mode 100755 new mode 100644 index d400d0b91a2d02e9b3e0232d67e2ed6b00cdf541..eb61963de397db7773bf713bcc18de9f19b794b5 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -45,8 +45,10 @@ else inc_link_dir="/usr/local/include" install_main_dir="/usr/local/Cellar/tdengine/${verNumber}" - + install_main_2_dir="/usr/local/Cellar/tdengine@${verNumber}/${verNumber}" + bin_dir="/usr/local/Cellar/tdengine/${verNumber}/bin" + bin_2_dir="/usr/local/Cellar/tdengine@${verNumber}/${verNumber}/bin" fi service_config_dir="/etc/systemd/system" @@ -121,16 +123,25 @@ function kill_taosd() { function install_main_path() { #create install main dir and all sub dir - ${csudo} rm -rf ${install_main_dir} || : - ${csudo} mkdir -p ${install_main_dir} - ${csudo} mkdir -p ${install_main_dir}/cfg - ${csudo} mkdir -p ${install_main_dir}/bin - ${csudo} mkdir -p ${install_main_dir}/connector - ${csudo} mkdir -p ${install_main_dir}/driver - ${csudo} mkdir -p ${install_main_dir}/examples - ${csudo} mkdir -p ${install_main_dir}/include if [ "$osType" != "Darwin" ]; then + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/cfg + ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/connector + ${csudo} mkdir -p ${install_main_dir}/driver + ${csudo} mkdir -p ${install_main_dir}/examples + ${csudo} mkdir -p ${install_main_dir}/include ${csudo} mkdir -p ${install_main_dir}/init.d + else + ${csudo} rm -rf ${install_main_dir} || ${csudo} rm -rf ${install_main_2_dir} || : + ${csudo} mkdir -p ${install_main_dir} || ${csudo} mkdir -p ${install_main_2_dir} + ${csudo} mkdir -p ${install_main_dir}/cfg || ${csudo} mkdir -p ${install_main_2_dir}/cfg + ${csudo} mkdir -p ${install_main_dir}/bin || ${csudo} mkdir -p ${install_main_2_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/connector || ${csudo} mkdir -p ${install_main_2_dir}/connector + ${csudo} mkdir -p ${install_main_dir}/driver || ${csudo} mkdir -p ${install_main_2_dir}/driver + ${csudo} mkdir -p ${install_main_dir}/examples || ${csudo} mkdir -p ${install_main_2_dir}/examples + ${csudo} mkdir -p ${install_main_dir}/include || ${csudo} mkdir -p ${install_main_2_dir}/include fi } @@ -145,33 +156,34 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/perfMonitor || : ${csudo} rm -f ${bin_link_dir}/set_core || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : - fi - - ${csudo} cp -r ${binary_dir}/build/bin/* ${install_main_dir}/bin - ${csudo} cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin - - if [ "$osType" != "Darwin" ]; then + + ${csudo} cp -r ${binary_dir}/build/bin/* ${install_main_dir}/bin + ${csudo} cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin + ${csudo} cp -r ${script_dir}/remove.sh ${install_main_dir}/bin ${csudo} cp -r ${script_dir}/set_core.sh ${install_main_dir}/bin ${csudo} cp -r ${script_dir}/startPre.sh ${install_main_dir}/bin - else - ${csudo} cp -r ${script_dir}/remove_client.sh ${install_main_dir}/bin - fi - ${csudo} chmod 0555 ${install_main_dir}/bin/* - - #Make link - [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : - [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || : - [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : - [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : - - if [ "$osType" != "Darwin" ]; then + + ${csudo} chmod 0555 ${install_main_dir}/bin/* + #Make link + [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : + [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || : + [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : + [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo} ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || : [ -x ${install_main_dir}/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : - fi + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : + else - if [ "$osType" != "Darwin" ]; then - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : + ${csudo} cp -r ${binary_dir}/build/bin/* ${install_main_dir}/bin || ${csudo} cp -r ${binary_dir}/build/bin/* ${install_main_2_dir}/bin || : + ${csudo} cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin || ${csudo}cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_2_dir} || : + ${csudo} cp -r ${script_dir}/remove_client.sh ${install_main_dir}/bin || ${csudo} cp -r ${script_dir}/remove_client.sh ${install_main_2_dir}/bin + ${csudo} chmod 0555 ${install_main_dir}/bin/* || ${csudo} chmod 0555 ${install_main_2_dir}/bin/* + #Make link + [ -x ${install_main_dir}/bin/taos ] || [ -x ${install_main_2_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || ${csudo} ln -s ${install_main_2_dir}/bin/taos || : + [ -x ${install_main_dir}/bin/taosd ] || [ -x ${install_main_2_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || ${csudo} ln -s ${install_main_2_dir}/bin/taosd || : + [ -x ${install_main_dir}/bin/taosdump ] || [ -x ${install_main_2_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || ln -s ${install_main_2_dir}/bin/taosdump ${bin_link_dir}/taosdump || : + [ -x ${install_main_dir}/bin/taosdemo ] || [ -x ${install_main_2_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || ln -s ${install_main_2_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : fi } @@ -243,10 +255,11 @@ function install_lib() { ${csudo} ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so fi else - ${csudo} cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* - - ${csudo} ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib - ${csudo} ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib + ${csudo} cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib ${install_main_dir}/driver || ${csudo} cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib ${install_main_2_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* || ${csudo} chmod 777 ${install_main_2_dir}/driver/* + + ${csudo} ln -sf ${install_main_dir}/driver/libtaos.* ${install_main_dir}/driver/libtaos.dylib || ${csudo} ln -sf ${install_main_2_dir}/driver/libtaos.* ${install_main_2_dir}/driver/libtaos.dylib || : + ${csudo} ln -sf ${install_main_dir}/driver/libtaos.${verNumber}.dylib ${lib_link_dir}/libtaos.1.dylib || : + ${csudo} ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib || : fi install_jemalloc @@ -259,12 +272,12 @@ function install_lib() { function install_header() { if [ "$osType" != "Darwin" ]; then - ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : - fi - ${csudo} cp -f ${source_dir}/src/inc/taos.h ${source_dir}/src/inc/taoserror.h ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* - if [ "$osType" != "Darwin" ]; then + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${source_dir}/src/inc/taos.h ${source_dir}/src/inc/taoserror.h ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h + else + ${csudo} cp -f ${source_dir}/src/inc/taos.h ${source_dir}/src/inc/taoserror.h ${install_main_dir}/include || ${csudo} cp -f ${source_dir}/src/inc/taos.h ${source_dir}/src/inc/taoserror.h ${install_main_2_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* || ${csudo} chmod 644 ${install_main_2_dir}/include/* fi } @@ -276,23 +289,30 @@ function install_config() { [ -f ${script_dir}/../cfg/taos.cfg ] && ${csudo} cp ${script_dir}/../cfg/taos.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* - fi - - ${csudo} cp -f ${script_dir}/../cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org - - if [ "$osType" != "Darwin" ]; then ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + ${csudo} cp -f ${script_dir}/../cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org + ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + else + ${csudo} cp -f ${script_dir}/../cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org || ${csudo} cp -f ${script_dir}/../cfg/taos.cfg ${install_main_2_dir}/cfg/taos.cfg.org fi } function install_log() { ${csudo} rm -rf ${log_dir} || : ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} - ${csudo} ln -s ${log_dir} ${install_main_dir}/log + if [ "$osType" != "Darwin" ]; then + ${csudo} ln -s ${log_dir} ${install_main_dir}/log + else + ${csudo} ln -s ${log_dir} ${install_main_dir}/log || ${csudo} ln -s ${log_dir} ${install_main_2_dir}/log + fi } function install_data() { ${csudo} mkdir -p ${data_dir} - ${csudo} ln -s ${data_dir} ${install_main_dir}/data + if [ "$osType" != "Darwin" ]; then + ${csudo} ln -s ${data_dir} ${install_main_dir}/data + else + ${csudo} ln -s ${data_dir} ${install_main_dir}/data || ${csudo} ln -s ${data_dir} ${install_main_2_dir}/data + fi } function install_connector() { @@ -306,12 +326,21 @@ function install_connector() { else echo "WARNING: go connector not found, please check if want to use it!" fi - ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector - ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null + if [ "$osType" != "Darwin" ]; then + ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector + ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null + else + ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector || ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_2_dir}/connector} + ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null || cp ${binary_dir}/build/lib/*.jar ${install_main_2_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || ${csudo} chmod 777 ${install_main_2_dir}/connector/*.jar || echo &> /dev/null + fi } function install_examples() { - ${csudo} cp -rf ${source_dir}/tests/examples/* ${install_main_dir}/examples + if [ "$osType" != "Darwin" ]; then + ${csudo} cp -rf ${source_dir}/tests/examples/* ${install_main_dir}/examples + else + ${csudo} cp -rf ${source_dir}/tests/examples/* ${install_main_dir}/examples || ${csudo} cp -rf ${source_dir}/tests/examples/* ${install_main_2_dir}/examples + fi } function clean_service_on_sysvinit() { @@ -530,8 +559,16 @@ function install_TDengine() { ## ==============================Main program starts from here============================ echo source directory: $1 echo binary directory: $2 -if [ -x ${bin_dir}/taos ]; then - update_TDengine +if [ "$osType" != "Darwin" ]; then + if [ -x ${bin_dir}/taos ]; then + update_TDengine + else + install_TDengine + fi else - install_TDengine + if [ -x ${bin_dir}/taos ] || [ -x ${bin_2_dir}/taos ]; then + update_TDengine + else + install_TDengine + fi fi diff --git a/packaging/tools/makepkg_pro.sh b/packaging/tools/makepkg_pro.sh index ffe4566b42017a7bffa6166ae28e18ca29bd03cd..41b64509706238c9c58c5394102a10581dd7cb0a 100755 --- a/packaging/tools/makepkg_pro.sh +++ b/packaging/tools/makepkg_pro.sh @@ -76,6 +76,15 @@ if [ "$verMode" == "cluster" ]; then cp ${nginx_dir}/png/taos.png ${install_dir}/nginxd/admin/images/taos.png rm -rf ${install_dir}/nginxd/png + # replace the OEM name, add by yangzy@2021-09-22 + sed -i -e 's/www.taosdata.com/www.hanatech.com.cn/g' $(grep -r 'www.taosdata.com' ${install_dir}/nginxd | sed -r "s/(.*\.html):\s*(.*)/\1/g") + sed -i -e 's/TAOS Data/Hanatech/g' $(grep -r 'TAOS Data' ${install_dir}/nginxd | sed -r "s/(.*\.html):\s*(.*)/\1/g") + sed -i -e 's/taosd/prodbs/g' `grep -r 'taosd' ${install_dir}/nginxd | grep -E '*\.js\s*.*' | sed -r -e 's/(.*\.js):\s*(.*)/\1/g' | sort | uniq` + + sed -i -e 's/taosd<\/th>/prodbs<\/th>/g' ${install_dir}/nginxd/admin/monitor.html + sed -i -e "s/data:\['taosd', 'system'\],/data:\['prodbs', 'system'\],/g" ${install_dir}/nginxd/admin/monitor.html + sed -i -e "s/name: 'taosd',/name: 'prodbs',/g" ${install_dir}/nginxd/admin/monitor.html + sed -i "s/TDengine/ProDB/g" ${install_dir}/nginxd/admin/*.html sed -i "s/TDengine/ProDB/g" ${install_dir}/nginxd/admin/js/*.js diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 80a538b1a79dd6870b95d2a746050bc366606fa3..549f127b39383ac41e92f416c37681c8188edfda 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.2.0.2' +version: '2.2.0.5' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.2.0.2 + - usr/lib/libtaos.so.2.2.0.5 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index fdcfcb02de470a23afd552be81292c31572fcc55..125e28f17778baead6b2be8d8409f2259d7d77e5 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -771,6 +771,10 @@ void tscSortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) { TSKEY tj = *(TSKEY *)(pBlockData + dataBuf->rowSize * j); if (ti == tj) { + if (dataBuf->pTableMeta && dataBuf->pTableMeta->tableInfo.update != TD_ROW_DISCARD_UPDATE) { + memmove(pBlockData + dataBuf->rowSize * i, pBlockData + dataBuf->rowSize * j, dataBuf->rowSize); + } + ++j; continue; } @@ -841,6 +845,10 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk TSKEY tj = (pBlkKeyTuple + j)->skey; if (ti == tj) { + if (dataBuf->pTableMeta && dataBuf->pTableMeta->tableInfo.update != TD_ROW_DISCARD_UPDATE) { + memmove(pBlkKeyTuple + i, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); + } + ++j; continue; } diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 1bad1c72409baa9921a096e500c71a2e1a172ad4..8f1f21842015cecfdd71486673cba2dcc1dd73a5 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -1002,6 +1002,7 @@ static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, size_t rows = taosArrayGetSize(cTablePoints); SArray* rowsBind = taosArrayInit(rows, POINTER_BYTES); + int isNullColBind = TSDB_TRUE; for (int i = 0; i < rows; ++i) { TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, i); @@ -1012,7 +1013,6 @@ static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema, return TSDB_CODE_TSC_OUT_OF_MEMORY; } - int isNullColBind = TSDB_TRUE; for (int j = 0; j < numCols; ++j) { TAOS_BIND* bind = colBinds + j; bind->is_null = &isNullColBind; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 5b239d3bfb2d36264bb923935eabe2229186125c..ae5bee879ff1abd6950cc0d4c7c52e3cc2903e62 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3011,7 +3011,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg9); } - if (taosArrayGetSize(pItem->pNode->Expr.paramList) <= 0) { + if (pItem->pNode->Expr.paramList == NULL || taosArrayGetSize(pItem->pNode->Expr.paramList) <= 0) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg13); } @@ -3672,7 +3672,7 @@ int32_t validateGroupbyNode(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pSchema); } else { // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by - if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) { + if (pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index ac91d845f778bcb5672edbde677b88e8c50e7d33..2b16ffc011f875ba716e60de207c26bfad3fcaec 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -4400,6 +4400,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { pTableMeta->tableInfo.numOfTags = pChild->numOfTags; pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns; pTableMeta->tableInfo.precision = pChild->precision; + pTableMeta->tableInfo.update = pChild->update; pTableMeta->id.tid = 0; pTableMeta->id.uid = pChild->suid; diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index 32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df..016d8e82a24d72779be0ab0090580a372b4fffca 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit 32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df +Subproject commit 016d8e82a24d72779be0ab0090580a372b4fffca diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index fe553112cc969c9fdccc18b523cc4f65c2b0845f..dd4374761a6c90aa97b822ba8e814781b8ef0c7c 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -779,6 +779,7 @@ typedef struct STableMetaMsg { char tableFname[TSDB_TABLE_FNAME_LEN]; // table id uint8_t numOfTags; uint8_t precision; + uint8_t update; uint8_t tableType; int16_t numOfColumns; int16_t sversion; diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index bf2bbca14d25aff3b3717c7b9785f1dc470a013a..747b1bf28ddf24869dba4db261921f869bedd5e1 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -34,6 +34,8 @@ ELSEIF (TD_WINDOWS) IF (TD_POWER) SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME power) + ELSEIF (TD_PRO) + SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME prodbc) ELSE () SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ENDIF () diff --git a/src/kit/shell/src/shellCheck.c b/src/kit/shell/src/shellCheck.c index 7fc8b1409a7602df48108d0e7f4763da48ed6497..5821281a036674e7a60edc2f63500822a358b1bc 100644 --- a/src/kit/shell/src/shellCheck.c +++ b/src/kit/shell/src/shellCheck.c @@ -111,6 +111,7 @@ static void *shellCheckThreadFp(void *arg) { int32_t start = pThread->threadIndex * interval; int32_t end = (pThread->threadIndex + 1) * interval; + if (start >= tbNum) return NULL; if (end > tbNum) end = tbNum + 1; char file[32] = {0}; @@ -193,9 +194,11 @@ void shellCheck(TAOS *con, SShellArguments *_args) { return; } - fprintf(stdout, "total %d tables will be checked by %d threads\n", tbNum, _args->threadNum); - shellRunCheckThreads(con, _args); - + if (tbNum > 0) { + fprintf(stdout, "total %d tables will be checked by %d threads\n", tbNum, _args->threadNum); + shellRunCheckThreads(con, _args); + } + int64_t end = taosGetTimestampMs(); fprintf(stdout, "total %d tables checked, failed:%d, time spent %.2f seconds\n", checkedNum, errorNum, (end - start) / 1000.0); diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 5adf9f342a41f9b3886c9c9654e0ef9dd7571c54..060f856afe87f332720db30f584ca73be498211d 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -79,10 +79,10 @@ extern char configDir[]; #define DEFAULT_START_TIME 1500000000000 #define MAX_PREPARED_RAND 1000000 -#define INT_BUFF_LEN 11 +#define INT_BUFF_LEN 12 #define BIGINT_BUFF_LEN 21 -#define SMALLINT_BUFF_LEN 6 -#define TINYINT_BUFF_LEN 4 +#define SMALLINT_BUFF_LEN 7 +#define TINYINT_BUFF_LEN 5 #define BOOL_BUFF_LEN 6 #define FLOAT_BUFF_LEN 22 #define DOUBLE_BUFF_LEN 42 @@ -586,20 +586,27 @@ static int64_t getTSRandTail(int64_t timeStampStep, int32_t seq, int disorderRatio, int disorderRange); static bool getInfoFromJsonFile(char* file); static void init_rand_data(); +static int regexMatch(const char *s, const char *reg, int cflags); /* ************ Global variables ************ */ int32_t g_randint[MAX_PREPARED_RAND]; +uint32_t g_randuint[MAX_PREPARED_RAND]; int64_t g_randbigint[MAX_PREPARED_RAND]; +uint64_t g_randubigint[MAX_PREPARED_RAND]; float g_randfloat[MAX_PREPARED_RAND]; double g_randdouble[MAX_PREPARED_RAND]; char *g_randbool_buff = NULL; char *g_randint_buff = NULL; +char *g_randuint_buff = NULL; char *g_rand_voltage_buff = NULL; char *g_randbigint_buff = NULL; +char *g_randubigint_buff = NULL; char *g_randsmallint_buff = NULL; +char *g_randusmallint_buff = NULL; char *g_randtinyint_buff = NULL; +char *g_randutinyint_buff = NULL; char *g_randfloat_buff = NULL; char *g_rand_current_buff = NULL; char *g_rand_phase_buff = NULL; @@ -797,7 +804,7 @@ static void printHelp() { printf("%s%s%s%s\n", indent, "-q, --query-mode=MODE", "\t\t", "Query mode -- 0: SYNC, 1: ASYNC. By default use SYNC."); printf("%s%s%s%s\n", indent, "-b, --data-type=DATATYPE", "\t", - "The data_type of columns, By default use: FLOAT, INT, FLOAT."); + "The data_type of columns, By default use: FLOAT,INT,FLOAT. NCHAR and BINARY can also use custom length. Eg: NCHAR(16),BINARY(8)"); printf("%s%s%s%s%d\n", indent, "-w, --binwidth=WIDTH", "\t\t", "The width of data_type 'BINARY' or 'NCHAR'. By default use ", g_args.binwidth); @@ -979,36 +986,55 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->performance_print = true; } else if ((0 == strncmp(argv[i], "-P", strlen("-P"))) || (0 == strncmp(argv[i], "--port", strlen("--port")))) { + uint64_t port; + char strPort[BIGINT_BUFF_LEN]; + if (2 == strlen(argv[i])) { if (argc == i+1) { errorPrintReqArg(argv[0], "P"); exit(EXIT_FAILURE); - } else if (!isStringNumber(argv[i+1])) { + } else if (isStringNumber(argv[i+1])) { + tstrncpy(strPort, argv[++i], BIGINT_BUFF_LEN); + } else { errorPrintReqArg2(argv[0], "P"); exit(EXIT_FAILURE); } - arguments->port = atoi(argv[++i]); } else if (0 == strncmp(argv[i], "--port=", strlen("--port="))) { if (isStringNumber((char *)(argv[i] + strlen("--port=")))) { - arguments->port = atoi((char *)(argv[i]+strlen("--port="))); + tstrncpy(strPort, (char *)(argv[i]+strlen("--port=")), BIGINT_BUFF_LEN); + } else { + errorPrintReqArg2(argv[0], "--port"); + exit(EXIT_FAILURE); } } else if (0 == strncmp(argv[i], "-P", strlen("-P"))) { if (isStringNumber((char *)(argv[i] + strlen("-P")))) { - arguments->port = atoi((char *)(argv[i]+strlen("-P"))); + tstrncpy(strPort, (char *)(argv[i]+strlen("-P")), BIGINT_BUFF_LEN); + } else { + errorPrintReqArg2(argv[0], "--port"); + exit(EXIT_FAILURE); } } else if (strlen("--port") == strlen(argv[i])) { if (argc == i+1) { errorPrintReqArg3(argv[0], "--port"); exit(EXIT_FAILURE); - } else if (!isStringNumber(argv[i+1])) { + } else if (isStringNumber(argv[i+1])) { + tstrncpy(strPort, argv[++i], BIGINT_BUFF_LEN); + } else { errorPrintReqArg2(argv[0], "--port"); exit(EXIT_FAILURE); } - arguments->port = atoi(argv[++i]); } else { errorUnrecognized(argv[0], argv[i]); exit(EXIT_FAILURE); } + + port = atoi(strPort); + if (port > 65535) { + errorWrongValue("taosdump", "-P or --port", strPort); + exit(EXIT_FAILURE); + } + arguments->port = (uint16_t)port; + } else if ((0 == strncmp(argv[i], "-I", strlen("-I"))) || (0 == strncmp(argv[i], "--interface", strlen("--interface")))) { if (2 == strlen(argv[i])) { @@ -1573,9 +1599,14 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { && strcasecmp(dataType, "SMALLINT") && strcasecmp(dataType, "BIGINT") && strcasecmp(dataType, "DOUBLE") - && strcasecmp(dataType, "BINARY") && strcasecmp(dataType, "TIMESTAMP") - && strcasecmp(dataType, "NCHAR")) { + && !regexMatch(dataType, + "^(NCHAR|BINARY)(\\([1-9][0-9]*\\))?$", + REG_ICASE | REG_EXTENDED) + && strcasecmp(dataType, "UTINYINT") + && strcasecmp(dataType, "USMALLINT") + && strcasecmp(dataType, "UINT") + && strcasecmp(dataType, "UBIGINT")) { printHelp(); errorPrint("%s", "-b: Invalid data_type!\n"); exit(EXIT_FAILURE); @@ -1593,14 +1624,26 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->data_type[0] = TSDB_DATA_TYPE_FLOAT; } else if (0 == strcasecmp(dataType, "DOUBLE")) { arguments->data_type[0] = TSDB_DATA_TYPE_DOUBLE; - } else if (0 == strcasecmp(dataType, "BINARY")) { + } else if (1 == regexMatch(dataType, + "^BINARY(\\([1-9][0-9]*\\))?$", + REG_ICASE | REG_EXTENDED)) { arguments->data_type[0] = TSDB_DATA_TYPE_BINARY; - } else if (0 == strcasecmp(dataType, "NCHAR")) { + } else if (1 == regexMatch(dataType, + "^NCHAR(\\([1-9][0-9]*\\))?$", + REG_ICASE | REG_EXTENDED)) { arguments->data_type[0] = TSDB_DATA_TYPE_NCHAR; } else if (0 == strcasecmp(dataType, "BOOL")) { arguments->data_type[0] = TSDB_DATA_TYPE_BOOL; } else if (0 == strcasecmp(dataType, "TIMESTAMP")) { arguments->data_type[0] = TSDB_DATA_TYPE_TIMESTAMP; + } else if (0 == strcasecmp(dataType, "UTINYINT")) { + arguments->data_type[0] = TSDB_DATA_TYPE_UTINYINT; + } else if (0 == strcasecmp(dataType, "USMALLINT")) { + arguments->data_type[0] = TSDB_DATA_TYPE_USMALLINT; + } else if (0 == strcasecmp(dataType, "UINT")) { + arguments->data_type[0] = TSDB_DATA_TYPE_UINT; + } else if (0 == strcasecmp(dataType, "UBIGINT")) { + arguments->data_type[0] = TSDB_DATA_TYPE_UBIGINT; } else { arguments->data_type[0] = TSDB_DATA_TYPE_NULL; } @@ -1620,9 +1663,12 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { && strcasecmp(token, "SMALLINT") && strcasecmp(token, "BIGINT") && strcasecmp(token, "DOUBLE") - && strcasecmp(token, "BINARY") && strcasecmp(token, "TIMESTAMP") - && strcasecmp(token, "NCHAR")) { + && !regexMatch(token, "^(NCHAR|BINARY)(\\([1-9][0-9]*\\))?$", REG_ICASE | REG_EXTENDED) + && strcasecmp(token, "UTINYINT") + && strcasecmp(token, "USMALLINT") + && strcasecmp(token, "UINT") + && strcasecmp(token, "UBIGINT")) { printHelp(); free(g_dupstr); errorPrint("%s", "-b: Invalid data_type!\n"); @@ -1638,17 +1684,27 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } else if (0 == strcasecmp(token, "BIGINT")) { arguments->data_type[index] = TSDB_DATA_TYPE_BIGINT; } else if (0 == strcasecmp(token, "DOUBLE")) { - arguments->data_type[index] = TSDB_DATA_TYPE_FLOAT; + arguments->data_type[index] = TSDB_DATA_TYPE_DOUBLE; } else if (0 == strcasecmp(token, "TINYINT")) { arguments->data_type[index] = TSDB_DATA_TYPE_TINYINT; - } else if (0 == strcasecmp(token, "BINARY")) { + } else if (1 == regexMatch(token, "^BINARY(\\([1-9][0-9]*\\))?$", REG_ICASE | + REG_EXTENDED)) { arguments->data_type[index] = TSDB_DATA_TYPE_BINARY; - } else if (0 == strcasecmp(token, "NCHAR")) { + } else if (1 == regexMatch(token, "^NCHAR(\\([1-9][0-9]*\\))?$", REG_ICASE | + REG_EXTENDED)) { arguments->data_type[index] = TSDB_DATA_TYPE_NCHAR; } else if (0 == strcasecmp(token, "BOOL")) { arguments->data_type[index] = TSDB_DATA_TYPE_BOOL; } else if (0 == strcasecmp(token, "TIMESTAMP")) { arguments->data_type[index] = TSDB_DATA_TYPE_TIMESTAMP; + } else if (0 == strcasecmp(token, "UTINYINT")) { + arguments->data_type[index] = TSDB_DATA_TYPE_UTINYINT; + } else if (0 == strcasecmp(token, "USMALLINT")) { + arguments->data_type[index] = TSDB_DATA_TYPE_USMALLINT; + } else if (0 == strcasecmp(token, "UINT")) { + arguments->data_type[index] = TSDB_DATA_TYPE_UINT; + } else if (0 == strcasecmp(token, "UBIGINT")) { + arguments->data_type[index] = TSDB_DATA_TYPE_UBIGINT; } else { arguments->data_type[index] = TSDB_DATA_TYPE_NULL; } @@ -1952,18 +2008,22 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { break; case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: g_args.lenOfOneRow += INT_BUFF_LEN; break; case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: g_args.lenOfOneRow += BIGINT_BUFF_LEN; break; case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: g_args.lenOfOneRow += SMALLINT_BUFF_LEN; break; case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: g_args.lenOfOneRow += TINYINT_BUFF_LEN; break; @@ -2195,6 +2255,23 @@ static int32_t rand_tinyint() return g_randint[cursor % MAX_PREPARED_RAND] % 128; } +static char *rand_utinyint_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randutinyint_buff + + ((cursor % MAX_PREPARED_RAND) * TINYINT_BUFF_LEN); +} + +static int32_t rand_utinyint() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randuint[cursor % MAX_PREPARED_RAND] % 255; +} + static char *rand_smallint_str() { static int cursor; @@ -2209,7 +2286,24 @@ static int32_t rand_smallint() static int cursor; cursor++; if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; - return g_randint[cursor % MAX_PREPARED_RAND] % 32767; + return g_randint[cursor % MAX_PREPARED_RAND] % 32768; +} + +static char *rand_usmallint_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randusmallint_buff + + ((cursor % MAX_PREPARED_RAND) * SMALLINT_BUFF_LEN); +} + +static int32_t rand_usmallint() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randuint[cursor % MAX_PREPARED_RAND] % 65535; } static char *rand_int_str() @@ -2228,6 +2322,22 @@ static int32_t rand_int() return g_randint[cursor % MAX_PREPARED_RAND]; } +static char *rand_uint_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randuint_buff + ((cursor % MAX_PREPARED_RAND) * INT_BUFF_LEN); +} + +static int32_t rand_uint() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randuint[cursor % MAX_PREPARED_RAND]; +} + static char *rand_bigint_str() { static int cursor; @@ -2245,6 +2355,23 @@ static int64_t rand_bigint() return g_randbigint[cursor % MAX_PREPARED_RAND]; } +static char *rand_ubigint_str() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randubigint_buff + + ((cursor % MAX_PREPARED_RAND) * BIGINT_BUFF_LEN); +} + +static int64_t rand_ubigint() +{ + static int cursor; + cursor++; + if (cursor > (MAX_PREPARED_RAND - 1)) cursor = 0; + return g_randubigint[cursor % MAX_PREPARED_RAND]; +} + static char *rand_float_str() { static int cursor; @@ -2382,9 +2509,18 @@ static void init_rand_data() { assert(g_rand_phase_buff); g_randdouble_buff = calloc(1, DOUBLE_BUFF_LEN * MAX_PREPARED_RAND); assert(g_randdouble_buff); + g_randuint_buff = calloc(1, INT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randuint_buff); + g_randutinyint_buff = calloc(1, TINYINT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randutinyint_buff); + g_randusmallint_buff = calloc(1, SMALLINT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randusmallint_buff); + g_randubigint_buff = calloc(1, BIGINT_BUFF_LEN * MAX_PREPARED_RAND); + assert(g_randubigint_buff); for (int i = 0; i < MAX_PREPARED_RAND; i++) { - g_randint[i] = (int)(taosRandom() % 65535); + g_randint[i] = (int)(taosRandom() % RAND_MAX - (RAND_MAX >> 1)); + g_randuint[i] = (int)(taosRandom()); sprintf(g_randint_buff + i * INT_BUFF_LEN, "%d", g_randint[i]); sprintf(g_rand_voltage_buff + i * INT_BUFF_LEN, "%d", @@ -2393,15 +2529,24 @@ static void init_rand_data() { sprintf(g_randbool_buff + i * BOOL_BUFF_LEN, "%s", ((g_randint[i] % 2) & 1)?"true":"false"); sprintf(g_randsmallint_buff + i * SMALLINT_BUFF_LEN, "%d", - g_randint[i] % 32767); + g_randint[i] % 32768); sprintf(g_randtinyint_buff + i * TINYINT_BUFF_LEN, "%d", g_randint[i] % 128); - - g_randbigint[i] = (int64_t)(taosRandom() % 2147483648); + sprintf(g_randuint_buff + i * INT_BUFF_LEN, "%d", + g_randuint[i]); + sprintf(g_randusmallint_buff + i * SMALLINT_BUFF_LEN, "%d", + g_randuint[i] % 65535); + sprintf(g_randutinyint_buff + i * TINYINT_BUFF_LEN, "%d", + g_randuint[i] % 255); + + g_randbigint[i] = (int64_t)(taosRandom() % RAND_MAX - (RAND_MAX >> 1)); + g_randubigint[i] = (uint64_t)(taosRandom()); sprintf(g_randbigint_buff + i * BIGINT_BUFF_LEN, "%"PRId64"", g_randbigint[i]); + sprintf(g_randubigint_buff + i * BIGINT_BUFF_LEN, "%"PRId64"", + g_randubigint[i]); - g_randfloat[i] = (float)(taosRandom() / 1000.0); + g_randfloat[i] = (float)(taosRandom() / 1000.0) * (taosRandom() % 2 > 0.5 ? 1 : -1); sprintf(g_randfloat_buff + i * FLOAT_BUFF_LEN, "%f", g_randfloat[i]); sprintf(g_rand_current_buff + i * FLOAT_BUFF_LEN, "%f", @@ -2411,7 +2556,7 @@ static void init_rand_data() { (float)((115 + g_randint[i] % 10 + g_randfloat[i]/1000000000)/360)); - g_randdouble[i] = (double)(taosRandom() / 1000000.0); + g_randdouble[i] = (double)(taosRandom() / 1000000.0) * (taosRandom() % 2 > 0.5 ? 1 : -1); sprintf(g_randdouble_buff + i * DOUBLE_BUFF_LEN, "%f", g_randdouble[i]); } @@ -2541,117 +2686,127 @@ static int printfInsertMeta() { } } - printf(" super table count: \033[33m%"PRIu64"\033[0m\n", - g_Dbs.db[i].superTblCount); - for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { - printf(" super table[\033[33m%"PRIu64"\033[0m]:\n", j); - - printf(" stbName: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].stbName); - if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { - printf(" autoCreateTable: \033[33m%s\033[0m\n", "no"); - } else if (AUTO_CREATE_SUBTBL == - g_Dbs.db[i].superTbls[j].autoCreateTable) { - printf(" autoCreateTable: \033[33m%s\033[0m\n", "yes"); - } else { - printf(" autoCreateTable: \033[33m%s\033[0m\n", "error"); - } + if (g_args.use_metric) { + printf(" super table count: \033[33m%"PRIu64"\033[0m\n", + g_Dbs.db[i].superTblCount); + for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { + printf(" super table[\033[33m%"PRIu64"\033[0m]:\n", j); - if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { - printf(" childTblExists: \033[33m%s\033[0m\n", "no"); - } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { - printf(" childTblExists: \033[33m%s\033[0m\n", "yes"); - } else { - printf(" childTblExists: \033[33m%s\033[0m\n", "error"); - } + printf(" stbName: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].stbName); - printf(" childTblCount: \033[33m%"PRId64"\033[0m\n", - g_Dbs.db[i].superTbls[j].childTblCount); - printf(" childTblPrefix: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].childTblPrefix); - printf(" dataSource: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].dataSource); - printf(" iface: \033[33m%s\033[0m\n", - (g_Dbs.db[i].superTbls[j].iface==TAOSC_IFACE)?"taosc": - (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest":"stmt"); - if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) { - printf(" childTblLimit: \033[33m%"PRId64"\033[0m\n", - g_Dbs.db[i].superTbls[j].childTblLimit); - } - if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) { - printf(" childTblOffset: \033[33m%"PRIu64"\033[0m\n", - g_Dbs.db[i].superTbls[j].childTblOffset); - } - printf(" insertRows: \033[33m%"PRId64"\033[0m\n", - g_Dbs.db[i].superTbls[j].insertRows); - /* - if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { - printf(" multiThreadWriteOneTbl: \033[33m no\033[0m\n"); - }else { - printf(" multiThreadWriteOneTbl: \033[33m yes\033[0m\n"); - } - */ - printf(" interlaceRows: \033[33m%u\033[0m\n", - g_Dbs.db[i].superTbls[j].interlaceRows); - - if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) { - printf(" stable insert interval: \033[33m%"PRIu64"\033[0m\n", - g_Dbs.db[i].superTbls[j].insertInterval); - } - - printf(" disorderRange: \033[33m%d\033[0m\n", - g_Dbs.db[i].superTbls[j].disorderRange); - printf(" disorderRatio: \033[33m%d\033[0m\n", - g_Dbs.db[i].superTbls[j].disorderRatio); - printf(" maxSqlLen: \033[33m%"PRIu64"\033[0m\n", - g_Dbs.db[i].superTbls[j].maxSqlLen); - printf(" timeStampStep: \033[33m%"PRId64"\033[0m\n", - g_Dbs.db[i].superTbls[j].timeStampStep); - printf(" startTimestamp: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].startTimestamp); - printf(" sampleFormat: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].sampleFormat); - printf(" sampleFile: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].sampleFile); - printf(" tagsFile: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].tagsFile); - printf(" columnCount: \033[33m%d\033[0m\n", - g_Dbs.db[i].superTbls[j].columnCount); - for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { - //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); - if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, - "binary", 6)) - || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, - "nchar", 5))) { - printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k, - g_Dbs.db[i].superTbls[j].columns[k].dataType, - g_Dbs.db[i].superTbls[j].columns[k].dataLen); + if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "no"); + } else if (AUTO_CREATE_SUBTBL == + g_Dbs.db[i].superTbls[j].autoCreateTable) { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "yes"); } else { - printf("column[%d]:\033[33m%s\033[0m ", k, - g_Dbs.db[i].superTbls[j].columns[k].dataType); + printf(" autoCreateTable: \033[33m%s\033[0m\n", "error"); } - } - printf("\n"); - printf(" tagCount: \033[33m%d\033[0m\n ", - g_Dbs.db[i].superTbls[j].tagCount); - for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { - //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); - if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, - "binary", strlen("binary"))) - || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, - "nchar", strlen("nchar")))) { - printf("tag[%d]:\033[33m%s(%d)\033[0m ", k, - g_Dbs.db[i].superTbls[j].tags[k].dataType, - g_Dbs.db[i].superTbls[j].tags[k].dataLen); + if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + printf(" childTblExists: \033[33m%s\033[0m\n", "no"); + } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + printf(" childTblExists: \033[33m%s\033[0m\n", "yes"); } else { - printf("tag[%d]:\033[33m%s\033[0m ", k, - g_Dbs.db[i].superTbls[j].tags[k].dataType); + printf(" childTblExists: \033[33m%s\033[0m\n", "error"); + } + + printf(" childTblCount: \033[33m%"PRId64"\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblCount); + printf(" childTblPrefix: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblPrefix); + printf(" dataSource: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].dataSource); + printf(" iface: \033[33m%s\033[0m\n", + (g_Dbs.db[i].superTbls[j].iface==TAOSC_IFACE)?"taosc": + (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest":"stmt"); + if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) { + printf(" childTblLimit: \033[33m%"PRId64"\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblLimit); + } + if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) { + printf(" childTblOffset: \033[33m%"PRIu64"\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblOffset); + } + printf(" insertRows: \033[33m%"PRId64"\033[0m\n", + g_Dbs.db[i].superTbls[j].insertRows); + /* + if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { + printf(" multiThreadWriteOneTbl: \033[33m no\033[0m\n"); + }else { + printf(" multiThreadWriteOneTbl: \033[33m yes\033[0m\n"); + } + */ + printf(" interlaceRows: \033[33m%u\033[0m\n", + g_Dbs.db[i].superTbls[j].interlaceRows); + + if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) { + printf(" stable insert interval: \033[33m%"PRIu64"\033[0m\n", + g_Dbs.db[i].superTbls[j].insertInterval); + } + + printf(" disorderRange: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].disorderRange); + printf(" disorderRatio: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].disorderRatio); + printf(" maxSqlLen: \033[33m%"PRIu64"\033[0m\n", + g_Dbs.db[i].superTbls[j].maxSqlLen); + printf(" timeStampStep: \033[33m%"PRId64"\033[0m\n", + g_Dbs.db[i].superTbls[j].timeStampStep); + printf(" startTimestamp: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].startTimestamp); + printf(" sampleFormat: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].sampleFormat); + printf(" sampleFile: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].sampleFile); + printf(" tagsFile: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].tagsFile); + printf(" columnCount: \033[33m%d\033[0m\n ", + g_Dbs.db[i].superTbls[j].columnCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, + "binary", 6)) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, + "nchar", 5))) { + printf("column[%d]:\033[33m%s(%d)\033[0m ", k, + g_Dbs.db[i].superTbls[j].columns[k].dataType, + g_Dbs.db[i].superTbls[j].columns[k].dataLen); + } else { + printf("column[%d]:\033[33m%s\033[0m ", k, + g_Dbs.db[i].superTbls[j].columns[k].dataType); + } + } + printf("\n"); + + printf(" tagCount: \033[33m%d\033[0m\n ", + g_Dbs.db[i].superTbls[j].tagCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, + "binary", strlen("binary"))) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, + "nchar", strlen("nchar")))) { + printf("tag[%d]:\033[33m%s(%d)\033[0m ", k, + g_Dbs.db[i].superTbls[j].tags[k].dataType, + g_Dbs.db[i].superTbls[j].tags[k].dataLen); + } else { + printf("tag[%d]:\033[33m%s\033[0m ", k, + g_Dbs.db[i].superTbls[j].tags[k].dataType); + } } + printf("\n"); } - printf("\n"); + } else { + printf(" childTblCount: \033[33m%"PRId64"\033[0m\n", + g_args.ntables); + printf(" insertRows: \033[33m%"PRId64"\033[0m\n", + g_args.insertRows); } + + printf("\n"); } @@ -2974,18 +3129,34 @@ static void xDumpFieldToFile(FILE* fp, const char* val, fprintf(fp, "%d", *((int8_t *)val)); break; + case TSDB_DATA_TYPE_UTINYINT: + fprintf(fp, "%d", *((uint8_t *)val)); + break; + case TSDB_DATA_TYPE_SMALLINT: fprintf(fp, "%d", *((int16_t *)val)); break; + case TSDB_DATA_TYPE_USMALLINT: + fprintf(fp, "%d", *((uint16_t *)val)); + break; + case TSDB_DATA_TYPE_INT: fprintf(fp, "%d", *((int32_t *)val)); break; + case TSDB_DATA_TYPE_UINT: + fprintf(fp, "%d", *((uint32_t *)val)); + break; + case TSDB_DATA_TYPE_BIGINT: fprintf(fp, "%"PRId64"", *((int64_t *)val)); break; + case TSDB_DATA_TYPE_UBIGINT: + fprintf(fp, "%"PRId64"", *((uint64_t *)val)); + break; + case TSDB_DATA_TYPE_FLOAT: fprintf(fp, "%.5f", GET_FLOAT_VAL(val)); break; @@ -3472,7 +3643,23 @@ static char* generateTagValuesForStb(SSuperTable* stbInfo, int64_t tableSeq) { } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "timestamp", strlen("timestamp"))) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%"PRId64",", rand_bigint()); + "%"PRId64",", rand_ubigint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "utinyint", strlen("utinyint"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%d,", rand_utinyint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "usmallint", strlen("usmallint"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%d,", rand_usmallint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "uint", strlen("uint"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%d,", rand_uint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "ubigint", strlen("ubigint"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%"PRId64",", rand_ubigint()); } else { errorPrint2("No support data type: %s\n", stbInfo->tags[i].dataType); tmfree(dataBuf); @@ -3502,18 +3689,22 @@ static int calcRowLen(SSuperTable* superTbls) { break; case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: lenOfOneRow += INT_BUFF_LEN; break; case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: lenOfOneRow += BIGINT_BUFF_LEN; break; case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: lenOfOneRow += SMALLINT_BUFF_LEN; break; case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: lenOfOneRow += TINYINT_BUFF_LEN; break; @@ -3544,27 +3735,41 @@ static int calcRowLen(SSuperTable* superTbls) { int tagIndex; int lenOfTagOfOneRow = 0; for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { - char* dataType = superTbls->tags[tagIndex].dataType; - - if (strcasecmp(dataType, "BINARY") == 0) { + char * dataType = superTbls->tags[tagIndex].dataType; + switch (superTbls->tags[tagIndex].data_type) + { + case TSDB_DATA_TYPE_BINARY: lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; - } else if (strcasecmp(dataType, "NCHAR") == 0) { + break; + case TSDB_DATA_TYPE_NCHAR: lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; - } else if (strcasecmp(dataType, "INT") == 0) { + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + INT_BUFF_LEN; - } else if (strcasecmp(dataType, "BIGINT") == 0) { + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + BIGINT_BUFF_LEN; - } else if (strcasecmp(dataType, "SMALLINT") == 0) { + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + SMALLINT_BUFF_LEN; - } else if (strcasecmp(dataType, "TINYINT") == 0) { + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + TINYINT_BUFF_LEN; - } else if (strcasecmp(dataType, "BOOL") == 0) { + break; + case TSDB_DATA_TYPE_BOOL: lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + BOOL_BUFF_LEN; - } else if (strcasecmp(dataType, "FLOAT") == 0) { + break; + case TSDB_DATA_TYPE_FLOAT: lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + FLOAT_BUFF_LEN; - } else if (strcasecmp(dataType, "DOUBLE") == 0) { + break; + case TSDB_DATA_TYPE_DOUBLE: lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + DOUBLE_BUFF_LEN; - } else { + break; + default: errorPrint2("get error tag type : %s\n", dataType); exit(EXIT_FAILURE); } @@ -3697,40 +3902,60 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, tstrncpy(superTbls->tags[tagIndex].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); - tstrncpy(superTbls->tags[tagIndex].dataType, - (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - min(DATATYPE_BUFF_LEN, - fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); - if (0 == strncasecmp(superTbls->tags[tagIndex].dataType, + if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "INT", strlen("INT"))) { superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_INT; - } else if (0 == strncasecmp(superTbls->tags[tagIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "TINYINT", strlen("TINYINT"))) { superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_TINYINT; - } else if (0 == strncasecmp(superTbls->tags[tagIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "SMALLINT", strlen("SMALLINT"))) { superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_SMALLINT; - } else if (0 == strncasecmp(superTbls->tags[tagIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "BIGINT", strlen("BIGINT"))) { superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_BIGINT; - } else if (0 == strncasecmp(superTbls->tags[tagIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "FLOAT", strlen("FLOAT"))) { superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_FLOAT; - } else if (0 == strncasecmp(superTbls->tags[tagIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "DOUBLE", strlen("DOUBLE"))) { superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_DOUBLE; - } else if (0 == strncasecmp(superTbls->tags[tagIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "BINARY", strlen("BINARY"))) { superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_BINARY; - } else if (0 == strncasecmp(superTbls->tags[tagIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "NCHAR", strlen("NCHAR"))) { superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_NCHAR; - } else if (0 == strncasecmp(superTbls->tags[tagIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "BOOL", strlen("BOOL"))) { superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_BOOL; - } else if (0 == strncasecmp(superTbls->tags[tagIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "TIMESTAMP", strlen("TIMESTAMP"))) { superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_TIMESTAMP; + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + "TINYINT UNSIGNED", strlen("TINYINT UNSIGNED"))) { + superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_UTINYINT; + tstrncpy(superTbls->tags[tagIndex].dataType,"UTINYINT", + min(DATATYPE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + "SMALLINT UNSIGNED", strlen("SMALLINT UNSIGNED"))) { + superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_USMALLINT; + tstrncpy(superTbls->tags[tagIndex].dataType,"USMALLINT", + min(DATATYPE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + "INT UNSIGNED", strlen("INT UNSIGNED"))) { + superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_UINT; + tstrncpy(superTbls->tags[tagIndex].dataType,"UINT", + min(DATATYPE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); + }else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + "BIGINT UNSIGNED", strlen("BIGINT UNSIGNED"))) { + superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_UBIGINT; + tstrncpy(superTbls->tags[tagIndex].dataType,"UBIGINT", + min(DATATYPE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); } else { superTbls->tags[tagIndex].data_type = TSDB_DATA_TYPE_NULL; } @@ -3740,46 +3965,78 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], min(NOTE_BUFF_LEN, fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes) + 1); + if (strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) + { + tstrncpy(superTbls->tags[tagIndex].dataType, + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + min(DATATYPE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); + } tagIndex++; } else { tstrncpy(superTbls->columns[columnIndex].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); - tstrncpy(superTbls->columns[columnIndex].dataType, - (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - min(DATATYPE_BUFF_LEN, - fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); - if (0 == strncasecmp(superTbls->columns[columnIndex].dataType, - "INT", strlen("INT"))) { + + if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + "INT", strlen("INT")) && + strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_INT; - } else if (0 == strncasecmp(superTbls->columns[columnIndex].dataType, - "TINYINT", strlen("TINYINT"))) { + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + "TINYINT", strlen("TINYINT")) && + strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_TINYINT; - } else if (0 == strncasecmp(superTbls->columns[columnIndex].dataType, - "SMALLINT", strlen("SMALLINT"))) { + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + "SMALLINT", strlen("SMALLINT")) && + strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_SMALLINT; - } else if (0 == strncasecmp(superTbls->columns[columnIndex].dataType, - "BIGINT", strlen("BIGINT"))) { + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + "BIGINT", strlen("BIGINT")) && + strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_BIGINT; - } else if (0 == strncasecmp(superTbls->columns[columnIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "FLOAT", strlen("FLOAT"))) { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_FLOAT; - } else if (0 == strncasecmp(superTbls->columns[columnIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "DOUBLE", strlen("DOUBLE"))) { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_DOUBLE; - } else if (0 == strncasecmp(superTbls->columns[columnIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "BINARY", strlen("BINARY"))) { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_BINARY; - } else if (0 == strncasecmp(superTbls->columns[columnIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "NCHAR", strlen("NCHAR"))) { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_NCHAR; - } else if (0 == strncasecmp(superTbls->columns[columnIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "BOOL", strlen("BOOL"))) { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_BOOL; - } else if (0 == strncasecmp(superTbls->columns[columnIndex].dataType, + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "TIMESTAMP", strlen("TIMESTAMP"))) { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_TIMESTAMP; + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + "TINYINT UNSIGNED", strlen("TINYINT UNSIGNED"))) { + superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_UTINYINT; + tstrncpy(superTbls->columns[columnIndex].dataType,"UTINYINT", + min(DATATYPE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + "SMALLINT UNSIGNED", strlen("SMALLINT UNSIGNED"))) { + superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_USMALLINT; + tstrncpy(superTbls->columns[columnIndex].dataType,"USMALLINT", + min(DATATYPE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + "INT UNSIGNED", strlen("INT UNSIGNED"))) { + superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_UINT; + tstrncpy(superTbls->columns[columnIndex].dataType,"UINT", + min(DATATYPE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); + } else if (0 == strncasecmp((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + "BIGINT UNSIGNED", strlen("BIGINT UNSIGNED"))) { + superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_UBIGINT; + tstrncpy(superTbls->columns[columnIndex].dataType,"UBIGINT", + min(DATATYPE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); } else { superTbls->columns[columnIndex].data_type = TSDB_DATA_TYPE_NULL; } @@ -3789,6 +4046,13 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], min(NOTE_BUFF_LEN, fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes) + 1); + + if (strstr((char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], "UNSIGNED") == NULL) { + tstrncpy(superTbls->columns[columnIndex].dataType, + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + min(DATATYPE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); + } columnIndex++; } @@ -3913,6 +4177,30 @@ static int createSuperTable( lenOfOneRow += TIMESTAMP_BUFF_LEN; break; + case TSDB_DATA_TYPE_UTINYINT: + len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", + colIndex, "TINYINT UNSIGNED"); + lenOfOneRow += TINYINT_BUFF_LEN; + break; + + case TSDB_DATA_TYPE_USMALLINT: + len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", + colIndex, "SMALLINT UNSIGNED"); + lenOfOneRow += SMALLINT_BUFF_LEN; + break; + + case TSDB_DATA_TYPE_UINT: + len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", + colIndex, "INT UNSIGNED"); + lenOfOneRow += INT_BUFF_LEN; + break; + + case TSDB_DATA_TYPE_UBIGINT: + len += snprintf(cols + len, COL_BUFFER_LEN - len, ",C%d %s", + colIndex, "BIGINT UNSIGNED"); + lenOfOneRow += BIGINT_BUFF_LEN; + break; + default: taos_close(taos); free(command); @@ -4003,6 +4291,26 @@ static int createSuperTable( len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, "T%d %s,", tagIndex, "DOUBLE"); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + DOUBLE_BUFF_LEN; + } else if (strcasecmp(dataType, "UTINYINT") == 0) { + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "T%d %s,", tagIndex, "TINYINT UNSIGNED"); + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + TINYINT_BUFF_LEN; + } else if (strcasecmp(dataType, "USMALLINT") == 0) { + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "T%d %s,", tagIndex, "SMALLINT UNSIGNED"); + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + SMALLINT_BUFF_LEN; + } else if (strcasecmp(dataType, "UINT") == 0) { + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "T%d %s,", tagIndex, "INT UNSIGNED"); + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + INT_BUFF_LEN; + } else if (strcasecmp(dataType, "UBIGINT") == 0) { + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "T%d %s,", tagIndex, "BIGINT UNSIGNED"); + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + BIGINT_BUFF_LEN; + } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { + len += snprintf(tags + len, TSDB_MAX_TAGS_LEN - len, + "T%d %s,", tagIndex, "TIMESTAMP"); + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + TIMESTAMP_BUFF_LEN; } else { taos_close(taos); free(command); @@ -4158,19 +4466,17 @@ int createDatabasesAndStables(char *command) { errorPrint("create super table %"PRIu64" failed!\n\n", j); continue; } - } - - ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName, + } else { + ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName, &g_Dbs.db[i].superTbls[j]); - if (0 != ret) { - errorPrint2("\nget super table %s.%s info failed!\n\n", - g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].stbName); - continue; + if (0 != ret) { + errorPrint2("\nget super table %s.%s info failed!\n\n", + g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].stbName); + continue; + } } - validStbCount ++; } - g_Dbs.db[i].superTblCount = validStbCount; } @@ -4663,6 +4969,18 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } else if (0 == strncasecmp(superTbls->columns[c].dataType, "TIMESTAMP", strlen("TIMESTAMP"))) { superTbls->columns[c].data_type = TSDB_DATA_TYPE_TIMESTAMP; + } else if (0 == strncasecmp(superTbls->columns[c].dataType, + "UTINYINT", strlen("UTINYINT"))) { + superTbls->columns[c].data_type = TSDB_DATA_TYPE_UTINYINT; + } else if (0 == strncasecmp(superTbls->columns[c].dataType, + "USMALLINT", strlen("USMALLINT"))) { + superTbls->columns[c].data_type = TSDB_DATA_TYPE_USMALLINT; + } else if (0 == strncasecmp(superTbls->columns[c].dataType, + "UINT", strlen("UINT"))) { + superTbls->columns[c].data_type = TSDB_DATA_TYPE_UINT; + } else if (0 == strncasecmp(superTbls->columns[c].dataType, + "UBIGINT", strlen("UBIGINT"))) { + superTbls->columns[c].data_type = TSDB_DATA_TYPE_UBIGINT; } else { superTbls->columns[c].data_type = TSDB_DATA_TYPE_NULL; } @@ -4768,6 +5086,18 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } else if (0 == strncasecmp(superTbls->tags[t].dataType, "TIMESTAMP", strlen("TIMESTAMP"))) { superTbls->tags[t].data_type = TSDB_DATA_TYPE_TIMESTAMP; + } else if (0 == strncasecmp(superTbls->tags[t].dataType, + "UTINYINT", strlen("UTINYINT"))) { + superTbls->tags[t].data_type = TSDB_DATA_TYPE_UTINYINT; + } else if (0 == strncasecmp(superTbls->tags[t].dataType, + "USMALLINT", strlen("USMALLINT"))) { + superTbls->tags[t].data_type = TSDB_DATA_TYPE_USMALLINT; + } else if (0 == strncasecmp(superTbls->tags[t].dataType, + "UINT", strlen("UINT"))) { + superTbls->tags[t].data_type = TSDB_DATA_TYPE_UINT; + } else if (0 == strncasecmp(superTbls->tags[t].dataType, + "UBIGINT", strlen("UBIGINT"))) { + superTbls->tags[t].data_type = TSDB_DATA_TYPE_UBIGINT; } else { superTbls->tags[t].data_type = TSDB_DATA_TYPE_NULL; } @@ -6185,9 +6515,22 @@ static int64_t generateStbRowData( tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, INT_BUFF_LEN)); break; + case TSDB_DATA_TYPE_UINT: + tmp = rand_uint_str(); + tmpLen = strlen(tmp); + tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, INT_BUFF_LEN)); + break; + case TSDB_DATA_TYPE_BIGINT: tmp = rand_bigint_str(); - tstrncpy(pstr + dataLen, tmp, BIGINT_BUFF_LEN); + tmpLen = strlen(tmp); + tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, BIGINT_BUFF_LEN)); + break; + + case TSDB_DATA_TYPE_UBIGINT: + tmp = rand_ubigint_str(); + tmpLen = strlen(tmp); + tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, BIGINT_BUFF_LEN)); break; case TSDB_DATA_TYPE_FLOAT: @@ -6201,38 +6544,49 @@ static int64_t generateStbRowData( tmp = rand_float_str(); } tmpLen = strlen(tmp); - tstrncpy(pstr + dataLen, tmp, min(tmpLen +1, FLOAT_BUFF_LEN)); + tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, FLOAT_BUFF_LEN)); break; case TSDB_DATA_TYPE_DOUBLE: tmp = rand_double_str(); tmpLen = strlen(tmp); - tstrncpy(pstr + dataLen, tmp, min(tmpLen +1, DOUBLE_BUFF_LEN)); + tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, DOUBLE_BUFF_LEN)); break; case TSDB_DATA_TYPE_SMALLINT: tmp = rand_smallint_str(); tmpLen = strlen(tmp); - tstrncpy(pstr + dataLen, tmp, - min(tmpLen + 1, SMALLINT_BUFF_LEN)); + tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, SMALLINT_BUFF_LEN)); + break; + + case TSDB_DATA_TYPE_USMALLINT: + tmp = rand_usmallint_str(); + tmpLen = strlen(tmp); + tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, SMALLINT_BUFF_LEN)); break; case TSDB_DATA_TYPE_TINYINT: tmp = rand_tinyint_str(); tmpLen = strlen(tmp); - tstrncpy(pstr + dataLen, tmp, min(tmpLen +1, TINYINT_BUFF_LEN)); + tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, TINYINT_BUFF_LEN)); + break; + + case TSDB_DATA_TYPE_UTINYINT: + tmp = rand_utinyint_str(); + tmpLen = strlen(tmp); + tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, TINYINT_BUFF_LEN)); break; case TSDB_DATA_TYPE_BOOL: tmp = rand_bool_str(); tmpLen = strlen(tmp); - tstrncpy(pstr + dataLen, tmp, min(tmpLen +1, BOOL_BUFF_LEN)); + tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, BOOL_BUFF_LEN)); break; case TSDB_DATA_TYPE_TIMESTAMP: tmp = rand_bigint_str(); tmpLen = strlen(tmp); - tstrncpy(pstr + dataLen, tmp, min(tmpLen +1, BIGINT_BUFF_LEN)); + tstrncpy(pstr + dataLen, tmp, min(tmpLen + 1, BIGINT_BUFF_LEN)); break; case TSDB_DATA_TYPE_NULL: @@ -6243,9 +6597,8 @@ static int64_t generateStbRowData( stbInfo->columns[i].dataType); exit(EXIT_FAILURE); } - if (tmp) { - dataLen += strlen(tmp); + dataLen += tmpLen; } } @@ -6253,7 +6606,7 @@ static int64_t generateStbRowData( return 0; } - tstrncpy(pstr + dataLen, ")", 2); + dataLen += snprintf(pstr + dataLen, 2, ")"); verbosePrint("%s() LN%d, dataLen:%"PRId64"\n", __func__, __LINE__, dataLen); verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf); @@ -6330,6 +6683,22 @@ static int64_t generateData(char *recBuf, char *data_type, free(s); break; + case TSDB_DATA_TYPE_UTINYINT: + pstr += sprintf(pstr, ",%d", rand_utinyint() ); + break; + + case TSDB_DATA_TYPE_USMALLINT: + pstr += sprintf(pstr, ",%d", rand_usmallint()); + break; + + case TSDB_DATA_TYPE_UINT: + pstr += sprintf(pstr, ",%d", rand_uint()); + break; + + case TSDB_DATA_TYPE_UBIGINT: + pstr += sprintf(pstr, ",%"PRId64"", rand_ubigint()); + break; + case TSDB_DATA_TYPE_NULL: break; @@ -6388,7 +6757,7 @@ static int generateSampleFromRand( case TSDB_DATA_TYPE_NCHAR: dataLen = (columns)?columns[c].dataLen:g_args.binwidth; - rand_string(data, dataLen); + rand_string(data, dataLen - 1); pos += sprintf(buff + pos, "%s,", data); break; @@ -6401,10 +6770,18 @@ static int generateSampleFromRand( pos += sprintf(buff + pos, "%s,", tmp); break; + case TSDB_DATA_TYPE_UINT: + pos += sprintf(buff + pos, "%s,", rand_uint_str()); + break; + case TSDB_DATA_TYPE_BIGINT: pos += sprintf(buff + pos, "%s,", rand_bigint_str()); break; + case TSDB_DATA_TYPE_UBIGINT: + pos += sprintf(buff + pos, "%s,", rand_ubigint_str()); + break; + case TSDB_DATA_TYPE_FLOAT: if (g_args.demo_mode) { if (c == 0) { @@ -6426,10 +6803,18 @@ static int generateSampleFromRand( pos += sprintf(buff + pos, "%s,", rand_smallint_str()); break; + case TSDB_DATA_TYPE_USMALLINT: + pos += sprintf(buff + pos, "%s,", rand_usmallint_str()); + break; + case TSDB_DATA_TYPE_TINYINT: pos += sprintf(buff + pos, "%s,", rand_tinyint_str()); break; + case TSDB_DATA_TYPE_UTINYINT: + pos += sprintf(buff + pos, "%s,", rand_utinyint_str()); + break; + case TSDB_DATA_TYPE_BOOL: pos += sprintf(buff + pos, "%s,", rand_bool_str()); break; @@ -6953,13 +7338,17 @@ static int32_t prepareStmtBindArrayByType( char *value) { int32_t *bind_int; + uint32_t *bind_uint; int64_t *bind_bigint; + uint64_t *bind_ubigint; float *bind_float; double *bind_double; int8_t *bind_bool; int64_t *bind_ts2; int16_t *bind_smallint; + uint16_t *bind_usmallint; int8_t *bind_tinyint; + uint8_t *bind_utinyint; switch(data_type) { case TSDB_DATA_TYPE_BINARY: @@ -7024,6 +7413,22 @@ static int32_t prepareStmtBindArrayByType( bind->length = &bind->buffer_length; bind->is_null = NULL; break; + + case TSDB_DATA_TYPE_UINT: + bind_uint = malloc(sizeof(uint32_t)); + assert(bind_uint); + + if (value) { + *bind_uint = atoi(value); + } else { + *bind_uint = rand_int(); + } + bind->buffer_type = TSDB_DATA_TYPE_UINT; + bind->buffer_length = sizeof(uint32_t); + bind->buffer = bind_uint; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + break; case TSDB_DATA_TYPE_BIGINT: bind_bigint = malloc(sizeof(int64_t)); @@ -7041,6 +7446,22 @@ static int32_t prepareStmtBindArrayByType( bind->is_null = NULL; break; + case TSDB_DATA_TYPE_UBIGINT: + bind_ubigint = malloc(sizeof(uint64_t)); + assert(bind_ubigint); + + if (value) { + *bind_ubigint = atoll(value); + } else { + *bind_ubigint = rand_bigint(); + } + bind->buffer_type = TSDB_DATA_TYPE_UBIGINT; + bind->buffer_length = sizeof(uint64_t); + bind->buffer = bind_ubigint; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + break; + case TSDB_DATA_TYPE_FLOAT: bind_float = malloc(sizeof(float)); assert(bind_float); @@ -7089,6 +7510,22 @@ static int32_t prepareStmtBindArrayByType( bind->is_null = NULL; break; + case TSDB_DATA_TYPE_USMALLINT: + bind_usmallint = malloc(sizeof(uint16_t)); + assert(bind_usmallint); + + if (value) { + *bind_usmallint = (uint16_t)atoi(value); + } else { + *bind_usmallint = rand_smallint(); + } + bind->buffer_type = TSDB_DATA_TYPE_SMALLINT; + bind->buffer_length = sizeof(uint16_t); + bind->buffer = bind_usmallint; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + break; + case TSDB_DATA_TYPE_TINYINT: bind_tinyint = malloc(sizeof(int8_t)); assert(bind_tinyint); @@ -7105,6 +7542,22 @@ static int32_t prepareStmtBindArrayByType( bind->is_null = NULL; break; + case TSDB_DATA_TYPE_UTINYINT: + bind_utinyint = malloc(sizeof(uint8_t)); + assert(bind_utinyint); + + if (value) { + *bind_utinyint = (int8_t)atoi(value); + } else { + *bind_utinyint = rand_tinyint(); + } + bind->buffer_type = TSDB_DATA_TYPE_UTINYINT; + bind->buffer_length = sizeof(uint8_t); + bind->buffer = bind_utinyint; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + break; + case TSDB_DATA_TYPE_BOOL: bind_bool = malloc(sizeof(int8_t)); assert(bind_bool); @@ -7179,11 +7632,15 @@ static int32_t prepareStmtBindArrayByTypeForRand( char *value) { int32_t *bind_int; + uint32_t *bind_uint; int64_t *bind_bigint; + uint64_t *bind_ubigint; float *bind_float; double *bind_double; int16_t *bind_smallint; + uint16_t *bind_usmallint; int8_t *bind_tinyint; + uint8_t *bind_utinyint; int8_t *bind_bool; int64_t *bind_ts2; @@ -7253,6 +7710,23 @@ static int32_t prepareStmtBindArrayByTypeForRand( *ptr += bind->buffer_length; break; + case TSDB_DATA_TYPE_UINT: + bind_uint = (uint32_t *)*ptr; + + if (value) { + *bind_uint = atoi(value); + } else { + *bind_uint = rand_int(); + } + bind->buffer_type = TSDB_DATA_TYPE_UINT; + bind->buffer_length = sizeof(uint32_t); + bind->buffer = bind_uint; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + *ptr += bind->buffer_length; + break; + case TSDB_DATA_TYPE_BIGINT: bind_bigint = (int64_t *)*ptr; @@ -7270,6 +7744,23 @@ static int32_t prepareStmtBindArrayByTypeForRand( *ptr += bind->buffer_length; break; + case TSDB_DATA_TYPE_UBIGINT: + bind_ubigint = (uint64_t *)*ptr; + + if (value) { + *bind_ubigint = atoll(value); + } else { + *bind_ubigint = rand_bigint(); + } + bind->buffer_type = TSDB_DATA_TYPE_UBIGINT; + bind->buffer_length = sizeof(uint64_t); + bind->buffer = bind_ubigint; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + *ptr += bind->buffer_length; + break; + case TSDB_DATA_TYPE_FLOAT: bind_float = (float *)*ptr; @@ -7321,6 +7812,23 @@ static int32_t prepareStmtBindArrayByTypeForRand( *ptr += bind->buffer_length; break; + case TSDB_DATA_TYPE_USMALLINT: + bind_usmallint = (uint16_t *)*ptr; + + if (value) { + *bind_usmallint = (uint16_t)atoi(value); + } else { + *bind_usmallint = rand_smallint(); + } + bind->buffer_type = TSDB_DATA_TYPE_USMALLINT; + bind->buffer_length = sizeof(uint16_t); + bind->buffer = bind_usmallint; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + *ptr += bind->buffer_length; + break; + case TSDB_DATA_TYPE_TINYINT: bind_tinyint = (int8_t *)*ptr; @@ -7338,6 +7846,23 @@ static int32_t prepareStmtBindArrayByTypeForRand( *ptr += bind->buffer_length; break; + case TSDB_DATA_TYPE_UTINYINT: + bind_utinyint = (uint8_t *)*ptr; + + if (value) { + *bind_utinyint = (uint8_t)atoi(value); + } else { + *bind_utinyint = rand_tinyint(); + } + bind->buffer_type = TSDB_DATA_TYPE_UTINYINT; + bind->buffer_length = sizeof(uint8_t); + bind->buffer = bind_utinyint; + bind->length = &bind->buffer_length; + bind->is_null = NULL; + + *ptr += bind->buffer_length; + break; + case TSDB_DATA_TYPE_BOOL: bind_bool = (int8_t *)*ptr; @@ -7668,7 +8193,7 @@ UNUSED_FUNC static int32_t prepareStbStmtRand( } #if STMT_BIND_PARAM_BATCH == 1 -static int execBindParamBatch( +static int execStbBindParamBatch( threadInfo *pThreadInfo, char *tableName, int64_t tableSeq, @@ -7682,7 +8207,9 @@ static int execBindParamBatch( TAOS_STMT *stmt = pThreadInfo->stmt; SSuperTable *stbInfo = pThreadInfo->stbInfo; - uint32_t columnCount = (stbInfo)?pThreadInfo->stbInfo->columnCount:g_args.columnCount; + assert(stbInfo); + + uint32_t columnCount = pThreadInfo->stbInfo->columnCount; uint32_t thisBatch = MAX_SAMPLES - (*pSamplePos); @@ -7707,104 +8234,101 @@ static int execBindParamBatch( param->buffer = pThreadInfo->bind_ts_array; } else { - data_type = (stbInfo)?stbInfo->columns[c-1].data_type:g_args.data_type[c-1]; + data_type = stbInfo->columns[c-1].data_type; char *tmpP; switch(data_type) { case TSDB_DATA_TYPE_BINARY: + param->buffer_length = + stbInfo->columns[c-1].dataLen; + + tmpP = + (char *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray + +sizeof(char*)*(c-1))); + + verbosePrint("%s() LN%d, tmpP=%p pos=%"PRId64" width=%"PRIxPTR" position=%"PRId64"\n", + __func__, __LINE__, tmpP, *pSamplePos, param->buffer_length, + (*pSamplePos) * param->buffer_length); + + param->buffer = (void *)(tmpP + *pSamplePos * param->buffer_length); + break; + case TSDB_DATA_TYPE_NCHAR: param->buffer_length = - ((stbInfo)?stbInfo->columns[c-1].dataLen:g_args.binwidth); + stbInfo->columns[c-1].dataLen; tmpP = (char *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray +sizeof(char*)*(c-1))); - verbosePrint("%s() LN%d, tmpP=%p pos=%"PRId64" width=%d position=%"PRId64"\n", - __func__, __LINE__, tmpP, *pSamplePos, - (((stbInfo)?stbInfo->columns[c-1].dataLen:g_args.binwidth)), - (*pSamplePos) * - (((stbInfo)?stbInfo->columns[c-1].dataLen:g_args.binwidth))); + verbosePrint("%s() LN%d, tmpP=%p pos=%"PRId64" width=%"PRIxPTR" position=%"PRId64"\n", + __func__, __LINE__, tmpP, *pSamplePos, param->buffer_length, + (*pSamplePos) * param->buffer_length); - param->buffer = (void *)(tmpP + *pSamplePos * - (((stbInfo)?stbInfo->columns[c-1].dataLen:g_args.binwidth)) - ); + param->buffer = (void *)(tmpP + *pSamplePos * param->buffer_length); break; case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: param->buffer_length = sizeof(int32_t); - param->buffer = (stbInfo)? + param->buffer = (void *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray+sizeof(char*)*(c-1)) - + stbInfo->columns[c-1].dataLen * (*pSamplePos)): - (void *)((uintptr_t)*(uintptr_t*)(g_sampleBindBatchArray+sizeof(char*)*(c-1)) - + sizeof(int32_t)*(*pSamplePos)); + + stbInfo->columns[c-1].dataLen * (*pSamplePos)); break; case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: param->buffer_length = sizeof(int8_t); - param->buffer = (stbInfo)? + param->buffer = (void *)((uintptr_t)*(uintptr_t*)( stbInfo->sampleBindBatchArray +sizeof(char*)*(c-1)) - + stbInfo->columns[c-1].dataLen*(*pSamplePos)): - (void *)((uintptr_t)*(uintptr_t*)( - g_sampleBindBatchArray+sizeof(char*)*(c-1)) - + sizeof(int8_t)*(*pSamplePos)); + + stbInfo->columns[c-1].dataLen*(*pSamplePos)); break; case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: param->buffer_length = sizeof(int16_t); - param->buffer = (stbInfo)? + param->buffer = (void *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray+sizeof(char*)*(c-1)) - + stbInfo->columns[c-1].dataLen * (*pSamplePos)): - (void *)((uintptr_t)*(uintptr_t*)(g_sampleBindBatchArray+sizeof(char*)*(c-1)) - + sizeof(int16_t)*(*pSamplePos)); + + stbInfo->columns[c-1].dataLen * (*pSamplePos)); break; case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: param->buffer_length = sizeof(int64_t); - param->buffer = (stbInfo)? + param->buffer = (void *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray+sizeof(char*)*(c-1)) - + stbInfo->columns[c-1].dataLen * (*pSamplePos)): - (void *)((uintptr_t)*(uintptr_t*)(g_sampleBindBatchArray+sizeof(char*)*(c-1)) - + sizeof(int64_t)*(*pSamplePos)); + + stbInfo->columns[c-1].dataLen * (*pSamplePos)); break; case TSDB_DATA_TYPE_BOOL: param->buffer_length = sizeof(int8_t); - param->buffer = (stbInfo)? + param->buffer = (void *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray+sizeof(char*)*(c-1)) - + stbInfo->columns[c-1].dataLen * (*pSamplePos)): - (void *)((uintptr_t)*(uintptr_t*)(g_sampleBindBatchArray+sizeof(char*)*(c-1)) - + sizeof(int8_t)*(*pSamplePos)); + + stbInfo->columns[c-1].dataLen * (*pSamplePos)); break; case TSDB_DATA_TYPE_FLOAT: param->buffer_length = sizeof(float); - param->buffer = (stbInfo)? + param->buffer = (void *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray+sizeof(char*)*(c-1)) - + stbInfo->columns[c-1].dataLen * (*pSamplePos)): - (void *)((uintptr_t)*(uintptr_t*)(g_sampleBindBatchArray+sizeof(char*)*(c-1)) - + sizeof(float)*(*pSamplePos)); + + stbInfo->columns[c-1].dataLen * (*pSamplePos)); break; case TSDB_DATA_TYPE_DOUBLE: param->buffer_length = sizeof(double); - param->buffer = (stbInfo)? + param->buffer = (void *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray+sizeof(char*)*(c-1)) - + stbInfo->columns[c-1].dataLen * (*pSamplePos)): - (void *)((uintptr_t)*(uintptr_t*)(g_sampleBindBatchArray+sizeof(char*)*(c-1)) - + sizeof(double)*(*pSamplePos)); + + stbInfo->columns[c-1].dataLen * (*pSamplePos)); break; case TSDB_DATA_TYPE_TIMESTAMP: param->buffer_length = sizeof(int64_t); - param->buffer = (stbInfo)? + param->buffer = (void *)((uintptr_t)*(uintptr_t*)(stbInfo->sampleBindBatchArray+sizeof(char*)*(c-1)) - + stbInfo->columns[c-1].dataLen * (*pSamplePos)): - (void *)((uintptr_t)*(uintptr_t*)(g_sampleBindBatchArray+sizeof(char*)*(c-1)) - + sizeof(int64_t)*(*pSamplePos)); + + stbInfo->columns[c-1].dataLen * (*pSamplePos)); break; default: @@ -7825,7 +8349,7 @@ static int execBindParamBatch( if (param->buffer_type == TSDB_DATA_TYPE_NCHAR) { param->length[b] = strlen( (char *)param->buffer + b * - ((stbInfo)?stbInfo->columns[c].dataLen:g_args.binwidth) + stbInfo->columns[c].dataLen ); } else { param->length[b] = param->buffer_length; @@ -7909,24 +8433,28 @@ static int parseSamplefileToStmtBatch( switch(data_type) { case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: tmpP = calloc(1, sizeof(int) * MAX_SAMPLES); assert(tmpP); *(uintptr_t*)(sampleBindBatchArray+ sizeof(uintptr_t*)*c) = (uintptr_t)tmpP; break; case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: tmpP = calloc(1, sizeof(int8_t) * MAX_SAMPLES); assert(tmpP); *(uintptr_t*)(sampleBindBatchArray+ sizeof(uintptr_t*)*c) = (uintptr_t)tmpP; break; case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: tmpP = calloc(1, sizeof(int16_t) * MAX_SAMPLES); assert(tmpP); *(uintptr_t*)(sampleBindBatchArray+ sizeof(uintptr_t*)*c) = (uintptr_t)tmpP; break; case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: tmpP = calloc(1, sizeof(int64_t) * MAX_SAMPLES); assert(tmpP); *(uintptr_t*)(sampleBindBatchArray+ sizeof(uintptr_t*)*c) = (uintptr_t)tmpP; @@ -8005,6 +8533,7 @@ static int parseSamplefileToStmtBatch( switch(data_type) { case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: *((int32_t*)((uintptr_t)*(uintptr_t*)(sampleBindBatchArray +sizeof(char*)*c)+sizeof(int32_t)*i)) = atoi(tmpStr); @@ -8023,18 +8552,21 @@ static int parseSamplefileToStmtBatch( break; case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: *((int8_t*)((uintptr_t)*(uintptr_t*)(sampleBindBatchArray +sizeof(char*)*c)+sizeof(int8_t)*i)) = (int8_t)atoi(tmpStr); break; case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: *((int16_t*)((uintptr_t)*(uintptr_t*)(sampleBindBatchArray +sizeof(char*)*c)+sizeof(int16_t)*i)) = (int16_t)atoi(tmpStr); break; case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: *((int64_t*)((uintptr_t)*(uintptr_t*)(sampleBindBatchArray +sizeof(char*)*c)+sizeof(int64_t)*i)) = (int64_t)atol(tmpStr); @@ -8372,7 +8904,7 @@ static int32_t prepareStbStmt( } #if STMT_BIND_PARAM_BATCH == 1 - return execBindParamBatch( + return execStbBindParamBatch( pThreadInfo, tableName, tableSeq, @@ -9771,7 +10303,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, &stbInfo->childTblName, &childTblCount, limit, offset); - ntables = childTblCount; // CBD + ntables = childTblCount; } else { ntables = g_args.ntables; tableFrom = 0; @@ -9790,8 +10322,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, b = ntables % threads; } - if ((stbInfo) - && (stbInfo->iface == REST_IFACE)) { + if (g_args.iface == REST_IFACE || ((stbInfo) && (stbInfo->iface == REST_IFACE))) { if (convertHostToServAddr( g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) { ERROR_EXIT("convert host to server address"); @@ -10060,15 +10591,18 @@ static void startMultiThreadInsertData(int threads, char* db_name, } } - fprintf(stderr, "insert delay, avg: %10.2fms, max: %10.2fms, min: %10.2fms\n\n", - (double)avgDelay/1000.0, - (double)maxDelay/1000.0, - (double)minDelay/1000.0); - if (g_fpOfInsertResult) { - fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %10.2fms, min: %10.2fms\n\n", - (double)avgDelay/1000.0, - (double)maxDelay/1000.0, - (double)minDelay/1000.0); + if (minDelay != UINT64_MAX) { + fprintf(stderr, "insert delay, avg: %10.2fms, max: %10.2fms, min: %10.2fms\n\n", + (double)avgDelay/1000.0, + (double)maxDelay/1000.0, + (double)minDelay/1000.0); + + if (g_fpOfInsertResult) { + fprintf(g_fpOfInsertResult, "insert delay, avg:%10.2fms, max: %10.2fms, min: %10.2fms\n\n", + (double)avgDelay/1000.0, + (double)maxDelay/1000.0, + (double)minDelay/1000.0); + } } //taos_close(taos); @@ -11239,6 +11773,8 @@ static void initOfQueryMeta() { } static void setParaFromArg() { + char type[20]; + char length[20]; if (g_args.host) { tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE); } else { @@ -11320,7 +11856,17 @@ static void setParaFromArg() { g_Dbs.db[0].superTbls[0].columns[i].data_type = data_type[i]; tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, dataType[i], min(DATATYPE_BUFF_LEN, strlen(dataType[i]) + 1)); - g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.binwidth; + if (1 == regexMatch(dataType[i], "^(NCHAR|BINARY)(\\([1-9][0-9]*\\))$", REG_ICASE | + REG_EXTENDED)) { + sscanf(dataType[i], "%[^(](%[^)]", type, length); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = atoi(length); + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + type, min(DATATYPE_BUFF_LEN, strlen(type) + 1)); + } else { + g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.binwidth; + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + dataType[i], min(DATATYPE_BUFF_LEN, strlen(dataType[i]) + 1)); + } g_Dbs.db[0].superTbls[0].columnCount++; } @@ -11567,5 +12113,4 @@ int main(int argc, char *argv[]) { } return 0; -} - +} \ No newline at end of file diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index ad86effb3a3577f4b45b422a0f8d24a54bb70f74..56967904fe1af7cb769e10d6a415e18833b15d1e 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#include +#include #include #include #include @@ -25,7 +27,12 @@ #include "tsclient.h" #include "tsdb.h" #include "tutil.h" -#include + +#define AVRO_SUPPORT 0 + +#if AVRO_SUPPORT == 1 +#include +#endif #define TSDB_SUPPORT_NANOSECOND 1 @@ -39,8 +46,8 @@ static int converStringToReadable(char *str, int size, char *buf, int bufsize); static int convertNCharToReadable(char *str, int size, char *buf, int bufsize); -static void taosDumpCharset(FILE *fp); -static void taosLoadFileCharset(FILE *fp, char *fcharset); +static void dumpCharset(FILE *fp); +static void loadFileCharset(FILE *fp, char *fcharset); typedef struct { short bytes; @@ -120,7 +127,7 @@ enum _show_tables_index { TSDB_MAX_SHOW_TABLES }; -// ---------------------------------- DESCRIBE METRIC CONFIGURE ------------------------------ +// ---------------------------------- DESCRIBE STABLE CONFIGURE ------------------------------ enum _describe_table_index { TSDB_DESCRIBE_METRIC_FIELD_INDEX, TSDB_DESCRIBE_METRIC_TYPE_INDEX, @@ -129,29 +136,52 @@ enum _describe_table_index { TSDB_MAX_DESCRIBE_METRIC }; -#define COL_NOTE_LEN 128 +#define COL_NOTE_LEN 4 +#define COL_TYPEBUF_LEN 16 +#define COL_VALUEBUF_LEN 32 typedef struct { - char field[TSDB_COL_NAME_LEN + 1]; - char type[32]; + char field[TSDB_COL_NAME_LEN]; + char type[COL_TYPEBUF_LEN]; int length; char note[COL_NOTE_LEN]; -} SColDes; + char value[COL_VALUEBUF_LEN]; + char *var_value; +} ColDes; typedef struct { char name[TSDB_TABLE_NAME_LEN]; - SColDes cols[]; -} STableDef; + ColDes cols[]; +} TableDef; extern char version[]; #define DB_PRECISION_LEN 8 #define DB_STATUS_LEN 16 +typedef struct { + char name[TSDB_TABLE_NAME_LEN]; + bool belongStb; + char stable[TSDB_TABLE_NAME_LEN]; +} TableInfo; + +typedef struct { + char name[TSDB_TABLE_NAME_LEN]; + char stable[TSDB_TABLE_NAME_LEN]; +} TableRecord; + +typedef struct { + bool isStb; + bool belongStb; + int64_t dumpNtbCount; + TableRecord **dumpNtbInfos; + TableRecord tableRecord; +} TableRecordInfo; + typedef struct { char name[TSDB_DB_NAME_LEN]; char create_time[32]; - int32_t ntables; + int64_t ntables; int32_t vgroups; int16_t replica; int16_t quorum; @@ -171,28 +201,22 @@ typedef struct { char precision[DB_PRECISION_LEN]; // time resolution int8_t update; char status[DB_STATUS_LEN]; + int64_t dumpTbCount; + TableRecordInfo **dumpTbInfos; } SDbInfo; -typedef struct { - char name[TSDB_TABLE_NAME_LEN]; - char metric[TSDB_TABLE_NAME_LEN]; -} STableRecord; - -typedef struct { - bool isMetric; - STableRecord tableRecord; -} STableRecordInfo; - typedef struct { pthread_t threadID; int32_t threadIndex; int32_t totalThreads; char dbName[TSDB_DB_NAME_LEN]; - int precision; - void *taosCon; + char stbName[TSDB_TABLE_NAME_LEN]; + int precision; + TAOS *taos; int64_t rowsOfDumpOut; int64_t tablesOfDumpOut; -} SThreadParaObj; + int64_t tableFrom; +} threadInfo; typedef struct { int64_t totalRowsOfDumpOut; @@ -204,6 +228,7 @@ typedef struct { static int64_t g_totalDumpOutRows = 0; SDbInfo **g_dbInfos = NULL; +TableInfo *g_tablesList = NULL; const char *argp_program_version = version; const char *argp_program_bug_address = ""; @@ -300,13 +325,13 @@ typedef struct arguments { int32_t thread_num; int abort; char **arg_list; - int arg_list_len; - bool isDumpIn; - bool debug_print; - bool verbose_print; - bool performance_print; + int arg_list_len; + bool isDumpIn; + bool debug_print; + bool verbose_print; + bool performance_print; - int dbCount; + int dumpDbCount; } SArguments; /* Our argp parser. */ @@ -317,29 +342,21 @@ static resultStatistics g_resultStatistics = {0}; static FILE *g_fpOfResult = NULL; static int g_numOfCores = 1; -static int taosDumpOut(); -static int taosDumpIn(); -static void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, +static int dumpOut(); +static int dumpIn(); +static void dumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp); -static int taosDumpDb(SDbInfo *dbInfo, FILE *fp, TAOS *taosCon); -static int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, - char* dbName); -static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, +static int dumpCreateTableClause(TableDef *tableDes, int numOfCols, FILE *fp, char* dbName); -static void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, - int numOfCols, FILE *fp, char* dbName); -static int32_t taosDumpTable(char *tbName, char *metric, - FILE *fp, TAOS* taosCon, char* dbName, int precision); -static int taosDumpTableData(FILE *fp, char *tbName, - TAOS* taosCon, char* dbName, +static int getTableDes( + char* dbName, char *table, + TableDef *stableDes, bool isSuperTable); +static int64_t dumpTableData(FILE *fp, char *tbName, + char* dbName, int precision, char *jsonAvroSchema); -static int taosCheckParam(struct arguments *arguments); -static void taosFreeDbInfos(); -static void taosStartDumpOutWorkThreads( - int32_t numOfThread, - char *dbName, - int precision); +static int checkParam(); +static void freeDbInfos(); struct arguments g_args = { // connection option @@ -383,19 +400,46 @@ struct arguments g_args = { false, // debug_print false, // verbose_print false, // performance_print - 0, // dbCount + 0, // dumpDbCount }; -UNUSED_FUNC void errorWrongValue(char *program, char *wrong_arg, char *wrong_value) +// get taosdump commit number version +#ifndef TAOSDUMP_COMMIT_SHA1 +#define TAOSDUMP_COMMIT_SHA1 "unknown" +#endif + +#ifndef TD_VERNUMBER +#define TD_VERNUMBER "unknown" +#endif + +#ifndef TAOSDUMP_STATUS +#define TAOSDUMP_STATUS "unknown" +#endif + +static void printVersion() { + char tdengine_ver[] = TD_VERNUMBER; + char taosdump_ver[] = TAOSDUMP_COMMIT_SHA1; + char taosdump_status[] = TAOSDUMP_STATUS; + + if (strlen(taosdump_status) == 0) { + printf("taosdump version %s-%s\n", + tdengine_ver, taosdump_ver); + } else { + printf("taosdump version %s-%s, status:%s\n", + tdengine_ver, taosdump_ver, taosdump_status); + } +} + +void errorWrongValue(char *program, char *wrong_arg, char *wrong_value) { fprintf(stderr, "%s %s: %s is an invalid value\n", program, wrong_arg, wrong_value); - fprintf(stderr, "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); + fprintf(stderr, "Try `taosdump --help' or `taosdump --usage' for more information.\n"); } static void errorUnrecognized(char *program, char *wrong_arg) { fprintf(stderr, "%s: unrecognized options '%s'\n", program, wrong_arg); - fprintf(stderr, "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); + fprintf(stderr, "Try `taosdump --help' or `taosdump --usage' for more information.\n"); } static void errorPrintReqArg(char *program, char *wrong_arg) @@ -404,7 +448,7 @@ static void errorPrintReqArg(char *program, char *wrong_arg) "%s: option requires an argument -- '%s'\n", program, wrong_arg); fprintf(stderr, - "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); + "Try `taosdump --help' or `taosdump --usage' for more information.\n"); } static void errorPrintReqArg2(char *program, char *wrong_arg) @@ -413,7 +457,7 @@ static void errorPrintReqArg2(char *program, char *wrong_arg) "%s: option requires a number argument '-%s'\n", program, wrong_arg); fprintf(stderr, - "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); + "Try `taosdump --help' or `taosdump --usage' for more information.\n"); } static void errorPrintReqArg3(char *program, char *wrong_arg) @@ -422,7 +466,7 @@ static void errorPrintReqArg3(char *program, char *wrong_arg) "%s: option '%s' requires an argument\n", program, wrong_arg); fprintf(stderr, - "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); + "Try `taosdump --help' or `taosdump --usage' for more information.\n"); } /* Parse a single option. */ @@ -449,7 +493,14 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { errorPrintReqArg2("taosdump", "P"); exit(EXIT_FAILURE); } - g_args.port = atoi(arg); + + uint64_t port = atoi(arg); + if (port > 65535) { + errorWrongValue("taosdump", "-P or --port", arg); + exit(EXIT_FAILURE); + } + g_args.port = (uint16_t)port; + break; case 'q': g_args.mysqlFlag = atoi(arg); @@ -459,23 +510,38 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { errorPrint("Invalid path %s\n", arg); return -1; } - tstrncpy(g_args.outpath, full_path.we_wordv[0], - MAX_FILE_NAME_LEN); - wordfree(&full_path); + + if (full_path.we_wordv[0]) { + tstrncpy(g_args.outpath, full_path.we_wordv[0], + MAX_FILE_NAME_LEN); + wordfree(&full_path); + } else { + errorPrintReqArg3("taosdump", "-o or --outpath"); + exit(EXIT_FAILURE); + } break; + case 'g': g_args.debug_print = true; break; + case 'i': g_args.isDumpIn = true; if (wordexp(arg, &full_path, 0) != 0) { errorPrint("Invalid path %s\n", arg); return -1; } - tstrncpy(g_args.inpath, full_path.we_wordv[0], - MAX_FILE_NAME_LEN); - wordfree(&full_path); + + if (full_path.we_wordv[0]) { + tstrncpy(g_args.inpath, full_path.we_wordv[0], + MAX_FILE_NAME_LEN); + wordfree(&full_path); + } else { + errorPrintReqArg3("taosdump", "-i or --inpath"); + exit(EXIT_FAILURE); + } break; + case 'r': g_args.resultFile = arg; break; @@ -557,67 +623,41 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { return 0; } +static void freeTbDes(TableDef *tableDes) +{ + for (int i = 0; i < TSDB_MAX_COLUMNS; i ++) { + if (tableDes->cols[i].var_value) { + free(tableDes->cols[i].var_value); + } + } + + free(tableDes); +} + static int queryDbImpl(TAOS *taos, char *command) { - int i; TAOS_RES *res = NULL; int32_t code = -1; - for (i = 0; i < 5; i++) { - if (NULL != res) { - taos_free_result(res); - res = NULL; - } - - res = taos_query(taos, command); - code = taos_errno(res); - if (0 == code) { - break; - } + if (NULL != res) { + taos_free_result(res); + res = NULL; } + res = taos_query(taos, command); + code = taos_errno(res); + if (code != 0) { - errorPrint("Failed to run <%s>, reason: %s\n", command, taos_errstr(res)); + errorPrint("Failed to run <%s>, reason: %s\n", + command, taos_errstr(res)); taos_free_result(res); //taos_close(taos); - return -1; + return code; } taos_free_result(res); return 0; } -UNUSED_FUNC static void parse_precision_first( - int argc, char *argv[], SArguments *arguments) { - for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "-C") == 0) { - if (NULL == argv[i+1]) { - errorPrint("%s need a valid value following!\n", argv[i]); - exit(-1); - } - char *tmp = strdup(argv[i+1]); - if (tmp == NULL) { - errorPrint("%s() LN%d, strdup() cannot allocate memory\n", - __func__, __LINE__); - exit(-1); - } - if ((0 != strncasecmp(tmp, "ms", strlen("ms"))) - && (0 != strncasecmp(tmp, "us", strlen("us"))) -#if TSDB_SUPPORT_NANOSECOND == 1 - && (0 != strncasecmp(tmp, "ns", strlen("ns"))) -#endif - ) { - // - errorPrint("input precision: %s is invalid value\n", tmp); - free(tmp); - exit(-1); - } - tstrncpy(g_args.precision, tmp, - min(DB_PRECISION_LEN, strlen(tmp) + 1)); - free(tmp); - } - } -} - static void parse_args( int argc, char *argv[], SArguments *arguments) { @@ -673,6 +713,10 @@ static void parse_args( exit(EXIT_FAILURE); } g_args.databases = true; + } else if (0 == strncmp(argv[i], "--version", strlen("--version")) || + 0 == strncmp(argv[i], "-V", strlen("-V"))) { + printVersion(); + exit(EXIT_SUCCESS); } else { continue; } @@ -748,245 +792,49 @@ static int getPrecisionByString(char *precision) return -1; } -/* -static void parse_timestamp( - int argc, char *argv[], SArguments *arguments) { - for (int i = 1; i < argc; i++) { - if ((strcmp(argv[i], "-S") == 0) - || (strcmp(argv[i], "-E") == 0)) { - if (NULL == argv[i+1]) { - errorPrint("%s need a valid value following!\n", argv[i]); - exit(-1); - } - char *tmp = strdup(argv[i+1]); - if (NULL == tmp) { - errorPrint("%s() LN%d, strdup() cannot allocate memory\n", - __func__, __LINE__); - exit(-1); - } - - int64_t tmpEpoch; - if (strchr(tmp, ':') && strchr(tmp, '-')) { - strcpy(g_args.humanStartTime, tmp) - int32_t timePrec; - if (0 == strncasecmp(arguments->precision, - "ms", strlen("ms"))) { - timePrec = TSDB_TIME_PRECISION_MILLI; - } else if (0 == strncasecmp(arguments->precision, - "us", strlen("us"))) { - timePrec = TSDB_TIME_PRECISION_MICRO; -#if TSDB_SUPPORT_NANOSECOND == 1 - } else if (0 == strncasecmp(arguments->precision, - "ns", strlen("ns"))) { - timePrec = TSDB_TIME_PRECISION_NANO; -#endif - } else { - errorPrint("Invalid time precision: %s", - arguments->precision); - free(tmp); - return; - } - - if (TSDB_CODE_SUCCESS != taosParseTime( - tmp, &tmpEpoch, strlen(tmp), - timePrec, 0)) { - errorPrint("Input %s, end time error!\n", tmp); - free(tmp); - return; - } - } else { - tstrncpy(arguments->precision, "n/a", strlen("n/a") + 1); - tmpEpoch = atoll(tmp); - } - - sprintf(argv[i+1], "%"PRId64"", tmpEpoch); - debugPrint("%s() LN%d, tmp is: %s, argv[%d]: %s\n", - __func__, __LINE__, tmp, i, argv[i]); - free(tmp); - } - } -} -*/ - -int main(int argc, char *argv[]) { - static char verType[32] = {0}; - sprintf(verType, "version: %s\n", version); - argp_program_version = verType; - - int ret = 0; - /* Parse our arguments; every option seen by parse_opt will be - reflected in arguments. */ - if (argc > 1) { -// parse_precision_first(argc, argv, &g_args); - parse_timestamp(argc, argv, &g_args); - parse_args(argc, argv, &g_args); - } - - argp_parse(&argp, argc, argv, 0, 0, &g_args); - - if (g_args.abort) { -#ifndef _ALPINE - error(10, 0, "ABORTED"); -#else - abort(); -#endif - } - - printf("====== arguments config ======\n"); - { - printf("host: %s\n", g_args.host); - printf("user: %s\n", g_args.user); - printf("password: %s\n", g_args.password); - printf("port: %u\n", g_args.port); - printf("mysqlFlag: %d\n", g_args.mysqlFlag); - printf("outpath: %s\n", g_args.outpath); - printf("inpath: %s\n", g_args.inpath); - printf("resultFile: %s\n", g_args.resultFile); - printf("encode: %s\n", g_args.encode); - printf("all_databases: %s\n", g_args.all_databases?"true":"false"); - printf("databases: %d\n", g_args.databases); - printf("databasesSeq: %s\n", g_args.databasesSeq); - printf("schemaonly: %s\n", g_args.schemaonly?"true":"false"); - printf("with_property: %s\n", g_args.with_property?"true":"false"); - printf("avro format: %s\n", g_args.avro?"true":"false"); - printf("start_time: %" PRId64 "\n", g_args.start_time); - printf("human readable start time: %s \n", g_args.humanStartTime); - printf("end_time: %" PRId64 "\n", g_args.end_time); - printf("human readable end time: %s \n", g_args.humanEndTime); - printf("precision: %s\n", g_args.precision); - printf("data_batch: %d\n", g_args.data_batch); - printf("max_sql_len: %d\n", g_args.max_sql_len); - printf("table_batch: %d\n", g_args.table_batch); - printf("thread_num: %d\n", g_args.thread_num); - printf("allow_sys: %d\n", g_args.allow_sys); - printf("abort: %d\n", g_args.abort); - printf("isDumpIn: %d\n", g_args.isDumpIn); - printf("arg_list_len: %d\n", g_args.arg_list_len); - printf("debug_print: %d\n", g_args.debug_print); - - for (int32_t i = 0; i < g_args.arg_list_len; i++) { - printf("arg_list[%d]: %s\n", i, g_args.arg_list[i]); - } - } - printf("==============================\n"); - if (taosCheckParam(&g_args) < 0) { - exit(EXIT_FAILURE); - } - - g_fpOfResult = fopen(g_args.resultFile, "a"); - if (NULL == g_fpOfResult) { - errorPrint("Failed to open %s for save result\n", g_args.resultFile); - exit(-1); - }; - - fprintf(g_fpOfResult, "#############################################################################\n"); - fprintf(g_fpOfResult, "============================== arguments config =============================\n"); - { - fprintf(g_fpOfResult, "host: %s\n", g_args.host); - fprintf(g_fpOfResult, "user: %s\n", g_args.user); - fprintf(g_fpOfResult, "password: %s\n", g_args.password); - fprintf(g_fpOfResult, "port: %u\n", g_args.port); - fprintf(g_fpOfResult, "mysqlFlag: %d\n", g_args.mysqlFlag); - fprintf(g_fpOfResult, "outpath: %s\n", g_args.outpath); - fprintf(g_fpOfResult, "inpath: %s\n", g_args.inpath); - fprintf(g_fpOfResult, "resultFile: %s\n", g_args.resultFile); - fprintf(g_fpOfResult, "encode: %s\n", g_args.encode); - fprintf(g_fpOfResult, "all_databases: %s\n", g_args.all_databases?"true":"false"); - fprintf(g_fpOfResult, "databases: %d\n", g_args.databases); - fprintf(g_fpOfResult, "databasesSeq: %s\n", g_args.databasesSeq); - fprintf(g_fpOfResult, "schemaonly: %s\n", g_args.schemaonly?"true":"false"); - fprintf(g_fpOfResult, "with_property: %s\n", g_args.with_property?"true":"false"); - fprintf(g_fpOfResult, "avro format: %s\n", g_args.avro?"true":"false"); - fprintf(g_fpOfResult, "start_time: %" PRId64 "\n", g_args.start_time); - fprintf(g_fpOfResult, "human readable start time: %s \n", g_args.humanStartTime); - fprintf(g_fpOfResult, "end_time: %" PRId64 "\n", g_args.end_time); - fprintf(g_fpOfResult, "human readable end time: %s \n", g_args.humanEndTime); - fprintf(g_fpOfResult, "precision: %s\n", g_args.precision); - fprintf(g_fpOfResult, "data_batch: %d\n", g_args.data_batch); - fprintf(g_fpOfResult, "max_sql_len: %d\n", g_args.max_sql_len); - fprintf(g_fpOfResult, "table_batch: %d\n", g_args.table_batch); - fprintf(g_fpOfResult, "thread_num: %d\n", g_args.thread_num); - fprintf(g_fpOfResult, "allow_sys: %d\n", g_args.allow_sys); - fprintf(g_fpOfResult, "abort: %d\n", g_args.abort); - fprintf(g_fpOfResult, "isDumpIn: %d\n", g_args.isDumpIn); - fprintf(g_fpOfResult, "arg_list_len: %d\n", g_args.arg_list_len); - - for (int32_t i = 0; i < g_args.arg_list_len; i++) { - fprintf(g_fpOfResult, "arg_list[%d]: %s\n", i, g_args.arg_list[i]); - } - } - - g_numOfCores = (int32_t)sysconf(_SC_NPROCESSORS_ONLN); - - time_t tTime = time(NULL); - struct tm tm = *localtime(&tTime); - - if (g_args.isDumpIn) { - fprintf(g_fpOfResult, "============================== DUMP IN ============================== \n"); - fprintf(g_fpOfResult, "# DumpIn start time: %d-%02d-%02d %02d:%02d:%02d\n", - tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - if (taosDumpIn() < 0) { - ret = -1; - } - } else { - fprintf(g_fpOfResult, "============================== DUMP OUT ============================== \n"); - fprintf(g_fpOfResult, "# DumpOut start time: %d-%02d-%02d %02d:%02d:%02d\n", - tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - if (taosDumpOut() < 0) { - ret = -1; - } else { - fprintf(g_fpOfResult, "\n============================== TOTAL STATISTICS ============================== \n"); - fprintf(g_fpOfResult, "# total database count: %d\n", - g_resultStatistics.totalDatabasesOfDumpOut); - fprintf(g_fpOfResult, "# total super table count: %d\n", - g_resultStatistics.totalSuperTblsOfDumpOut); - fprintf(g_fpOfResult, "# total child table count: %"PRId64"\n", - g_resultStatistics.totalChildTblsOfDumpOut); - fprintf(g_fpOfResult, "# total row count: %"PRId64"\n", - g_resultStatistics.totalRowsOfDumpOut); - } - } - - fprintf(g_fpOfResult, "\n"); - fclose(g_fpOfResult); - - return ret; -} - -static void taosFreeDbInfos() { +static void freeDbInfos() { if (g_dbInfos == NULL) return; - for (int i = 0; i < g_args.dbCount; i++) + for (int i = 0; i < g_args.dumpDbCount; i++) tfree(g_dbInfos[i]); tfree(g_dbInfos); } // check table is normal table or super table -static int taosGetTableRecordInfo( - char *table, STableRecordInfo *pTableRecordInfo, TAOS *taosCon) { +static int getTableRecordInfo( + char *dbName, + char *table, TableRecordInfo *pTableRecordInfo) { + TAOS *taos = taos_connect(g_args.host, g_args.user, g_args.password, + dbName, g_args.port); + if (taos == NULL) { + errorPrint("Failed to connect to TDengine server %s\n", g_args.host); + return -1; + } + TAOS_ROW row = NULL; bool isSet = false; TAOS_RES *result = NULL; - memset(pTableRecordInfo, 0, sizeof(STableRecordInfo)); + memset(pTableRecordInfo, 0, sizeof(TableRecordInfo)); - char* tempCommand = (char *)malloc(COMMAND_SIZE); - if (tempCommand == NULL) { - errorPrint("%s() LN%d, failed to allocate memory\n", - __func__, __LINE__); - return -1; + char command[COMMAND_SIZE]; + + sprintf(command, "USE %s", dbName); + result = taos_query(taos, command); + int32_t code = taos_errno(result); + if (code != 0) { + errorPrint("invalid database %s, reason: %s\n", + dbName, taos_errstr(result)); + return 0; } - sprintf(tempCommand, "show tables like %s", table); + sprintf(command, "SHOW TABLES LIKE \'%s\'", table); - result = taos_query(taosCon, tempCommand); - int32_t code = taos_errno(result); + result = taos_query(taos, command); + code = taos_errno(result); if (code != 0) { - errorPrint("%s() LN%d, failed to run command %s\n", - __func__, __LINE__, tempCommand); - free(tempCommand); + errorPrint("%s() LN%d, failed to run command <%s>. reason: %s\n", + __func__, __LINE__, command, taos_errstr(result)); taos_free_result(result); return -1; } @@ -995,15 +843,20 @@ static int taosGetTableRecordInfo( while ((row = taos_fetch_row(result)) != NULL) { isSet = true; - pTableRecordInfo->isMetric = false; + pTableRecordInfo->isStb = false; tstrncpy(pTableRecordInfo->tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], min(TSDB_TABLE_NAME_LEN, fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes + 1)); - tstrncpy(pTableRecordInfo->tableRecord.metric, - (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], - min(TSDB_TABLE_NAME_LEN, - fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes + 1)); + if (strlen((char *)row[TSDB_SHOW_TABLES_METRIC_INDEX]) > 0) { + pTableRecordInfo->belongStb = true; + tstrncpy(pTableRecordInfo->tableRecord.stable, + (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], + min(TSDB_TABLE_NAME_LEN, + fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes + 1)); + } else { + pTableRecordInfo->belongStb = false; + } break; } @@ -1011,27 +864,25 @@ static int taosGetTableRecordInfo( result = NULL; if (isSet) { - free(tempCommand); return 0; } - sprintf(tempCommand, "show stables like %s", table); + sprintf(command, "SHOW STABLES LIKE \'%s\'", table); - result = taos_query(taosCon, tempCommand); + result = taos_query(taos, command); code = taos_errno(result); if (code != 0) { - errorPrint("%s() LN%d, failed to run command %s\n", - __func__, __LINE__, tempCommand); - free(tempCommand); + errorPrint("%s() LN%d, failed to run command <%s>. reason: %s\n", + __func__, __LINE__, command, taos_errstr(result)); taos_free_result(result); return -1; } while ((row = taos_fetch_row(result)) != NULL) { isSet = true; - pTableRecordInfo->isMetric = true; - tstrncpy(pTableRecordInfo->tableRecord.metric, table, + pTableRecordInfo->isStb = true; + tstrncpy(pTableRecordInfo->tableRecord.stable, table, TSDB_TABLE_NAME_LEN); break; } @@ -1040,254 +891,349 @@ static int taosGetTableRecordInfo( result = NULL; if (isSet) { - free(tempCommand); return 0; } - errorPrint("%s() LN%d, invalid table/metric %s\n", + errorPrint("%s() LN%d, invalid table/stable %s\n", __func__, __LINE__, table); - free(tempCommand); return -1; } +static int inDatabasesSeq( + char *name, + int len) +{ + if (strstr(g_args.databasesSeq, ",") == NULL) { + if (0 == strncmp(g_args.databasesSeq, name, len)) { + return 0; + } + } else { + char *dupSeq = strdup(g_args.databasesSeq); + char *running = dupSeq; + char *dbname = strsep(&running, ","); + while (dbname) { + if (0 == strncmp(dbname, name, len)) { + tfree(dupSeq); + return 0; + } -static int32_t taosSaveAllNormalTableToTempFile(TAOS *taosCon, char*meter, - char* metric, int* fd) { - STableRecord tableRecord; - - if (-1 == *fd) { - *fd = open(".tables.tmp.0", - O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (*fd == -1) { - errorPrint("%s() LN%d, failed to open temp file: .tables.tmp.0\n", - __func__, __LINE__); - return -1; + dbname = strsep(&running, ","); } } - memset(&tableRecord, 0, sizeof(STableRecord)); - tstrncpy(tableRecord.name, meter, TSDB_TABLE_NAME_LEN); - tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN); - - taosWrite(*fd, &tableRecord, sizeof(STableRecord)); - return 0; + return -1; } -static int32_t taosSaveTableOfMetricToTempFile( - TAOS *taosCon, char* metric, - int32_t* totalNumOfThread) { +static int getDumpDbCount() +{ + int count = 0; + + TAOS *taos = NULL; + TAOS_RES *result = NULL; + char *command = "show databases"; TAOS_ROW row; - int fd = -1; - STableRecord tableRecord; - char* tmpCommand = (char *)malloc(COMMAND_SIZE); - if (tmpCommand == NULL) { - errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__); - return -1; + /* Connect to server */ + taos = taos_connect(g_args.host, g_args.user, g_args.password, + NULL, g_args.port); + if (NULL == taos) { + errorPrint("Failed to connect to TDengine server %s\n", g_args.host); + return 0; } - sprintf(tmpCommand, "select tbname from %s", metric); + result = taos_query(taos, command); + int32_t code = taos_errno(result); - TAOS_RES *res = taos_query(taosCon, tmpCommand); - int32_t code = taos_errno(res); - if (code != 0) { - errorPrint("%s() LN%d, failed to run command %s\n", - __func__, __LINE__, tmpCommand); - free(tmpCommand); - taos_free_result(res); - return -1; + if (0 != code) { + errorPrint("%s() LN%d, failed to run command <%s>, reason: %s\n", + __func__, __LINE__, command, taos_errstr(result)); + taos_close(taos); + return 0; } - free(tmpCommand); - char tmpBuf[MAX_FILE_NAME_LEN]; - memset(tmpBuf, 0, MAX_FILE_NAME_LEN); - sprintf(tmpBuf, ".select-tbname.tmp"); - fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - errorPrint("%s() LN%d, failed to open temp file: %s\n", - __func__, __LINE__, tmpBuf); - taos_free_result(res); - return -1; + TAOS_FIELD *fields = taos_fetch_fields(result); + + while ((row = taos_fetch_row(result)) != NULL) { + // sys database name : 'log', but subsequent version changed to 'log' + if ((strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", + fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) + && (!g_args.allow_sys)) { + continue; + } + + if (g_args.databases) { // input multi dbs + if (inDatabasesSeq( + (char *)row[TSDB_SHOW_DB_NAME_INDEX], + fields[TSDB_SHOW_DB_NAME_INDEX].bytes) != 0) + continue; + } else if (!g_args.all_databases) { // only input one db + if (strncasecmp(g_args.arg_list[0], + (char *)row[TSDB_SHOW_DB_NAME_INDEX], + fields[TSDB_SHOW_DB_NAME_INDEX].bytes) != 0) + continue; + } + + count++; } - TAOS_FIELD *fields = taos_fetch_fields(res); + if (count == 0) { + errorPrint("%d databases valid to dump\n", count); + } - int32_t numOfTable = 0; - while ((row = taos_fetch_row(res)) != NULL) { + taos_close(taos); + return count; +} - memset(&tableRecord, 0, sizeof(STableRecord)); - tstrncpy(tableRecord.name, (char *)row[0], fields[0].bytes); - tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN); +static void dumpCreateMTableClause( + char* dbName, + char *stable, + TableDef *tableDes, + int numOfCols, + FILE *fp + ) { + int counter = 0; + int count_temp = 0; - taosWrite(fd, &tableRecord, sizeof(STableRecord)); - numOfTable++; + char* tmpBuf = (char *)malloc(COMMAND_SIZE); + if (tmpBuf == NULL) { + errorPrint("%s() LN%d, failed to allocate %d memory\n", + __func__, __LINE__, COMMAND_SIZE); + return; } - taos_free_result(res); - lseek(fd, 0, SEEK_SET); - int maxThreads = g_args.thread_num; - int tableOfPerFile ; - if (numOfTable <= g_args.thread_num) { - tableOfPerFile = 1; - maxThreads = numOfTable; - } else { - tableOfPerFile = numOfTable / g_args.thread_num; - if (0 != numOfTable % g_args.thread_num) { - tableOfPerFile += 1; + char *pstr = NULL; + pstr = tmpBuf; + + pstr += sprintf(tmpBuf, + "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", + dbName, tableDes->name, dbName, stable); + + for (; counter < numOfCols; counter++) { + if (tableDes->cols[counter].note[0] != '\0') break; + } + + assert(counter < numOfCols); + count_temp = counter; + + for (; counter < numOfCols; counter++) { + if (counter != count_temp) { + if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || + strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { + //pstr += sprintf(pstr, ", \'%s\'", tableDes->cols[counter].note); + if (tableDes->cols[counter].var_value) { + pstr += sprintf(pstr, ", %s", + tableDes->cols[counter].var_value); + } else { + pstr += sprintf(pstr, ", %s", tableDes->cols[counter].value); + } + } else { + pstr += sprintf(pstr, ", %s", tableDes->cols[counter].value); + } + } else { + if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || + strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { + //pstr += sprintf(pstr, "\'%s\'", tableDes->cols[counter].note); + if (tableDes->cols[counter].var_value) { + pstr += sprintf(pstr, "%s", tableDes->cols[counter].var_value); + } else { + pstr += sprintf(pstr, "%s", tableDes->cols[counter].value); + } + } else { + pstr += sprintf(pstr, "%s", tableDes->cols[counter].value); + } + /* pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); */ } + + /* if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || strcasecmp(tableDes->cols[counter].type, "nchar") + * == 0) { */ + /* pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length); */ + /* } */ } - char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord)); - if (NULL == tblBuf){ - errorPrint("%s() LN%d, failed to calloc %" PRIzu "\n", - __func__, __LINE__, tableOfPerFile * sizeof(STableRecord)); - close(fd); + pstr += sprintf(pstr, ");"); + + fprintf(fp, "%s\n", tmpBuf); + free(tmpBuf); +} + +static int convertTbDesToAvroSchema( + char *dbName, char *tbName, TableDef *tableDes, int colCount, + char **avroSchema) +{ + errorPrint("%s() LN%d TODO: covert table schema to avro schema\n", + __func__, __LINE__); + // { + // "namesapce": "database name", + // "type": "record", + // "name": "table name", + // "fields": [ + // { + // "name": "col0 name", + // "type": "long" + // }, + // { + // "name": "col1 name", + // "type": ["int", "null"] + // }, + // { + // "name": "col2 name", + // "type": ["float", "null"] + // }, + // ... + // { + // "name": "coln name", + // "type": ["string", "null"] + // } + // ] + // } + *avroSchema = (char *)calloc(1, + 17 + TSDB_DB_NAME_LEN /* dbname section */ + + 17 /* type: record */ + + 11 + TSDB_TABLE_NAME_LEN /* tbname section */ + + 10 /* fields section */ + + (TSDB_COL_NAME_LEN + 11 + 16) * colCount + 4); /* fields section */ + if (*avroSchema == NULL) { + errorPrint("%s() LN%d, memory allocation failed!\n", __func__, __LINE__); return -1; } - int32_t numOfThread = *totalNumOfThread; - int subFd = -1; - for (; numOfThread <= maxThreads; numOfThread++) { - memset(tmpBuf, 0, MAX_FILE_NAME_LEN); - sprintf(tmpBuf, ".tables.tmp.%d", numOfThread); - subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (subFd == -1) { - errorPrint("%s() LN%d, failed to open temp file: %s\n", - __func__, __LINE__, tmpBuf); - for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { - sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); - (void)remove(tmpBuf); + char *pstr = *avroSchema; + pstr += sprintf(pstr, + "{\"namespace\": \"%s\", \"type\": \"record\", \"name\": \"%s\", \"fields\": [", + dbName, tbName); + for (int i = 0; i < colCount; i ++) { + if (0 == i) { + pstr += sprintf(pstr, + "{\"name\": \"%s\", \"type\": \"%s\"", + tableDes->cols[i].field, "long"); + } else { + if (strcasecmp(tableDes->cols[i].type, "binary") == 0 || + strcasecmp(tableDes->cols[i].type, "nchar") == 0) { + pstr += sprintf(pstr, + "{\"name\": \"%s\", \"type\": [\"%s\", \"null\"]", + tableDes->cols[i].field, "string"); + } else { + pstr += sprintf(pstr, + "{\"name\": \"%s\", \"type\": [\"%s\", \"null\"]", + tableDes->cols[i].field, tableDes->cols[i].type); } - sprintf(tmpBuf, ".select-tbname.tmp"); - (void)remove(tmpBuf); - free(tblBuf); - close(fd); - return -1; } - - // read tableOfPerFile for fd, write to subFd - ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord)); - if (readLen <= 0) { - close(subFd); + if ((i != (colCount -1)) + && (strcmp(tableDes->cols[i + 1].note, "TAG") != 0)) { + pstr += sprintf(pstr, "},"); + } else { + pstr += sprintf(pstr, "}"); break; } - taosWrite(subFd, tblBuf, readLen); - close(subFd); } - sprintf(tmpBuf, ".select-tbname.tmp"); - (void)remove(tmpBuf); - - if (fd >= 0) { - close(fd); - fd = -1; - } + pstr += sprintf(pstr, "]}"); - *totalNumOfThread = numOfThread; + debugPrint("%s() LN%d, avroSchema: %s\n", __func__, __LINE__, *avroSchema); - free(tblBuf); return 0; } -static int inDatabasesSeq( - char *name, - int len) -{ - if (strstr(g_args.databasesSeq, ",") == NULL) { - if (0 == strncmp(g_args.databasesSeq, name, len)) { - return 0; - } - } else { - char *dupSeq = strdup(g_args.databasesSeq); - char *running = dupSeq; - char *dbname = strsep(&running, ","); - while (dbname) { - if (0 == strncmp(dbname, name, len)) { - tfree(dupSeq); - return 0; - } +static int64_t dumpNormalTable( + char *dbName, + char *stable, + char *tbName, + int precision, + FILE *fp + ) { + int colCount = 0; - dbname = strsep(&running, ","); - } + TableDef *tableDes = (TableDef *)calloc(1, sizeof(TableDef) + + sizeof(ColDes) * TSDB_MAX_COLUMNS); - free(dupSeq); - } + if (stable != NULL && stable[0] != '\0') { // dump table schema which is created by using super table + colCount = getTableDes(dbName, tbName, tableDes, false); - return -1; -} + if (colCount < 0) { + free(tableDes); + return -1; + } -static int getDbCount() -{ - int count = 0; + // create child-table using super-table + dumpCreateMTableClause(dbName, stable, tableDes, colCount, fp); + } else { // dump table definition + colCount = getTableDes(dbName, tbName, tableDes, false); - TAOS *taos = NULL; - TAOS_RES *result = NULL; - char *command = "show databases"; - TAOS_ROW row; + if (colCount < 0) { + free(tableDes); + return -1; + } - /* Connect to server */ - taos = taos_connect(g_args.host, g_args.user, g_args.password, - NULL, g_args.port); - if (NULL == taos) { - errorPrint("Failed to connect to TDengine server %s\n", g_args.host); - return 0; + // create normal-table or super-table + dumpCreateTableClause(tableDes, colCount, fp, dbName); } - result = taos_query(taos, command); - int32_t code = taos_errno(result); + char *jsonAvroSchema = NULL; + if (g_args.avro) { + if (0 != convertTbDesToAvroSchema( + dbName, tbName, tableDes, colCount, &jsonAvroSchema)) { + freeTbDes(tableDes); + return -1; + } + } - if (0 != code) { - errorPrint("%s() LN%d, failed to run command: %s, reason: %s\n", - __func__, __LINE__, command, taos_errstr(result)); - return 0; + free(tableDes); + + int64_t ret = 0; + if (!g_args.schemaonly) { + ret = dumpTableData(fp, tbName, dbName, precision, + jsonAvroSchema); } - TAOS_FIELD *fields = taos_fetch_fields(result); + return ret; +} - while ((row = taos_fetch_row(result)) != NULL) { - // sys database name : 'log', but subsequent version changed to 'log' - if ((strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", - fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) - && (!g_args.allow_sys)) { - continue; - } +static int64_t dumpNormalTableBelongStb( + SDbInfo *dbInfo, char *stbName, char *ntbName) +{ + int64_t count = 0; - if (g_args.databases) { // input multi dbs - if (inDatabasesSeq( - (char *)row[TSDB_SHOW_DB_NAME_INDEX], - fields[TSDB_SHOW_DB_NAME_INDEX].bytes) != 0) - continue; - } else if (!g_args.all_databases) { // only input one db - if (strncasecmp(g_args.arg_list[0], - (char *)row[TSDB_SHOW_DB_NAME_INDEX], - fields[TSDB_SHOW_DB_NAME_INDEX].bytes) != 0) - continue; - } + char tmpBuf[4096] = {0}; + FILE *fp = NULL; - count++; + if (g_args.outpath[0] != 0) { + sprintf(tmpBuf, "%s/%s.%s.sql", + g_args.outpath, dbInfo->name, ntbName); + } else { + sprintf(tmpBuf, "%s.%s.sql", + dbInfo->name, ntbName); } - if (count == 0) { - errorPrint("%d databases valid to dump\n", count); + fp = fopen(tmpBuf, "w"); + if (fp == NULL) { + errorPrint("%s() LN%d, failed to open file %s\n", + __func__, __LINE__, tmpBuf); + return -1; } + count = dumpNormalTable( + dbInfo->name, + stbName, + ntbName, + getPrecisionByString(dbInfo->precision), + fp); + + fclose(fp); return count; } -static int taosDumpOut() { - TAOS *taos = NULL; - TAOS_RES *result = NULL; - char *command = NULL; +static int64_t dumpNormalTableWithoutStb(SDbInfo *dbInfo, char *ntbName) +{ + int64_t count = 0; - TAOS_ROW row; + char tmpBuf[4096] = {0}; FILE *fp = NULL; - int32_t count = 0; - STableRecordInfo tableRecordInfo; - char tmpBuf[4096] = {0}; if (g_args.outpath[0] != 0) { - sprintf(tmpBuf, "%s/dbs.sql", g_args.outpath); + sprintf(tmpBuf, "%s/%s.%s.sql", + g_args.outpath, dbInfo->name, ntbName); } else { - sprintf(tmpBuf, "dbs.sql"); + sprintf(tmpBuf, "%s.%s.sql", + dbInfo->name, ntbName); } fp = fopen(tmpBuf, "w"); @@ -1297,66 +1243,527 @@ static int taosDumpOut() { return -1; } - g_args.dbCount = getDbCount(); + count = dumpNormalTable( + dbInfo->name, + NULL, + ntbName, + getPrecisionByString(dbInfo->precision), + fp); - if (0 == g_args.dbCount) { - fclose(fp); - errorPrint("%d databases valid to dump\n", g_args.dbCount); - return -1; - } + fclose(fp); + return count; +} - g_dbInfos = (SDbInfo **)calloc(g_args.dbCount, sizeof(SDbInfo *)); - if (g_dbInfos == NULL) { - errorPrint("%s() LN%d, failed to allocate memory\n", - __func__, __LINE__); - goto _exit_failure; +static void *dumpNtbOfDb(void *arg) { + threadInfo *pThreadInfo = (threadInfo *)arg; + + debugPrint("dump table from = \t%"PRId64"\n", pThreadInfo->tableFrom); + debugPrint("dump table count = \t%"PRId64"\n", + pThreadInfo->tablesOfDumpOut); + + FILE *fp = NULL; + char tmpBuf[4096] = {0}; + + if (g_args.outpath[0] != 0) { + sprintf(tmpBuf, "%s/%s.%d.sql", + g_args.outpath, pThreadInfo->dbName, pThreadInfo->threadIndex); + } else { + sprintf(tmpBuf, "%s.%d.sql", + pThreadInfo->dbName, pThreadInfo->threadIndex); } - command = (char *)malloc(COMMAND_SIZE); - if (command == NULL) { - errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__); - goto _exit_failure; + fp = fopen(tmpBuf, "w"); + + if (fp == NULL) { + errorPrint("%s() LN%d, failed to open file %s\n", + __func__, __LINE__, tmpBuf); + return NULL; } - /* Connect to server */ - taos = taos_connect(g_args.host, g_args.user, g_args.password, - NULL, g_args.port); - if (taos == NULL) { - errorPrint("Failed to connect to TDengine server %s\n", g_args.host); - goto _exit_failure; + for (int64_t i = 0; i < pThreadInfo->tablesOfDumpOut; i++) { + debugPrint("[%d] No.\t%"PRId64" table name: %s\n", + pThreadInfo->threadIndex, i, + ((TableInfo *)(g_tablesList + pThreadInfo->tableFrom+i))->name); + dumpNormalTable( + pThreadInfo->dbName, + ((TableInfo *)(g_tablesList + pThreadInfo->tableFrom+i))->stable, + ((TableInfo *)(g_tablesList + pThreadInfo->tableFrom+i))->name, + pThreadInfo->precision, + fp); } - /* --------------------------------- Main Code -------------------------------- */ - /* if (g_args.databases || g_args.all_databases) { // dump part of databases or all databases */ - /* */ - taosDumpCharset(fp); + fclose(fp); - sprintf(command, "show databases"); - result = taos_query(taos, command); - int32_t code = taos_errno(result); + return NULL; +} - if (code != 0) { - errorPrint("%s() LN%d, failed to run command: %s, reason: %s\n", - __func__, __LINE__, command, taos_errstr(result)); - goto _exit_failure; - } +static void *dumpNormalTablesOfStb(void *arg) { + threadInfo *pThreadInfo = (threadInfo *)arg; - TAOS_FIELD *fields = taos_fetch_fields(result); + debugPrint("dump table from = \t%"PRId64"\n", pThreadInfo->tableFrom); + debugPrint("dump table count = \t%"PRId64"\n", pThreadInfo->tablesOfDumpOut); - while ((row = taos_fetch_row(result)) != NULL) { - // sys database name : 'log', but subsequent version changed to 'log' - if ((strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", - fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) - && (!g_args.allow_sys)) { - continue; - } + char command[COMMAND_SIZE]; - if (g_args.databases) { // input multi dbs - if (inDatabasesSeq( - (char *)row[TSDB_SHOW_DB_NAME_INDEX], - fields[TSDB_SHOW_DB_NAME_INDEX].bytes) != 0) { - continue; - } + sprintf(command, "SELECT TBNAME FROM %s.%s LIMIT %"PRId64" OFFSET %"PRId64"", + pThreadInfo->dbName, pThreadInfo->stbName, + pThreadInfo->tablesOfDumpOut, pThreadInfo->tableFrom); + + TAOS_RES *res = taos_query(pThreadInfo->taos, command); + int32_t code = taos_errno(res); + if (code) { + errorPrint("%s() LN%d, failed to run command <%s>. reason: %s\n", + __func__, __LINE__, command, taos_errstr(res)); + taos_free_result(res); + return NULL; + } + + FILE *fp = NULL; + char tmpBuf[4096] = {0}; + + if (g_args.outpath[0] != 0) { + sprintf(tmpBuf, "%s/%s.%d.sql", + g_args.outpath, pThreadInfo->dbName, pThreadInfo->threadIndex); + } else { + sprintf(tmpBuf, "%s.%d.sql", + pThreadInfo->dbName, pThreadInfo->threadIndex); + } + + fp = fopen(tmpBuf, "w"); + + if (fp == NULL) { + errorPrint("%s() LN%d, failed to open file %s\n", + __func__, __LINE__, tmpBuf); + return NULL; + } + + TAOS_ROW row = NULL; + int64_t i = 0; + while((row = taos_fetch_row(res)) != NULL) { + debugPrint("[%d] sub table %"PRId64": name: %s\n", + pThreadInfo->threadIndex, i++, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX]); + + dumpNormalTable( + pThreadInfo->dbName, + pThreadInfo->stbName, + (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], + pThreadInfo->precision, + fp); + } + + fclose(fp); + return NULL; +} + +static int64_t dumpNtbOfDbByThreads( + SDbInfo *dbInfo, + int64_t ntbCount) +{ + if (ntbCount <= 0) { + return 0; + } + + int threads = g_args.thread_num; + + int64_t a = ntbCount / threads; + if (a < 1) { + threads = ntbCount; + a = 1; + } + + assert(threads); + int64_t b = ntbCount % threads; + + threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); + pthread_t *pids = calloc(1, threads * sizeof(pthread_t)); + assert(pids); + assert(infos); + + for (int64_t i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + pThreadInfo->taos = taos_connect( + g_args.host, + g_args.user, + g_args.password, + dbInfo->name, + g_args.port + ); + if (NULL == pThreadInfo->taos) { + errorPrint("%s() LN%d, Failed to connect to TDengine, reason: %s\n", + __func__, + __LINE__, + taos_errstr(NULL)); + free(pids); + free(infos); + + return -1; + } + + pThreadInfo->threadIndex = i; + pThreadInfo->tablesOfDumpOut = (itableFrom = (i==0)?0: + ((threadInfo *)(infos + i - 1))->tableFrom + + ((threadInfo *)(infos + i - 1))->tablesOfDumpOut; + strcpy(pThreadInfo->dbName, dbInfo->name); + pThreadInfo->precision = getPrecisionByString(dbInfo->precision); + + pthread_create(pids + i, NULL, dumpNtbOfDb, pThreadInfo); + } + + for (int64_t i = 0; i < threads; i++) { + pthread_join(pids[i], NULL); + } + + for (int64_t i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + taos_close(pThreadInfo->taos); + } + + free(pids); + free(infos); + + return 0; +} + +static int64_t getNtbCountOfStb(char *dbName, char *stbName) +{ + TAOS *taos = taos_connect(g_args.host, g_args.user, g_args.password, + dbName, g_args.port); + if (taos == NULL) { + errorPrint("Failed to connect to TDengine server %s\n", g_args.host); + return -1; + } + + int64_t count = 0; + + char command[COMMAND_SIZE]; + + sprintf(command, "SELECT COUNT(TBNAME) FROM %s.%s", dbName, stbName); + + TAOS_RES *res = taos_query(taos, command); + int32_t code = taos_errno(res); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command <%s>. reason: %s\n", + __func__, __LINE__, command, taos_errstr(res)); + taos_free_result(res); + taos_close(taos); + return -1; + } + + TAOS_ROW row = NULL; + + if ((row = taos_fetch_row(res)) != NULL) { + count = *(int64_t*)row[TSDB_SHOW_TABLES_NAME_INDEX]; + } + + taos_close(taos); + return count; +} + +static int64_t dumpNtbOfStbByThreads( + SDbInfo *dbInfo, char *stbName) +{ + int64_t ntbCount = getNtbCountOfStb(dbInfo->name, stbName); + + if (ntbCount <= 0) { + return 0; + } + + int threads = g_args.thread_num; + + int64_t a = ntbCount / threads; + if (a < 1) { + threads = ntbCount; + a = 1; + } + + assert(threads); + int64_t b = ntbCount % threads; + + pthread_t *pids = calloc(1, threads * sizeof(pthread_t)); + threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); + assert(pids); + assert(infos); + + for (int64_t i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + pThreadInfo->taos = taos_connect( + g_args.host, + g_args.user, + g_args.password, + dbInfo->name, + g_args.port + ); + if (NULL == pThreadInfo->taos) { + errorPrint("%s() LN%d, Failed to connect to TDengine, reason: %s\n", + __func__, + __LINE__, + taos_errstr(NULL)); + free(pids); + free(infos); + + return -1; + } + + pThreadInfo->threadIndex = i; + pThreadInfo->tablesOfDumpOut = (itableFrom = (i==0)?0: + ((threadInfo *)(infos + i - 1))->tableFrom + + ((threadInfo *)(infos + i - 1))->tablesOfDumpOut; + strcpy(pThreadInfo->dbName, dbInfo->name); + pThreadInfo->precision = getPrecisionByString(dbInfo->precision); + + strcpy(pThreadInfo->stbName, stbName); + pthread_create(pids + i, NULL, dumpNormalTablesOfStb, pThreadInfo); + } + + for (int64_t i = 0; i < threads; i++) { + pthread_join(pids[i], NULL); + } + + int64_t records = 0; + for (int64_t i = 0; i < threads; i++) { + threadInfo *pThreadInfo = infos + i; + records += pThreadInfo->rowsOfDumpOut; + taos_close(pThreadInfo->taos); + } + + free(pids); + free(infos); + + return records; +} + +static int dumpStableClasuse(SDbInfo *dbInfo, char *stbName, FILE *fp) +{ + uint64_t sizeOfTableDes = + (uint64_t)(sizeof(TableDef) + sizeof(ColDes) * TSDB_MAX_COLUMNS); + + TableDef *tableDes = (TableDef *)calloc(1, sizeOfTableDes); + if (NULL == tableDes) { + errorPrint("%s() LN%d, failed to allocate %"PRIu64" memory\n", + __func__, __LINE__, sizeOfTableDes); + exit(-1); + } + + int colCount = getTableDes(dbInfo->name, + stbName, tableDes, true); + + if (colCount < 0) { + free(tableDes); + errorPrint("%s() LN%d, failed to get stable[%s] schema\n", + __func__, __LINE__, stbName); + exit(-1); + } + + dumpCreateTableClause(tableDes, colCount, fp, dbInfo->name); + free(tableDes); + + return 0; +} + +static int64_t dumpCreateSTableClauseOfDb( + SDbInfo *dbInfo, FILE *fp) +{ + TAOS *taos = taos_connect(g_args.host, + g_args.user, g_args.password, dbInfo->name, g_args.port); + if (NULL == taos) { + errorPrint( + "Failed to connect to TDengine server %s by specified database %s\n", + g_args.host, dbInfo->name); + return 0; + } + + TAOS_ROW row; + char command[COMMAND_SIZE] = {0}; + + sprintf(command, "SHOW %s.STABLES", dbInfo->name); + + TAOS_RES* res = taos_query(taos, command); + int32_t code = taos_errno(res); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command <%s>, reason: %s\n", + __func__, __LINE__, command, taos_errstr(res)); + taos_free_result(res); + taos_close(taos); + exit(-1); + } + + int64_t superTblCnt = 0; + while ((row = taos_fetch_row(res)) != NULL) { + if (0 == dumpStableClasuse(dbInfo, row[TSDB_SHOW_TABLES_NAME_INDEX], fp)) { + superTblCnt ++; + } + } + + taos_free_result(res); + + fprintf(g_fpOfResult, + "# super table counter: %"PRId64"\n", + superTblCnt); + g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; + + taos_close(taos); + + return superTblCnt; +} + +static int64_t dumpNTablesOfDb(SDbInfo *dbInfo) +{ + TAOS *taos = taos_connect(g_args.host, + g_args.user, g_args.password, dbInfo->name, g_args.port); + if (NULL == taos) { + errorPrint( + "Failed to connect to TDengine server %s by specified database %s\n", + g_args.host, dbInfo->name); + return 0; + } + + char command[COMMAND_SIZE]; + TAOS_RES *result; + int32_t code; + + sprintf(command, "USE %s", dbInfo->name); + result = taos_query(taos, command); + code = taos_errno(result); + if (code != 0) { + errorPrint("invalid database %s, reason: %s\n", + dbInfo->name, taos_errstr(result)); + taos_close(taos); + return 0; + } + + sprintf(command, "SHOW TABLES"); + result = taos_query(taos, command); + code = taos_errno(result); + if (code != 0) { + errorPrint("Failed to show %s\'s tables, reason: %s\n", + dbInfo->name, taos_errstr(result)); + taos_close(taos); + return 0; + } + + g_tablesList = calloc(1, dbInfo->ntables * sizeof(TableInfo)); + + TAOS_ROW row; + int64_t count = 0; + while(NULL != (row = taos_fetch_row(result))) { + debugPrint("%s() LN%d, No.\t%"PRId64" table name: %s\n", + __func__, __LINE__, + count, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX]); + tstrncpy(((TableInfo *)(g_tablesList + count))->name, + (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], TSDB_TABLE_NAME_LEN); + char *stbName = (char *) row[TSDB_SHOW_TABLES_METRIC_INDEX]; + if (stbName) { + tstrncpy(((TableInfo *)(g_tablesList + count))->stable, + (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], TSDB_TABLE_NAME_LEN); + ((TableInfo *)(g_tablesList + count))->belongStb = true; + } + count ++; + } + taos_close(taos); + + int64_t records = dumpNtbOfDbByThreads(dbInfo, count); + + free(g_tablesList); + g_tablesList = NULL; + + return records; +} + +static int64_t dumpWholeDatabase(SDbInfo *dbInfo, FILE *fp) +{ + dumpCreateDbClause(dbInfo, g_args.with_property, fp); + + fprintf(g_fpOfResult, "\n#### database: %s\n", + dbInfo->name); + g_resultStatistics.totalDatabasesOfDumpOut++; + + dumpCreateSTableClauseOfDb(dbInfo, fp); + + return dumpNTablesOfDb(dbInfo); +} + +static int dumpOut() { + TAOS *taos = NULL; + TAOS_RES *result = NULL; + + TAOS_ROW row; + FILE *fp = NULL; + int32_t count = 0; + + char tmpBuf[4096] = {0}; + if (g_args.outpath[0] != 0) { + sprintf(tmpBuf, "%s/dbs.sql", g_args.outpath); + } else { + sprintf(tmpBuf, "dbs.sql"); + } + + fp = fopen(tmpBuf, "w"); + if (fp == NULL) { + errorPrint("%s() LN%d, failed to open file %s\n", + __func__, __LINE__, tmpBuf); + return -1; + } + + g_args.dumpDbCount = getDumpDbCount(); + debugPrint("%s() LN%d, dump db count: %d\n", + __func__, __LINE__, g_args.dumpDbCount); + + if (0 == g_args.dumpDbCount) { + errorPrint("%d databases valid to dump\n", g_args.dumpDbCount); + fclose(fp); + return -1; + } + + g_dbInfos = (SDbInfo **)calloc(g_args.dumpDbCount, sizeof(SDbInfo *)); + if (g_dbInfos == NULL) { + errorPrint("%s() LN%d, failed to allocate memory\n", + __func__, __LINE__); + goto _exit_failure; + } + + char command[COMMAND_SIZE]; + + /* Connect to server */ + taos = taos_connect(g_args.host, g_args.user, g_args.password, + NULL, g_args.port); + if (taos == NULL) { + errorPrint("Failed to connect to TDengine server %s\n", g_args.host); + goto _exit_failure; + } + + /* --------------------------------- Main Code -------------------------------- */ + /* if (g_args.databases || g_args.all_databases) { // dump part of databases or all databases */ + /* */ + dumpCharset(fp); + + sprintf(command, "show databases"); + result = taos_query(taos, command); + int32_t code = taos_errno(result); + + if (code != 0) { + errorPrint("%s() LN%d, failed to run command <%s>, reason: %s\n", + __func__, __LINE__, command, taos_errstr(result)); + goto _exit_failure; + } + + TAOS_FIELD *fields = taos_fetch_fields(result); + + while ((row = taos_fetch_row(result)) != NULL) { + // sys database name : 'log', but subsequent version changed to 'log' + if ((strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", + fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) + && (!g_args.allow_sys)) { + continue; + } + + if (g_args.databases) { // input multi dbs + if (inDatabasesSeq( + (char *)row[TSDB_SHOW_DB_NAME_INDEX], + fields[TSDB_SHOW_DB_NAME_INDEX].bytes) != 0) { + continue; + } } else if (!g_args.all_databases) { // only input one db if (strncasecmp(g_args.arg_list[0], (char *)row[TSDB_SHOW_DB_NAME_INDEX], @@ -1419,10 +1826,11 @@ static int taosDumpOut() { count++; if (g_args.databases) { - if (count > g_args.dbCount) break; - + if (count > g_args.dumpDbCount) + break; } else if (!g_args.all_databases) { - if (count >= 1) break; + if (count >= 1) + break; } } @@ -1431,104 +1839,74 @@ static int taosDumpOut() { goto _exit_failure; } - if (g_args.databases || g_args.all_databases) { // case: taosdump --databases dbx dby ... OR taosdump --all-databases - for (int i = 0; i < count; i++) { - taosDumpDb(g_dbInfos[i], fp, taos); + taos_close(taos); + + if (g_args.databases || g_args.all_databases) { // case: taosdump --databases dbx,dby ... OR taosdump --all-databases + for (int i = 0; i < count; i++) { + int64_t records = 0; + records = dumpWholeDatabase(g_dbInfos[i], fp); + if (records >= 0) { + okPrint("Database %s dumped\n", g_dbInfos[i]->name); + g_totalDumpOutRows += records; + } } } else { - if (g_args.dbCount == 1) { // case: taosdump - taosDumpDb(g_dbInfos[0], fp, taos); - } else { // case: taosdump tablex tabley ... - taosDumpCreateDbClause(g_dbInfos[0], g_args.with_property, fp); - fprintf(g_fpOfResult, "\n#### database: %s\n", - g_dbInfos[0]->name); - g_resultStatistics.totalDatabasesOfDumpOut++; - - sprintf(command, "use %s", g_dbInfos[0]->name); - - result = taos_query(taos, command); - code = taos_errno(result); - if (code != 0) { - errorPrint("invalid database %s\n", g_dbInfos[0]->name); - goto _exit_failure; + if (1 == g_args.arg_list_len) { + int64_t records = dumpWholeDatabase(g_dbInfos[0], fp); + if (records >= 0) { + okPrint("Database %s dumped\n", g_dbInfos[0]->name); + g_totalDumpOutRows += records; } + } else { + dumpCreateDbClause(g_dbInfos[0], g_args.with_property, fp); + } - fprintf(fp, "USE %s;\n\n", g_dbInfos[0]->name); - - int32_t totalNumOfThread = 1; // 0: all normal table into .tables.tmp.0 - int normalTblFd = -1; - int32_t retCode; - int superTblCnt = 0 ; - for (int i = 1; g_args.arg_list[i]; i++) { - if (taosGetTableRecordInfo(g_args.arg_list[i], - &tableRecordInfo, taos) < 0) { - errorPrint("input the invalid table %s\n", - g_args.arg_list[i]); - continue; - } - - if (tableRecordInfo.isMetric) { // dump all table of this metric - int ret = taosDumpStable( - tableRecordInfo.tableRecord.metric, - fp, taos, g_dbInfos[0]->name); - if (0 == ret) { - superTblCnt++; - } - retCode = taosSaveTableOfMetricToTempFile( - taos, tableRecordInfo.tableRecord.metric, - &totalNumOfThread); - } else { - if (tableRecordInfo.tableRecord.metric[0] != '\0') { // dump this sub table and it's metric - int ret = taosDumpStable( - tableRecordInfo.tableRecord.metric, - fp, taos, g_dbInfos[0]->name); - if (0 == ret) { - superTblCnt++; - } - } - retCode = taosSaveAllNormalTableToTempFile( - taos, tableRecordInfo.tableRecord.name, - tableRecordInfo.tableRecord.metric, &normalTblFd); - } + int superTblCnt = 0 ; + for (int i = 1; g_args.arg_list[i]; i++) { + TableRecordInfo tableRecordInfo; - if (retCode < 0) { - if (-1 != normalTblFd){ - taosClose(normalTblFd); - } - goto _clean_tmp_file; - } + if (getTableRecordInfo(g_dbInfos[0]->name, + g_args.arg_list[i], + &tableRecordInfo) < 0) { + errorPrint("input the invalid table %s\n", + g_args.arg_list[i]); + continue; } - // TODO: save dump super table into result_output.txt - fprintf(g_fpOfResult, "# super table counter: %d\n", - superTblCnt); - g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; - - if (-1 != normalTblFd){ - taosClose(normalTblFd); + int64_t records = 0; + if (tableRecordInfo.isStb) { // dump all table of this stable + int ret = dumpStableClasuse( + g_dbInfos[0], + tableRecordInfo.tableRecord.stable, + fp); + if (ret >= 0) { + superTblCnt++; + records = dumpNtbOfStbByThreads(g_dbInfos[0], g_args.arg_list[i]); + } + } else if (tableRecordInfo.belongStb){ + dumpStableClasuse( + g_dbInfos[0], + tableRecordInfo.tableRecord.stable, + fp); + records = dumpNormalTableBelongStb( + g_dbInfos[0], + tableRecordInfo.tableRecord.stable, + g_args.arg_list[i]); + } else { + records = dumpNormalTableWithoutStb(g_dbInfos[0], g_args.arg_list[i]); } - // start multi threads to dumpout - - taosStartDumpOutWorkThreads(totalNumOfThread, - g_dbInfos[0]->name, - getPrecisionByString(g_dbInfos[0]->precision)); - - char tmpFileName[MAX_FILE_NAME_LEN]; -_clean_tmp_file: - for (int loopCnt = 0; loopCnt < totalNumOfThread; loopCnt++) { - sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); - remove(tmpFileName); + if (records >= 0) { + okPrint("table: %s dumped\n", g_args.arg_list[i]); + g_totalDumpOutRows += records; } } } /* Close the handle and return */ fclose(fp); - taos_close(taos); taos_free_result(result); - tfree(command); - taosFreeDbInfos(); + freeDbInfos(); fprintf(stderr, "dump out rows: %" PRId64 "\n", g_totalDumpOutRows); return 0; @@ -1536,656 +1914,221 @@ _exit_failure: fclose(fp); taos_close(taos); taos_free_result(result); - tfree(command); - taosFreeDbInfos(); + freeDbInfos(); errorPrint("dump out rows: %" PRId64 "\n", g_totalDumpOutRows); return -1; } -static int taosGetTableDes( +static int getTableDes( char* dbName, char *table, - STableDef *stableDes, TAOS* taosCon, bool isSuperTable) { + TableDef *tableDes, bool isSuperTable) { TAOS_ROW row = NULL; TAOS_RES* res = NULL; - int count = 0; + int colCount = 0; - char sqlstr[COMMAND_SIZE]; - sprintf(sqlstr, "describe %s.%s;", dbName, table); - - res = taos_query(taosCon, sqlstr); - int32_t code = taos_errno(res); - if (code != 0) { - errorPrint("%s() LN%d, failed to run command <%s>, reason:%s\n", - __func__, __LINE__, sqlstr, taos_errstr(res)); - taos_free_result(res); + TAOS *taos = taos_connect(g_args.host, + g_args.user, g_args.password, dbName, g_args.port); + if (NULL == taos) { + errorPrint( + "Failed to connect to TDengine server %s by specified database %s\n", + g_args.host, dbName); return -1; } - TAOS_FIELD *fields = taos_fetch_fields(res); - - tstrncpy(stableDes->name, table, TSDB_TABLE_NAME_LEN); - while ((row = taos_fetch_row(res)) != NULL) { - tstrncpy(stableDes->cols[count].field, - (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], - min(TSDB_COL_NAME_LEN + 1, - fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes + 1)); - tstrncpy(stableDes->cols[count].type, - (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - min(32, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes + 1)); - stableDes->cols[count].length = - *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - tstrncpy(stableDes->cols[count].note, - (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], - min(COL_NOTE_LEN, - fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes + 1)); - - count++; - } - - taos_free_result(res); - res = NULL; - - if (isSuperTable) { - return count; - } - - // if child-table have tag, using select tagName from table to get tagValue - for (int i = 0 ; i < count; i++) { - if (strcmp(stableDes->cols[i].note, "TAG") != 0) continue; - - sprintf(sqlstr, "select %s from %s.%s", - stableDes->cols[i].field, dbName, table); - - res = taos_query(taosCon, sqlstr); - code = taos_errno(res); - if (code != 0) { - errorPrint("%s() LN%d, failed to run command <%s>, reason:%s\n", - __func__, __LINE__, sqlstr, taos_errstr(res)); - taos_free_result(res); - return -1; - } - - fields = taos_fetch_fields(res); - - row = taos_fetch_row(res); - if (NULL == row) { - errorPrint("%s() LN%d, fetch failed to run command <%s>, reason:%s\n", - __func__, __LINE__, sqlstr, taos_errstr(res)); - taos_free_result(res); - return -1; - } - - if (row[0] == NULL) { - sprintf(stableDes->cols[i].note, "%s", "NULL"); - taos_free_result(res); - res = NULL; - continue; - } - - int32_t* length = taos_fetch_lengths(res); - - //int32_t* length = taos_fetch_lengths(tmpResult); - switch (fields[0].type) { - case TSDB_DATA_TYPE_BOOL: - sprintf(stableDes->cols[i].note, "%d", - ((((int32_t)(*((char *)row[0]))) == 1) ? 1 : 0)); - break; - case TSDB_DATA_TYPE_TINYINT: - sprintf(stableDes->cols[i].note, "%d", *((int8_t *)row[0])); - break; - case TSDB_DATA_TYPE_SMALLINT: - sprintf(stableDes->cols[i].note, "%d", *((int16_t *)row[0])); - break; - case TSDB_DATA_TYPE_INT: - sprintf(stableDes->cols[i].note, "%d", *((int32_t *)row[0])); - break; - case TSDB_DATA_TYPE_BIGINT: - sprintf(stableDes->cols[i].note, "%" PRId64 "", *((int64_t *)row[0])); - break; - case TSDB_DATA_TYPE_UTINYINT: - sprintf(stableDes->cols[i].note, "%d", *((uint8_t *)row[0])); - break; - case TSDB_DATA_TYPE_USMALLINT: - sprintf(stableDes->cols[i].note, "%d", *((uint16_t *)row[0])); - break; - case TSDB_DATA_TYPE_UINT: - sprintf(stableDes->cols[i].note, "%" PRIu32 "", *((uint32_t *)row[0])); - break; - case TSDB_DATA_TYPE_UBIGINT: - sprintf(stableDes->cols[i].note, "%" PRIu64 "", *((uint64_t *)row[0])); - break; - case TSDB_DATA_TYPE_FLOAT: - sprintf(stableDes->cols[i].note, "%f", GET_FLOAT_VAL(row[0])); - break; - case TSDB_DATA_TYPE_DOUBLE: - sprintf(stableDes->cols[i].note, "%f", GET_DOUBLE_VAL(row[0])); - break; - case TSDB_DATA_TYPE_BINARY: - { - memset(stableDes->cols[i].note, 0, sizeof(stableDes->cols[i].note)); - stableDes->cols[i].note[0] = '\''; - char tbuf[COL_NOTE_LEN]; - converStringToReadable((char *)row[0], length[0], tbuf, COL_NOTE_LEN); - char* pstr = stpcpy(&(stableDes->cols[i].note[1]), tbuf); - *(pstr++) = '\''; - break; - } - case TSDB_DATA_TYPE_NCHAR: - { - memset(stableDes->cols[i].note, 0, sizeof(stableDes->cols[i].note)); - char tbuf[COL_NOTE_LEN-2]; // need reserve 2 bytes for ' ' - convertNCharToReadable((char *)row[0], length[0], tbuf, COL_NOTE_LEN); - sprintf(stableDes->cols[i].note, "\'%s\'", tbuf); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - sprintf(stableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]); -#if 0 - if (!g_args.mysqlFlag) { - sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]); - } else { - char buf[64] = "\0"; - int64_t ts = *((int64_t *)row[0]); - time_t tt = (time_t)(ts / 1000); - struct tm *ptm = localtime(&tt); - strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm); - sprintf(tableDes->cols[i].note, "\'%s.%03d\'", buf, (int)(ts % 1000)); - } -#endif - break; - default: - break; - } - - taos_free_result(res); - res = NULL; - } - - return count; -} - -static int convertSchemaToAvroSchema(STableDef *stableDes, char **avroSchema) -{ - errorPrint("%s() LN%d TODO: covert table schema to avro schema\n", - __func__, __LINE__); - return 0; -} - -static int32_t taosDumpTable( - char *tbName, char *metric, - FILE *fp, TAOS* taosCon, char* dbName, int precision) { - int count = 0; - - STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) - + sizeof(SColDes) * TSDB_MAX_COLUMNS); - - if (metric != NULL && metric[0] != '\0') { // dump table schema which is created by using super table - /* - count = taosGetTableDes(metric, tableDes, taosCon); - - if (count < 0) { - free(tableDes); - return -1; - } - - taosDumpCreateTableClause(tableDes, count, fp); - - memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); - */ - - count = taosGetTableDes(dbName, tbName, tableDes, taosCon, false); - - if (count < 0) { - free(tableDes); - return -1; - } - - // create child-table using super-table - taosDumpCreateMTableClause(tableDes, metric, count, fp, dbName); - - } else { // dump table definition - count = taosGetTableDes(dbName, tbName, tableDes, taosCon, false); - - if (count < 0) { - free(tableDes); - return -1; - } - - // create normal-table or super-table - taosDumpCreateTableClause(tableDes, count, fp, dbName); - } - - char *jsonAvroSchema = NULL; - if (g_args.avro) { - convertSchemaToAvroSchema(tableDes, &jsonAvroSchema); - } - - free(tableDes); - - int32_t ret = 0; - if (!g_args.schemaonly) { - ret = taosDumpTableData(fp, tbName, taosCon, dbName, precision, - jsonAvroSchema); - } - - return ret; -} - -static void taosDumpCreateDbClause( - SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { - char sqlstr[TSDB_MAX_SQL_LEN] = {0}; - - char *pstr = sqlstr; - pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s ", dbInfo->name); - if (isDumpProperty) { - pstr += sprintf(pstr, - "REPLICA %d QUORUM %d DAYS %d KEEP %s CACHE %d BLOCKS %d MINROWS %d MAXROWS %d FSYNC %d CACHELAST %d COMP %d PRECISION '%s' UPDATE %d", - dbInfo->replica, dbInfo->quorum, dbInfo->days, - dbInfo->keeplist, - dbInfo->cache, - dbInfo->blocks, dbInfo->minrows, dbInfo->maxrows, - dbInfo->fsync, - dbInfo->cachelast, - dbInfo->comp, dbInfo->precision, dbInfo->update); - } - - pstr += sprintf(pstr, ";"); - fprintf(fp, "%s\n\n", sqlstr); -} - -static void* taosDumpOutWorkThreadFp(void *arg) -{ - SThreadParaObj *pThread = (SThreadParaObj*)arg; - STableRecord tableRecord; - int fd; - - setThreadName("dumpOutWorkThrd"); - - char tmpBuf[4096] = {0}; - sprintf(tmpBuf, ".tables.tmp.%d", pThread->threadIndex); - fd = open(tmpBuf, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - errorPrint("%s() LN%d, failed to open temp file: %s\n", - __func__, __LINE__, tmpBuf); - return NULL; - } - - FILE *fp = NULL; - memset(tmpBuf, 0, 4096); - - if (g_args.outpath[0] != 0) { - sprintf(tmpBuf, "%s/%s.tables.%d.sql", - g_args.outpath, pThread->dbName, pThread->threadIndex); - } else { - sprintf(tmpBuf, "%s.tables.%d.sql", - pThread->dbName, pThread->threadIndex); - } - - fp = fopen(tmpBuf, "w"); - if (fp == NULL) { - errorPrint("%s() LN%d, failed to open file %s\n", - __func__, __LINE__, tmpBuf); - close(fd); - return NULL; - } - - memset(tmpBuf, 0, 4096); - sprintf(tmpBuf, "use %s", pThread->dbName); - - TAOS_RES* tmpResult = taos_query(pThread->taosCon, tmpBuf); - int32_t code = taos_errno(tmpResult); - if (code != 0) { - errorPrint("%s() LN%d, invalid database %s. reason: %s\n", - __func__, __LINE__, pThread->dbName, taos_errstr(tmpResult)); - taos_free_result(tmpResult); - fclose(fp); - close(fd); - return NULL; - } - -#if 0 - int fileNameIndex = 1; - int tablesInOneFile = 0; -#endif - int64_t lastRowsPrint = 5000000; - fprintf(fp, "USE %s;\n\n", pThread->dbName); - while (1) { - ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); - if (readLen <= 0) break; - - int ret = taosDumpTable( - tableRecord.name, tableRecord.metric, - fp, pThread->taosCon, pThread->dbName, - pThread->precision); - if (ret >= 0) { - // TODO: sum table count and table rows by self - pThread->tablesOfDumpOut++; - pThread->rowsOfDumpOut += ret; - - if (pThread->rowsOfDumpOut >= lastRowsPrint) { - printf(" %"PRId64 " rows already be dumpout from database %s\n", - pThread->rowsOfDumpOut, pThread->dbName); - lastRowsPrint += 5000000; - } - -#if 0 - tablesInOneFile++; - if (tablesInOneFile >= g_args.table_batch) { - fclose(fp); - tablesInOneFile = 0; - - memset(tmpBuf, 0, 4096); - if (g_args.outpath[0] != 0) { - sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", - g_args.outpath, pThread->dbName, - pThread->threadIndex, fileNameIndex); - } else { - sprintf(tmpBuf, "%s.tables.%d-%d.sql", - pThread->dbName, pThread->threadIndex, fileNameIndex); - } - fileNameIndex++; - - fp = fopen(tmpBuf, "w"); - if (fp == NULL) { - errorPrint("%s() LN%d, failed to open file %s\n", - __func__, __LINE__, tmpBuf); - close(fd); - taos_free_result(tmpResult); - return NULL; - } - } -#endif - } - } - - taos_free_result(tmpResult); - close(fd); - fclose(fp); - - return NULL; -} - -static void taosStartDumpOutWorkThreads(int32_t numOfThread, char *dbName, int precision) -{ - pthread_attr_t thattr; - SThreadParaObj *threadObj = - (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); - - if (threadObj == NULL) { - errorPrint("%s() LN%d, memory allocation failed!\n", - __func__, __LINE__); - return; - } - - for (int t = 0; t < numOfThread; ++t) { - SThreadParaObj *pThread = threadObj + t; - pThread->rowsOfDumpOut = 0; - pThread->tablesOfDumpOut = 0; - pThread->threadIndex = t; - pThread->totalThreads = numOfThread; - tstrncpy(pThread->dbName, dbName, TSDB_DB_NAME_LEN); - pThread->precision = precision; - pThread->taosCon = taos_connect(g_args.host, g_args.user, g_args.password, - NULL, g_args.port); - if (pThread->taosCon == NULL) { - errorPrint("Failed to connect to TDengine server %s\n", g_args.host); - free(threadObj); - return; - } - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&(pThread->threadID), &thattr, - taosDumpOutWorkThreadFp, - (void*)pThread) != 0) { - errorPrint("%s() LN%d, thread:%d failed to start\n", - __func__, __LINE__, pThread->threadIndex); - exit(-1); - } - } - - for (int32_t t = 0; t < numOfThread; ++t) { - pthread_join(threadObj[t].threadID, NULL); - } - - // TODO: sum all thread dump table count and rows of per table, then save into result_output.txt - int64_t totalRowsOfDumpOut = 0; - int64_t totalChildTblsOfDumpOut = 0; - for (int32_t t = 0; t < numOfThread; ++t) { - totalChildTblsOfDumpOut += threadObj[t].tablesOfDumpOut; - totalRowsOfDumpOut += threadObj[t].rowsOfDumpOut; - } - - fprintf(g_fpOfResult, "# child table counter: %"PRId64"\n", - totalChildTblsOfDumpOut); - fprintf(g_fpOfResult, "# row counter: %"PRId64"\n", - totalRowsOfDumpOut); - g_resultStatistics.totalChildTblsOfDumpOut += totalChildTblsOfDumpOut; - g_resultStatistics.totalRowsOfDumpOut += totalRowsOfDumpOut; - free(threadObj); -} - -static int32_t taosDumpStable(char *table, FILE *fp, - TAOS* taosCon, char* dbName) { - - uint64_t sizeOfTableDes = - (uint64_t)(sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); - STableDef *stableDes = (STableDef *)calloc(1, sizeOfTableDes); - if (NULL == stableDes) { - errorPrint("%s() LN%d, failed to allocate %"PRIu64" memory\n", - __func__, __LINE__, sizeOfTableDes); - exit(-1); - } - - int count = taosGetTableDes(dbName, table, stableDes, taosCon, true); - - if (count < 0) { - free(stableDes); - errorPrint("%s() LN%d, failed to get stable[%s] schema\n", - __func__, __LINE__, table); - exit(-1); - } - - taosDumpCreateTableClause(stableDes, count, fp, dbName); - - free(stableDes); - return 0; -} - -static int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp) -{ - TAOS_ROW row; - int fd = -1; - STableRecord tableRecord; - char sqlstr[TSDB_MAX_SQL_LEN] = {0}; - - sprintf(sqlstr, "show %s.stables", dbName); - - TAOS_RES* res = taos_query(taosCon, sqlstr); - int32_t code = taos_errno(res); - if (code != 0) { - errorPrint("%s() LN%d, failed to run command <%s>, reason: %s\n", - __func__, __LINE__, sqlstr, taos_errstr(res)); - taos_free_result(res); - exit(-1); - } - - TAOS_FIELD *fields = taos_fetch_fields(res); - - char tmpFileName[MAX_FILE_NAME_LEN]; - memset(tmpFileName, 0, MAX_FILE_NAME_LEN); - sprintf(tmpFileName, ".stables.tmp"); - fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - errorPrint("%s() LN%d, failed to open temp file: %s\n", - __func__, __LINE__, tmpFileName); - taos_free_result(res); - (void)remove(".stables.tmp"); - exit(-1); - } - - while ((row = taos_fetch_row(res)) != NULL) { - memset(&tableRecord, 0, sizeof(STableRecord)); - tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], - min(TSDB_TABLE_NAME_LEN, - fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes + 1)); - taosWrite(fd, &tableRecord, sizeof(STableRecord)); - } - - taos_free_result(res); - (void)lseek(fd, 0, SEEK_SET); - - int superTblCnt = 0; - while (1) { - ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); - if (readLen <= 0) break; - - int ret = taosDumpStable(tableRecord.name, fp, taosCon, dbName); - if (0 == ret) { - superTblCnt++; - } - } - - // TODO: save dump super table into result_output.txt - fprintf(g_fpOfResult, "# super table counter: %d\n", superTblCnt); - g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; - - close(fd); - (void)remove(".stables.tmp"); - - return 0; -} - - -static int taosDumpDb(SDbInfo *dbInfo, FILE *fp, TAOS *taosCon) { - TAOS_ROW row; - int fd = -1; - STableRecord tableRecord; - - taosDumpCreateDbClause(dbInfo, g_args.with_property, fp); - - fprintf(g_fpOfResult, "\n#### database: %s\n", - dbInfo->name); - g_resultStatistics.totalDatabasesOfDumpOut++; - - char sqlstr[TSDB_MAX_SQL_LEN] = {0}; - - fprintf(fp, "USE %s;\n\n", dbInfo->name); - - (void)taosDumpCreateSuperTableClause(taosCon, dbInfo->name, fp); - - sprintf(sqlstr, "show %s.tables", dbInfo->name); - - TAOS_RES* res = taos_query(taosCon, sqlstr); - int code = taos_errno(res); - if (code != 0) { - errorPrint("%s() LN%d, failed to run command <%s>, reason:%s\n", - __func__, __LINE__, sqlstr, taos_errstr(res)); - taos_free_result(res); - return -1; - } + char sqlstr[COMMAND_SIZE]; + sprintf(sqlstr, "describe %s.%s;", dbName, table); - char tmpBuf[MAX_FILE_NAME_LEN]; - memset(tmpBuf, 0, MAX_FILE_NAME_LEN); - sprintf(tmpBuf, ".show-tables.tmp"); - fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - errorPrint("%s() LN%d, failed to open temp file: %s\n", - __func__, __LINE__, tmpBuf); + res = taos_query(taos, sqlstr); + int32_t code = taos_errno(res); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command <%s>, reason: %s\n", + __func__, __LINE__, sqlstr, taos_errstr(res)); taos_free_result(res); + taos_close(taos); return -1; } TAOS_FIELD *fields = taos_fetch_fields(res); - int32_t numOfTable = 0; + tstrncpy(tableDes->name, table, TSDB_TABLE_NAME_LEN); while ((row = taos_fetch_row(res)) != NULL) { - memset(&tableRecord, 0, sizeof(STableRecord)); - tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], - min(TSDB_TABLE_NAME_LEN, - fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes + 1)); - tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], - min(TSDB_TABLE_NAME_LEN, - fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes + 1)); + tstrncpy(tableDes->cols[colCount].field, + (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], + min(TSDB_COL_NAME_LEN + 1, + fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes + 1)); + tstrncpy(tableDes->cols[colCount].type, + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + min(16, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes + 1)); + tableDes->cols[colCount].length = + *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + tstrncpy(tableDes->cols[colCount].note, + (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], + min(COL_NOTE_LEN, + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes + 1)); + colCount++; + } - taosWrite(fd, &tableRecord, sizeof(STableRecord)); + taos_free_result(res); + res = NULL; - numOfTable++; + if (isSuperTable) { + return colCount; } - taos_free_result(res); - lseek(fd, 0, SEEK_SET); - int maxThreads = g_args.thread_num; - int tableOfPerFile ; - if (numOfTable <= g_args.thread_num) { - tableOfPerFile = 1; - maxThreads = numOfTable; - } else { - tableOfPerFile = numOfTable / g_args.thread_num; - if (0 != numOfTable % g_args.thread_num) { - tableOfPerFile += 1; + // if child-table have tag, using select tagName from table to get tagValue + for (int i = 0 ; i < colCount; i++) { + if (strcmp(tableDes->cols[i].note, "TAG") != 0) continue; + + sprintf(sqlstr, "select %s from %s.%s", + tableDes->cols[i].field, dbName, table); + + res = taos_query(taos, sqlstr); + code = taos_errno(res); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command <%s>, reason: %s\n", + __func__, __LINE__, sqlstr, taos_errstr(res)); + taos_free_result(res); + taos_close(taos); + return -1; } - } - char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord)); - if (NULL == tblBuf){ - errorPrint("failed to calloc %" PRIzu "\n", - tableOfPerFile * sizeof(STableRecord)); - close(fd); - return -1; - } + fields = taos_fetch_fields(res); - int32_t numOfThread = 0; - int subFd = -1; - for (numOfThread = 0; numOfThread < maxThreads; numOfThread++) { - memset(tmpBuf, 0, MAX_FILE_NAME_LEN); - sprintf(tmpBuf, ".tables.tmp.%d", numOfThread); - subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (subFd == -1) { - errorPrint("%s() LN%d, failed to open temp file: %s\n", - __func__, __LINE__, tmpBuf); - for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { - sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); - (void)remove(tmpBuf); - } - sprintf(tmpBuf, ".show-tables.tmp"); - (void)remove(tmpBuf); - free(tblBuf); - close(fd); + row = taos_fetch_row(res); + if (NULL == row) { + errorPrint("%s() LN%d, fetch failed to run command <%s>, reason:%s\n", + __func__, __LINE__, sqlstr, taos_errstr(res)); + taos_free_result(res); + taos_close(taos); return -1; } - // read tableOfPerFile for fd, write to subFd - ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord)); - if (readLen <= 0) { - close(subFd); - break; + if (row[TSDB_SHOW_TABLES_NAME_INDEX] == NULL) { + sprintf(tableDes->cols[i].note, "%s", "NUL"); + taos_free_result(res); + res = NULL; + taos_close(taos); + continue; } - taosWrite(subFd, tblBuf, readLen); - close(subFd); - } - sprintf(tmpBuf, ".show-tables.tmp"); - (void)remove(tmpBuf); + int32_t* length = taos_fetch_lengths(res); + + //int32_t* length = taos_fetch_lengths(tmpResult); + switch (fields[0].type) { + case TSDB_DATA_TYPE_BOOL: + sprintf(tableDes->cols[i].value, "%d", + ((((int32_t)(*((char *)row[TSDB_SHOW_TABLES_NAME_INDEX]))) == 1) ? 1 : 0)); + break; + case TSDB_DATA_TYPE_TINYINT: + sprintf(tableDes->cols[i].value, "%d", + *((int8_t *)row[TSDB_SHOW_TABLES_NAME_INDEX])); + break; + case TSDB_DATA_TYPE_SMALLINT: + sprintf(tableDes->cols[i].value, "%d", + *((int16_t *)row[TSDB_SHOW_TABLES_NAME_INDEX])); + break; + case TSDB_DATA_TYPE_INT: + sprintf(tableDes->cols[i].value, "%d", + *((int32_t *)row[TSDB_SHOW_TABLES_NAME_INDEX])); + break; + case TSDB_DATA_TYPE_BIGINT: + sprintf(tableDes->cols[i].value, "%" PRId64 "", + *((int64_t *)row[TSDB_SHOW_TABLES_NAME_INDEX])); + break; + case TSDB_DATA_TYPE_FLOAT: + sprintf(tableDes->cols[i].value, "%f", + GET_FLOAT_VAL(row[TSDB_SHOW_TABLES_NAME_INDEX])); + break; + case TSDB_DATA_TYPE_DOUBLE: + sprintf(tableDes->cols[i].value, "%f", + GET_DOUBLE_VAL(row[TSDB_SHOW_TABLES_NAME_INDEX])); + break; + case TSDB_DATA_TYPE_BINARY: + memset(tableDes->cols[i].value, 0, + sizeof(tableDes->cols[i].value)); + int len = strlen((char *)row[0]); + // FIXME for long value + if (len < (COL_VALUEBUF_LEN - 2)) { + tableDes->cols[i].value[0] = '\''; + converStringToReadable( + (char *)row[0], + length[0], + tableDes->cols[i].value + 1, + len); + tableDes->cols[i].value[len+1] = '\''; + } else { + tableDes->cols[i].var_value = calloc(1, len + 2); + if (tableDes->cols[i].var_value == NULL) { + errorPrint("%s() LN%d, memory alalocation failed!\n", + __func__, __LINE__); + taos_free_result(res); + return -1; + } + tableDes->cols[i].var_value[0] = '\''; + converStringToReadable((char *)row[0], + length[0], + (char *)(tableDes->cols[i].var_value + 1), len); + tableDes->cols[i].var_value[len+1] = '\''; + } + break; + + case TSDB_DATA_TYPE_NCHAR: + { + memset(tableDes->cols[i].value, 0, sizeof(tableDes->cols[i].note)); + char tbuf[COL_NOTE_LEN-2]; // need reserve 2 bytes for ' ' + convertNCharToReadable((char *)row[TSDB_SHOW_TABLES_NAME_INDEX], length[0], tbuf, COL_NOTE_LEN); + sprintf(tableDes->cols[i].value, "\'%s\'", tbuf); + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + sprintf(tableDes->cols[i].value, "%" PRId64 "", *(int64_t *)row[TSDB_SHOW_TABLES_NAME_INDEX]); +#if 0 + if (!g_args.mysqlFlag) { + sprintf(tableDes->cols[i].value, "%" PRId64 "", *(int64_t *)row[TSDB_SHOW_TABLES_NAME_INDEX]); + } else { + char buf[64] = "\0"; + int64_t ts = *((int64_t *)row[TSDB_SHOW_TABLES_NAME_INDEX]); + time_t tt = (time_t)(ts / 1000); + struct tm *ptm = localtime(&tt); + strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm); + sprintf(tableDes->cols[i].value, "\'%s.%03d\'", buf, (int)(ts % 1000)); + } +#endif + break; + default: + break; + } - if (fd >= 0) { - close(fd); - fd = -1; + taos_free_result(res); } - // start multi threads to dumpout - taosStartDumpOutWorkThreads(numOfThread, dbInfo->name, - getPrecisionByString(dbInfo->precision)); - for (int loopCnt = 0; loopCnt < numOfThread; loopCnt++) { - sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); - (void)remove(tmpBuf); + taos_close(taos); + return colCount; +} + +static void dumpCreateDbClause( + SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { + char sqlstr[TSDB_MAX_SQL_LEN] = {0}; + + char *pstr = sqlstr; + pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s ", dbInfo->name); + if (isDumpProperty) { + pstr += sprintf(pstr, + "REPLICA %d QUORUM %d DAYS %d KEEP %s CACHE %d BLOCKS %d MINROWS %d MAXROWS %d FSYNC %d CACHELAST %d COMP %d PRECISION '%s' UPDATE %d", + dbInfo->replica, dbInfo->quorum, dbInfo->days, + dbInfo->keeplist, + dbInfo->cache, + dbInfo->blocks, dbInfo->minrows, dbInfo->maxrows, + dbInfo->fsync, + dbInfo->cachelast, + dbInfo->comp, dbInfo->precision, dbInfo->update); } - free(tblBuf); - return 0; + pstr += sprintf(pstr, ";"); + fprintf(fp, "%s\n\n", sqlstr); } -static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, +static int dumpCreateTableClause(TableDef *tableDes, int numOfCols, FILE *fp, char* dbName) { int counter = 0; int count_temp = 0; @@ -2232,65 +2175,8 @@ static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, pstr += sprintf(pstr, ");"); - fprintf(fp, "%s\n\n", sqlstr); -} - -static void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, - int numOfCols, FILE *fp, char* dbName) { - int counter = 0; - int count_temp = 0; - - char* tmpBuf = (char *)malloc(COMMAND_SIZE); - if (tmpBuf == NULL) { - errorPrint("%s() LN%d, failed to allocate %d memory\n", - __func__, __LINE__, COMMAND_SIZE); - return; - } - - char *pstr = NULL; - pstr = tmpBuf; - - pstr += sprintf(tmpBuf, - "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", - dbName, tableDes->name, dbName, metric); - - for (; counter < numOfCols; counter++) { - if (tableDes->cols[counter].note[0] != '\0') break; - } - - assert(counter < numOfCols); - count_temp = counter; - - for (; counter < numOfCols; counter++) { - if (counter != count_temp) { - if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || - strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { - //pstr += sprintf(pstr, ", \'%s\'", tableDes->cols[counter].note); - pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note); - } else { - pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note); - } - } else { - if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || - strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { - //pstr += sprintf(pstr, "\'%s\'", tableDes->cols[counter].note); - pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); - } else { - pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); - } - /* pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); */ - } - - /* if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || strcasecmp(tableDes->cols[counter].type, "nchar") - * == 0) { */ - /* pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length); */ - /* } */ - } - - pstr += sprintf(pstr, ");"); - - fprintf(fp, "%s\n", tmpBuf); - free(tmpBuf); + debugPrint("%s() LN%d, write string: %s\n", __func__, __LINE__, sqlstr); + return fprintf(fp, "%s\n\n", sqlstr); } static int writeSchemaToAvro(char *jsonAvroSchema) @@ -2380,19 +2266,6 @@ static int64_t writeResultToSql(TAOS_RES *res, FILE *fp, char *dbName, char *tbN curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRId64 "", *((int64_t *)row[col])); break; - case TSDB_DATA_TYPE_UTINYINT: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((uint8_t *)row[col])); - break; - case TSDB_DATA_TYPE_USMALLINT: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((uint16_t *)row[col])); - break; - case TSDB_DATA_TYPE_UINT: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRIu32 "", *((uint32_t *)row[col])); - break; - case TSDB_DATA_TYPE_UBIGINT: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRIu64 "", - *((uint64_t *)row[col])); - break; case TSDB_DATA_TYPE_FLOAT: curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%f", GET_FLOAT_VAL(row[col])); break; @@ -2402,10 +2275,7 @@ static int64_t writeResultToSql(TAOS_RES *res, FILE *fp, char *dbName, char *tbN case TSDB_DATA_TYPE_BINARY: { char tbuf[COMMAND_SIZE] = {0}; - //*(pstr++) = '\''; converStringToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); - //pstr = stpcpy(pstr, tbuf); - //*(pstr++) = '\''; curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); break; } @@ -2465,8 +2335,8 @@ static int64_t writeResultToSql(TAOS_RES *res, FILE *fp, char *dbName, char *tbN return 0; } -static int taosDumpTableData(FILE *fp, char *tbName, - TAOS* taosCon, char* dbName, int precision, +static int64_t dumpTableData(FILE *fp, char *tbName, + char* dbName, int precision, char *jsonAvroSchema) { int64_t totalRows = 0; @@ -2499,12 +2369,22 @@ static int taosDumpTableData(FILE *fp, char *tbName, "select * from %s.%s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc;", dbName, tbName, start_time, end_time); - TAOS_RES* res = taos_query(taosCon, sqlstr); + TAOS *taos = taos_connect(g_args.host, + g_args.user, g_args.password, dbName, g_args.port); + if (NULL == taos) { + errorPrint( + "Failed to connect to TDengine server %s by specified database %s\n", + g_args.host, dbName); + return -1; + } + + TAOS_RES* res = taos_query(taos, sqlstr); int32_t code = taos_errno(res); if (code != 0) { errorPrint("failed to run command %s, reason: %s\n", sqlstr, taos_errstr(res)); taos_free_result(res); + taos_close(taos); return -1; } @@ -2516,20 +2396,27 @@ static int taosDumpTableData(FILE *fp, char *tbName, } taos_free_result(res); + taos_close(taos); return totalRows; } -static int taosCheckParam(struct arguments *arguments) { +static int checkParam() { if (g_args.all_databases && g_args.databases) { - fprintf(stderr, "conflict option --all-databases and --databases\n"); + errorPrint("%s", "conflict option --all-databases and --databases\n"); return -1; } if (g_args.start_time > g_args.end_time) { - fprintf(stderr, "start time is larger than end time\n"); + errorPrint("%s", "start time is larger than end time\n"); return -1; } + if (g_args.arg_list_len == 0) { + if ((!g_args.all_databases) && (!g_args.databases) && (!g_args.isDumpIn)) { + errorPrint("%s", "taosdump requires parameters\n"); + return -1; + } + } /* if (g_args.isDumpIn && (strcmp(g_args.outpath, DEFAULT_DUMP_FILE) != 0)) { fprintf(stderr, "duplicate parameter input and output file path\n"); @@ -2644,7 +2531,6 @@ static int converStringToReadable(char *str, int size, char *buf, int bufsize) { static int convertNCharToReadable(char *str, int size, char *buf, int bufsize) { char *pstr = str; char *pbuf = buf; - // TODO wchar_t wc; while (size > 0) { if (*pstr == '\0') break; @@ -2668,7 +2554,7 @@ static int convertNCharToReadable(char *str, int size, char *buf, int bufsize) { return 0; } -static void taosDumpCharset(FILE *fp) { +static void dumpCharset(FILE *fp) { char charsetline[256]; (void)fseek(fp, 0, SEEK_SET); @@ -2676,7 +2562,7 @@ static void taosDumpCharset(FILE *fp) { (void)fwrite(charsetline, strlen(charsetline), 1, fp); } -static void taosLoadFileCharset(FILE *fp, char *fcharset) { +static void loadFileCharset(FILE *fp, char *fcharset) { char * line = NULL; size_t line_size = 0; @@ -2808,7 +2694,7 @@ static void taosMallocDumpFiles() } } -static void taosFreeDumpFiles() +static void freeDumpFiles() { for (int i = 0; i < g_tsSqlFileNum; i++) { tfree(g_tsDumpInSqlFiles[i]); @@ -2876,7 +2762,7 @@ static FILE* taosOpenDumpInFile(char *fptr) { return f; } -static int taosDumpInOneFile(TAOS* taos, FILE* fp, char* fcharset, +static int dumpInOneFile(TAOS* taos, FILE* fp, char* fcharset, char* encode, char* fileName) { int read_len = 0; char * cmd = NULL; @@ -2933,9 +2819,9 @@ static int taosDumpInOneFile(TAOS* taos, FILE* fp, char* fcharset, return 0; } -static void* taosDumpInWorkThreadFp(void *arg) +static void* dumpInWorkThreadFp(void *arg) { - SThreadParaObj *pThread = (SThreadParaObj*)arg; + threadInfo *pThread = (threadInfo*)arg; setThreadName("dumpInWorkThrd"); for (int32_t f = 0; f < g_tsSqlFileNum; ++f) { @@ -2947,25 +2833,25 @@ static void* taosDumpInWorkThreadFp(void *arg) } fprintf(stderr, ", Success Open input file: %s\n", SQLFileName); - taosDumpInOneFile(pThread->taosCon, fp, g_tsCharset, g_args.encode, SQLFileName); + dumpInOneFile(pThread->taos, fp, g_tsCharset, g_args.encode, SQLFileName); } } return NULL; } -static void taosStartDumpInWorkThreads() +static void startDumpInWorkThreads() { pthread_attr_t thattr; - SThreadParaObj *pThread; + threadInfo *pThread; int32_t totalThreads = g_args.thread_num; if (totalThreads > g_tsSqlFileNum) { totalThreads = g_tsSqlFileNum; } - SThreadParaObj *threadObj = (SThreadParaObj *)calloc( - totalThreads, sizeof(SThreadParaObj)); + threadInfo *threadObj = (threadInfo *)calloc( + totalThreads, sizeof(threadInfo)); if (NULL == threadObj) { errorPrint("%s() LN%d, memory allocation failed\n", __func__, __LINE__); @@ -2975,9 +2861,9 @@ static void taosStartDumpInWorkThreads() pThread = threadObj + t; pThread->threadIndex = t; pThread->totalThreads = totalThreads; - pThread->taosCon = taos_connect(g_args.host, g_args.user, g_args.password, + pThread->taos = taos_connect(g_args.host, g_args.user, g_args.password, NULL, g_args.port); - if (pThread->taosCon == NULL) { + if (pThread->taos == NULL) { errorPrint("Failed to connect to TDengine server %s\n", g_args.host); free(threadObj); return; @@ -2986,7 +2872,7 @@ static void taosStartDumpInWorkThreads() pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pThread->threadID), &thattr, - taosDumpInWorkThreadFp, (void*)pThread) != 0) { + dumpInWorkThreadFp, (void*)pThread) != 0) { errorPrint("%s() LN%d, thread:%d failed to start\n", __func__, __LINE__, pThread->threadIndex); exit(0); @@ -2998,12 +2884,12 @@ static void taosStartDumpInWorkThreads() } for (int t = 0; t < totalThreads; ++t) { - taos_close(threadObj[t].taosCon); + taos_close(threadObj[t].taos); } free(threadObj); } -static int taosDumpIn() { +static int dumpIn() { assert(g_args.isDumpIn); TAOS *taos = NULL; @@ -3032,19 +2918,169 @@ static int taosDumpIn() { } fprintf(stderr, "Success Open input file: %s\n", g_tsDbSqlFile); - taosLoadFileCharset(fp, g_tsCharset); + loadFileCharset(fp, g_tsCharset); - taosDumpInOneFile(taos, fp, g_tsCharset, g_args.encode, + dumpInOneFile(taos, fp, g_tsCharset, g_args.encode, g_tsDbSqlFile); } taos_close(taos); if (0 != tsSqlFileNumOfTbls) { - taosStartDumpInWorkThreads(); + startDumpInWorkThreads(); } - taosFreeDumpFiles(); + freeDumpFiles(); return 0; } +int main(int argc, char *argv[]) { + static char verType[32] = {0}; + sprintf(verType, "version: %s\n", version); + argp_program_version = verType; + + int ret = 0; + /* Parse our arguments; every option seen by parse_opt will be + reflected in arguments. */ + if (argc > 1) { +// parse_precision_first(argc, argv, &g_args); + parse_timestamp(argc, argv, &g_args); + parse_args(argc, argv, &g_args); + } + + argp_parse(&argp, argc, argv, 0, 0, &g_args); + + if (g_args.abort) { +#ifndef _ALPINE + error(10, 0, "ABORTED"); +#else + abort(); +#endif + } + + printf("====== arguments config ======\n"); + + printf("host: %s\n", g_args.host); + printf("user: %s\n", g_args.user); + printf("password: %s\n", g_args.password); + printf("port: %u\n", g_args.port); + printf("mysqlFlag: %d\n", g_args.mysqlFlag); + printf("outpath: %s\n", g_args.outpath); + printf("inpath: %s\n", g_args.inpath); + printf("resultFile: %s\n", g_args.resultFile); + printf("encode: %s\n", g_args.encode); + printf("all_databases: %s\n", g_args.all_databases?"true":"false"); + printf("databases: %d\n", g_args.databases); + printf("databasesSeq: %s\n", g_args.databasesSeq); + printf("schemaonly: %s\n", g_args.schemaonly?"true":"false"); + printf("with_property: %s\n", g_args.with_property?"true":"false"); + printf("avro format: %s\n", g_args.avro?"true":"false"); + printf("start_time: %" PRId64 "\n", g_args.start_time); + printf("human readable start time: %s \n", g_args.humanStartTime); + printf("end_time: %" PRId64 "\n", g_args.end_time); + printf("human readable end time: %s \n", g_args.humanEndTime); + printf("precision: %s\n", g_args.precision); + printf("data_batch: %d\n", g_args.data_batch); + printf("max_sql_len: %d\n", g_args.max_sql_len); + printf("table_batch: %d\n", g_args.table_batch); + printf("thread_num: %d\n", g_args.thread_num); + printf("allow_sys: %d\n", g_args.allow_sys); + printf("abort: %d\n", g_args.abort); + printf("isDumpIn: %d\n", g_args.isDumpIn); + printf("arg_list_len: %d\n", g_args.arg_list_len); + printf("debug_print: %d\n", g_args.debug_print); + + for (int32_t i = 0; i < g_args.arg_list_len; i++) { + printf("arg_list[%d]: %s\n", i, g_args.arg_list[i]); + } + + printf("==============================\n"); + if (checkParam(&g_args) < 0) { + exit(EXIT_FAILURE); + } + + g_fpOfResult = fopen(g_args.resultFile, "a"); + if (NULL == g_fpOfResult) { + errorPrint("Failed to open %s for save result\n", g_args.resultFile); + exit(-1); + }; + + fprintf(g_fpOfResult, "#############################################################################\n"); + fprintf(g_fpOfResult, "============================== arguments config =============================\n"); + + fprintf(g_fpOfResult, "host: %s\n", g_args.host); + fprintf(g_fpOfResult, "user: %s\n", g_args.user); + fprintf(g_fpOfResult, "password: %s\n", g_args.password); + fprintf(g_fpOfResult, "port: %u\n", g_args.port); + fprintf(g_fpOfResult, "mysqlFlag: %d\n", g_args.mysqlFlag); + fprintf(g_fpOfResult, "outpath: %s\n", g_args.outpath); + fprintf(g_fpOfResult, "inpath: %s\n", g_args.inpath); + fprintf(g_fpOfResult, "resultFile: %s\n", g_args.resultFile); + fprintf(g_fpOfResult, "encode: %s\n", g_args.encode); + fprintf(g_fpOfResult, "all_databases: %s\n", g_args.all_databases?"true":"false"); + fprintf(g_fpOfResult, "databases: %d\n", g_args.databases); + fprintf(g_fpOfResult, "databasesSeq: %s\n", g_args.databasesSeq); + fprintf(g_fpOfResult, "schemaonly: %s\n", g_args.schemaonly?"true":"false"); + fprintf(g_fpOfResult, "with_property: %s\n", g_args.with_property?"true":"false"); + fprintf(g_fpOfResult, "avro format: %s\n", g_args.avro?"true":"false"); + fprintf(g_fpOfResult, "start_time: %" PRId64 "\n", g_args.start_time); + fprintf(g_fpOfResult, "human readable start time: %s \n", g_args.humanStartTime); + fprintf(g_fpOfResult, "end_time: %" PRId64 "\n", g_args.end_time); + fprintf(g_fpOfResult, "human readable end time: %s \n", g_args.humanEndTime); + fprintf(g_fpOfResult, "precision: %s\n", g_args.precision); + fprintf(g_fpOfResult, "data_batch: %d\n", g_args.data_batch); + fprintf(g_fpOfResult, "max_sql_len: %d\n", g_args.max_sql_len); + fprintf(g_fpOfResult, "table_batch: %d\n", g_args.table_batch); + fprintf(g_fpOfResult, "thread_num: %d\n", g_args.thread_num); + fprintf(g_fpOfResult, "allow_sys: %d\n", g_args.allow_sys); + fprintf(g_fpOfResult, "abort: %d\n", g_args.abort); + fprintf(g_fpOfResult, "isDumpIn: %d\n", g_args.isDumpIn); + fprintf(g_fpOfResult, "arg_list_len: %d\n", g_args.arg_list_len); + + for (int32_t i = 0; i < g_args.arg_list_len; i++) { + fprintf(g_fpOfResult, "arg_list[%d]: %s\n", i, g_args.arg_list[i]); + } + + g_numOfCores = (int32_t)sysconf(_SC_NPROCESSORS_ONLN); + + time_t tTime = time(NULL); + struct tm tm = *localtime(&tTime); + + if (g_args.isDumpIn) { + fprintf(g_fpOfResult, "============================== DUMP IN ============================== \n"); + fprintf(g_fpOfResult, "# DumpIn start time: %d-%02d-%02d %02d:%02d:%02d\n", + tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + if (dumpIn() < 0) { + ret = -1; + } + } else { + fprintf(g_fpOfResult, "============================== DUMP OUT ============================== \n"); + fprintf(g_fpOfResult, "# DumpOut start time: %d-%02d-%02d %02d:%02d:%02d\n", + tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + if (dumpOut() < 0) { + ret = -1; + } else { + fprintf(g_fpOfResult, "\n============================== TOTAL STATISTICS ============================== \n"); + fprintf(g_fpOfResult, "# total database count: %d\n", + g_resultStatistics.totalDatabasesOfDumpOut); + fprintf(g_fpOfResult, "# total super table count: %d\n", + g_resultStatistics.totalSuperTblsOfDumpOut); + fprintf(g_fpOfResult, "# total child table count: %"PRId64"\n", + g_resultStatistics.totalChildTblsOfDumpOut); + fprintf(g_fpOfResult, "# total row count: %"PRId64"\n", + g_resultStatistics.totalRowsOfDumpOut); + } + } + + fprintf(g_fpOfResult, "\n"); + fclose(g_fpOfResult); + + if (g_tablesList) { + free(g_tablesList); + } + + return ret; +} + diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index e3c1af9bc01fddd8c7df78ace6c5c2b6ce13576c..e692f1d0b7fabc0f13ba8cf88c9c16e4c52e436e 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1793,6 +1793,7 @@ static int32_t mnodeDoGetSuperTableMeta(SMnodeMsg *pMsg, STableMetaMsg* pMeta) { pMeta->sversion = htons(pTable->sversion); pMeta->tversion = htons(pTable->tversion); pMeta->precision = pMsg->pDb->cfg.precision; + pMeta->update = pMsg->pDb->cfg.update; pMeta->numOfTags = (uint8_t)pTable->numOfTags; pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns); pMeta->tableType = pTable->info.type; @@ -2510,6 +2511,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { pMeta->uid = htobe64(pTable->uid); pMeta->tid = htonl(pTable->tid); pMeta->precision = pDb->cfg.precision; + pMeta->update = pDb->cfg.update; pMeta->tableType = pTable->info.type; tstrncpy(pMeta->tableFname, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); diff --git a/src/os/src/detail/osTime.c b/src/os/src/detail/osTime.c index a95d919a83fa09be4e8b51990c56df42ac1affa9..ab3f3668c57f24e167dd102ec887b5045fb2c5a6 100644 --- a/src/os/src/detail/osTime.c +++ b/src/os/src/detail/osTime.c @@ -343,10 +343,40 @@ int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrec assert(toPrecision == TSDB_TIME_PRECISION_MILLI || toPrecision == TSDB_TIME_PRECISION_MICRO || toPrecision == TSDB_TIME_PRECISION_NANO); - static double factors[3][3] = { {1., 1000., 1000000.}, - {1.0 / 1000, 1., 1000.}, - {1.0 / 1000000, 1.0 / 1000, 1.} }; - return (int64_t)((double)time * factors[fromPrecision][toPrecision]); + switch(fromPrecision) { + case TSDB_TIME_PRECISION_MILLI: { + switch (toPrecision) { + case TSDB_TIME_PRECISION_MILLI: + return time; + case TSDB_TIME_PRECISION_MICRO: + return time * 1000; + case TSDB_TIME_PRECISION_NANO: + return time * 1000000; + } + } // end from milli + case TSDB_TIME_PRECISION_MICRO: { + switch (toPrecision) { + case TSDB_TIME_PRECISION_MILLI: + return time / 1000; + case TSDB_TIME_PRECISION_MICRO: + return time; + case TSDB_TIME_PRECISION_NANO: + return time * 1000; + } + } //end from micro + case TSDB_TIME_PRECISION_NANO: { + switch (toPrecision) { + case TSDB_TIME_PRECISION_MILLI: + return time / 1000000; + case TSDB_TIME_PRECISION_MICRO: + return time / 1000; + case TSDB_TIME_PRECISION_NANO: + return time; + } + } //end from nano + default: + assert(0); + } //end switch fromPrecision } static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { diff --git a/src/query/inc/qAggMain.h b/src/query/inc/qAggMain.h index d4116fbfb2daec9b47c4a891c3c886728e6ca515..b05c1ea29eee803c3de132105627c02a70dbf9f8 100644 --- a/src/query/inc/qAggMain.h +++ b/src/query/inc/qAggMain.h @@ -191,6 +191,7 @@ typedef struct SQLFunctionCtx { SResultRowCellInfo *resultInfo; + int16_t colId; SExtTagsInfo tagInfo; SPoint1 start; SPoint1 end; diff --git a/src/query/inc/qTableMeta.h b/src/query/inc/qTableMeta.h index 6e9ba9c26f413e475d9f9ecba6f0705154a97bdd..fce8e23d1c0f2aa4b2bf4b35f2b99a1b5344658d 100644 --- a/src/query/inc/qTableMeta.h +++ b/src/query/inc/qTableMeta.h @@ -53,6 +53,7 @@ typedef struct SGroupbyExpr { typedef struct STableComInfo { uint8_t numOfTags; uint8_t precision; + uint8_t update; int16_t numOfColumns; int32_t rowSize; } STableComInfo; diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index fd8729e37925647279648ebd1bc4eadbdfa7feef..da4cb62351b7244c050239688df82bf8bd396afe 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -3697,7 +3697,7 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) { bool ascQuery = (pCtx->order == TSDB_ORDER_ASC); - if (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { + if (pCtx->colId == 0 && pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { *(TSKEY *)pCtx->pOutput = pCtx->startTs; } else if (type == TSDB_FILL_NULL) { setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 634667915adddc05c368c08c1da7dafd0b408ca3..42787f22190e3fd7685af6598b51fc9c9699d0c2 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1232,6 +1232,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, // in case of the block distribution query, the inputBytes is not a constant value. pCtx[i].pInput = p->pData; + pCtx[i].colId = p->info.colId; assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type); if (pCtx[i].functionId < 0) { diff --git a/src/query/src/qTableMeta.c b/src/query/src/qTableMeta.c index f687b8aa1ffc530d0c4a71c553809dd3bfb83932..f786f4438c2915299fa320818d7a36811eef40dd 100644 --- a/src/query/src/qTableMeta.c +++ b/src/query/src/qTableMeta.c @@ -84,6 +84,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { pTableMeta->tableInfo = (STableComInfo) { .numOfTags = pTableMetaMsg->numOfTags, .precision = pTableMetaMsg->precision, + .update = pTableMetaMsg->update, .numOfColumns = pTableMetaMsg->numOfColumns, }; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 02dbe00bf7d5647ca410683647c69e62b248b758..705ede0cf105c3aee49dc42fb2f312dba6b67806 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -653,55 +653,8 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) { return res; } -// leave only one table for each group -static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGroupList) { - assert(pGroupList); - size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList); - - STableGroupInfo* pNew = calloc(1, sizeof(STableGroupInfo)); - pNew->pGroupList = taosArrayInit(numOfGroup, POINTER_BYTES); - - for(int32_t i = 0; i < numOfGroup; ++i) { - SArray* oneGroup = taosArrayGetP(pGroupList->pGroupList, i); - size_t numOfTables = taosArrayGetSize(oneGroup); - - SArray* px = taosArrayInit(4, sizeof(STableKeyInfo)); - for (int32_t j = 0; j < numOfTables; ++j) { - STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(oneGroup, j); - if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) { - taosArrayPush(px, pInfo); - pNew->numOfTables += 1; - break; - } - } - - // there are no data in this group - if (taosArrayGetSize(px) == 0) { - taosArrayDestroy(px); - } else { - taosArrayPush(pNew->pGroupList, &px); - } - } - - return pNew; -} - TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pRef) { - STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); - - if (pNew->numOfTables == 0) { - tsdbDebug("update query time range to invalidate time window"); - - assert(taosArrayGetSize(pNew->pGroupList) == 0); - bool asc = ASCENDING_TRAVERSE(pCond->order); - if (asc) { - pCond->twindow.ekey = pCond->twindow.skey - 1; - } else { - pCond->twindow.skey = pCond->twindow.ekey - 1; - } - } - - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef); + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pRef); pQueryHandle->loadExternalRow = true; pQueryHandle->currentLoadExternalRows = true; diff --git a/tests/gotest/batchtest.bat b/tests/gotest/batchtest.bat index efd8961bb0be2eb6f20e291114b92b00469b984f..f9e6f83d50b1f1fa04cb18972376b3951447cc81 100755 --- a/tests/gotest/batchtest.bat +++ b/tests/gotest/batchtest.bat @@ -1,3 +1,4 @@ + @echo off echo ==== start Go connector test cases test ==== cd /d %~dp0 @@ -8,7 +9,7 @@ if "%severIp%"=="" (set severIp=127.0.0.1) if "%serverPort%"=="" (set serverPort=6030) go env -w GO111MODULE=on -go env -w GOPROXY=https://goproxy.io,direct +go env -w GOPROXY=https://goproxy.cn,direct cd case001 case001.bat %severIp% %serverPort% @@ -18,3 +19,10 @@ rem case002.bat :: cd case002 :: case002.bat + + +rem cd nanosupport +rem nanoCase.bat + +:: cd nanosupport +:: nanoCase.bat diff --git a/tests/gotest/batchtest.sh b/tests/gotest/batchtest.sh index 8f5a7fe8f032134e55c9d9675361590ed6d5b19b..046249bcf7e8abab57d43b6b6e268361ccc1a695 100755 --- a/tests/gotest/batchtest.sh +++ b/tests/gotest/batchtest.sh @@ -14,8 +14,9 @@ if [ ! -n "$serverPort" ]; then fi go env -w GO111MODULE=on -go env -w GOPROXY=https://goproxy.io,direct +go env -w GOPROXY=https://goproxy.cn,direct bash ./case001/case001.sh $severIp $serverPort bash ./case002/case002.sh $severIp $serverPort #bash ./case003/case003.sh $severIp $serverPort +bash ./nanosupport/nanoCase.sh $severIp $serverPort diff --git a/tests/gotest/case001/case001.sh b/tests/gotest/case001/case001.sh index 94e5bb44e03a1f7d2704752fcf9c080abcb4f23f..831e9f83ac482c0a2c668e2ad0d16c4bf59f19aa 100644 --- a/tests/gotest/case001/case001.sh +++ b/tests/gotest/case001/case001.sh @@ -15,8 +15,7 @@ script_dir="$(dirname $(readlink -f $0))" ###### step 3: start build cd $script_dir rm -f go.* -go mod init demotest > /dev/null 2>&1 -go mod tidy > /dev/null 2>&1 -go build > /dev/null 2>&1 +go mod init demotest +go build sleep 1s ./demotest -h $1 -p $2 diff --git a/tests/gotest/case002/case002.bat b/tests/gotest/case002/case002.bat index ebec576e724ccb14319dd380c9783a783ac0db62..385677acae826e248a410472bfc7a022ff3003ab 100644 --- a/tests/gotest/case002/case002.bat +++ b/tests/gotest/case002/case002.bat @@ -1,5 +1,5 @@ @echo off -echo ==== start run cases001.go +echo ==== start run cases002.go del go.* go mod init demotest diff --git a/tests/gotest/case002/case002.go b/tests/gotest/case002/case002.go index c69da04cb271c24e33953ca8fdfea71c67349b4f..e2ba5ea28ee4f92cfbdca27c78d47268a387c693 100644 --- a/tests/gotest/case002/case002.go +++ b/tests/gotest/case002/case002.go @@ -43,10 +43,9 @@ func main() { os.Exit(1) } defer db.Close() - db.Exec("drop if exists database test") - db.Exec("create if not exists database test") + db.Exec("drop database if exists test") + db.Exec("create database if not exists test ") db.Exec("use test") - db.Exec("drop if exists database test") db.Exec("create table test (ts timestamp ,level int)") for i := 0; i < 10; i++ { sqlcmd := fmt.Sprintf("insert into test values(%d,%d)", ts+i, i) diff --git a/tests/gotest/case002/case002.sh b/tests/gotest/case002/case002.sh index 94e5bb44e03a1f7d2704752fcf9c080abcb4f23f..d98337cce7cfeb51ec9305226b20abdd7b360a46 100644 --- a/tests/gotest/case002/case002.sh +++ b/tests/gotest/case002/case002.sh @@ -1,6 +1,6 @@ #!/bin/bash -echo "==== start run cases001.go" +echo "==== start run cases002.go" set +e #set -x diff --git a/tests/gotest/nanosupport/connector/executor.go b/tests/gotest/nanosupport/connector/executor.go new file mode 100644 index 0000000000000000000000000000000000000000..218ea29af3b34a8cfb5ab56585eeb07bc467d209 --- /dev/null +++ b/tests/gotest/nanosupport/connector/executor.go @@ -0,0 +1,208 @@ +package connector + +import ( + "context" + "fmt" + "reflect" + "time" + + "github.com/taosdata/go-utils/log" + "github.com/taosdata/go-utils/tdengine/config" + "github.com/taosdata/go-utils/tdengine/connector" + tdengineExecutor "github.com/taosdata/go-utils/tdengine/executor" +) + +type Executor struct { + executor *tdengineExecutor.Executor + ctx context.Context +} + +var Logger = log.NewLogger("taos test") + +func NewExecutor(conf *config.TDengineGo, db string, showSql bool) (*Executor, error) { + tdengineConnector, err := connector.NewTDengineConnector("go", conf) + if err != nil { + return nil, err + } + executor := tdengineExecutor.NewExecutor(tdengineConnector, db, showSql, Logger) + return &Executor{ + executor: executor, + ctx: context.Background(), + }, nil +} + +func (e *Executor) Execute(sql string) (int64, error) { + return e.executor.DoExec(e.ctx, sql) +} +func (e *Executor) Query(sql string) (*connector.Data, error) { + fmt.Println("query :", sql) + return e.executor.DoQuery(e.ctx, sql) +} +func (e *Executor) CheckData(row, col int, value interface{}, data *connector.Data) (bool, error) { + if data == nil { + return false, fmt.Errorf("data is nil") + } + if col >= len(data.Head) { + return false, fmt.Errorf("col out of data") + } + if row >= len(data.Data) { + return false, fmt.Errorf("row out of data") + } + dataValue := data.Data[row][col] + + if dataValue == nil && value != nil { + return false, fmt.Errorf("dataValue is nil but value is not nil") + } + if dataValue == nil && value == nil { + return true, nil + } + if reflect.TypeOf(dataValue) != reflect.TypeOf(value) { + return false, fmt.Errorf("type not match expect %s got %s", reflect.TypeOf(value), reflect.TypeOf(dataValue)) + } + switch value.(type) { + case time.Time: + t, _ := dataValue.(time.Time) + if value.(time.Time).Nanosecond() != t.Nanosecond() { + return false, fmt.Errorf("value not match expect %d got %d", value.(time.Time).Nanosecond(), t.Nanosecond()) + } + case string: + if value.(string) != dataValue.(string) { + return false, fmt.Errorf("value not match expect %s got %s", value.(string), dataValue.(string)) + } + case int8: + if value.(int8) != dataValue.(int8) { + return false, fmt.Errorf("value not match expect %d got %d", value.(int8), dataValue.(int8)) + } + case int16: + if value.(int16) != dataValue.(int16) { + return false, fmt.Errorf("value not match expect %d got %d", value.(int16), dataValue.(int16)) + } + case int32: + if value.(int32) != dataValue.(int32) { + return false, fmt.Errorf("value not match expect %d got %d", value.(int32), dataValue.(int32)) + } + case int64: + if value.(int64) != dataValue.(int64) { + return false, fmt.Errorf("value not match expect %d got %d", value.(int64), dataValue.(int64)) + } + case float32: + if value.(float32) != dataValue.(float32) { + return false, fmt.Errorf("value not match expect %f got %f", value.(float32), dataValue.(float32)) + } + case float64: + if value.(float64) != dataValue.(float64) { + return false, fmt.Errorf("value not match expect %f got %f", value.(float32), dataValue.(float32)) + } + case bool: + if value.(bool) != dataValue.(bool) { + return false, fmt.Errorf("value not match expect %t got %t", value.(bool), dataValue.(bool)) + } + default: + return false, fmt.Errorf("unsupport type %v", reflect.TypeOf(value)) + } + return true, nil +} + +func (e *Executor) CheckData2(row, col int, value interface{}, data *connector.Data) { + + match, err := e.CheckData(row, col, value, data) + fmt.Println("expect data is :", value) + fmt.Println("go got data is :", data.Data[row][col]) + if err != nil { + fmt.Println(err) + } + if !match { + fmt.Println(" data not match") + + } + + /* + fmt.Println(value) + if data == nil { + // return false, fmt.Errorf("data is nil") + // fmt.Println("check failed") + } + if col >= len(data.Head) { + // return false, fmt.Errorf("col out of data") + // fmt.Println("check failed") + } + if row >= len(data.Data) { + // return false, fmt.Errorf("row out of data") + // fmt.Println("check failed") + } + dataValue := data.Data[row][col] + + if dataValue == nil && value != nil { + // return false, fmt.Errorf("dataValue is nil but value is not nil") + // fmt.Println("check failed") + } + if dataValue == nil && value == nil { + // return true, nil + fmt.Println("check pass") + } + if reflect.TypeOf(dataValue) != reflect.TypeOf(value) { + // return false, fmt.Errorf("type not match expect %s got %s", reflect.TypeOf(value), reflect.TypeOf(dataValue)) + fmt.Println("check failed") + } + switch value.(type) { + case time.Time: + t, _ := dataValue.(time.Time) + if value.(time.Time).Nanosecond() != t.Nanosecond() { + // return false, fmt.Errorf("value not match expect %d got %d", value.(time.Time).Nanosecond(), t.Nanosecond()) + // fmt.Println("check failed") + } + case string: + if value.(string) != dataValue.(string) { + // return false, fmt.Errorf("value not match expect %s got %s", value.(string), dataValue.(string)) + // fmt.Println("check failed") + } + case int8: + if value.(int8) != dataValue.(int8) { + // return false, fmt.Errorf("value not match expect %d got %d", value.(int8), dataValue.(int8)) + // fmt.Println("check failed") + } + case int16: + if value.(int16) != dataValue.(int16) { + // return false, fmt.Errorf("value not match expect %d got %d", value.(int16), dataValue.(int16)) + // fmt.Println("check failed") + } + case int32: + if value.(int32) != dataValue.(int32) { + // return false, fmt.Errorf("value not match expect %d got %d", value.(int32), dataValue.(int32)) + // fmt.Println("check failed") + } + case int64: + if value.(int64) != dataValue.(int64) { + // return false, fmt.Errorf("value not match expect %d got %d", value.(int64), dataValue.(int64)) + // fmt.Println("check failed") + } + case float32: + if value.(float32) != dataValue.(float32) { + // return false, fmt.Errorf("value not match expect %f got %f", value.(float32), dataValue.(float32)) + // fmt.Println("check failed") + } + case float64: + if value.(float64) != dataValue.(float64) { + // return false, fmt.Errorf("value not match expect %f got %f", value.(float32), dataValue.(float32)) + // fmt.Println("check failed") + } + case bool: + if value.(bool) != dataValue.(bool) { + // return false, fmt.Errorf("value not match expect %t got %t", value.(bool), dataValue.(bool)) + // fmt.Println("check failed") + } + default: + // return false, fmt.Errorf("unsupport type %v", reflect.TypeOf(value)) + // fmt.Println("check failed") + } + // return true, nil + // fmt.Println("check pass") + */ +} + +func (e *Executor) CheckRow(count int, data *connector.Data) { + + if len(data.Data) != count { + fmt.Println("check failed !") + } +} diff --git a/tests/gotest/nanosupport/nanoCase.bat b/tests/gotest/nanosupport/nanoCase.bat new file mode 100644 index 0000000000000000000000000000000000000000..86bddd5b02c5399d5b8d70bd08020e96a7d1c0e5 --- /dev/null +++ b/tests/gotest/nanosupport/nanoCase.bat @@ -0,0 +1,9 @@ +@echo off +echo ==== start run nanosupport.go + +del go.* +go mod init nano +go mod tidy +go build +nano.exe -h %1 -p %2 +cd .. diff --git a/tests/gotest/nanosupport/nanoCase.sh b/tests/gotest/nanosupport/nanoCase.sh new file mode 100644 index 0000000000000000000000000000000000000000..bec8929f14c0a56e7c4074efa39d1e1e881fb12e --- /dev/null +++ b/tests/gotest/nanosupport/nanoCase.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +echo "==== start run nanosupport.go " + +set +e +#set -x + +script_dir="$(dirname $(readlink -f $0))" +#echo "pwd: $script_dir, para0: $0" + +#execName=$0 +#execName=`echo ${execName##*/}` +#goName=`echo ${execName%.*}` + +###### step 3: start build +cd $script_dir +rm -f go.* +go mod init nano +go mod tidy +go build +sleep 10s +./nano -h $1 -p $2 diff --git a/tests/gotest/nanosupport/nanosupport.go b/tests/gotest/nanosupport/nanosupport.go new file mode 100644 index 0000000000000000000000000000000000000000..e2f24a73c0a6db3c94b90879c73d0f05e2476307 --- /dev/null +++ b/tests/gotest/nanosupport/nanosupport.go @@ -0,0 +1,269 @@ +package main + +import ( + "fmt" + "log" + "nano/connector" + "time" + + "github.com/taosdata/go-utils/tdengine/config" +) + +func main() { + e, err := connector.NewExecutor(&config.TDengineGo{ + Address: "root:taosdata@/tcp(127.0.0.1:6030)/", + MaxIdle: 20, + MaxOpen: 30, + MaxLifetime: 30, + }, "db", false) + if err != nil { + panic(err) + } + prepareData(e) + data, err := e.Query("select * from tb") + if err != nil { + panic(err) + } + + layout := "2006-01-02 15:04:05.999999999" + t0, _ := time.Parse(layout, "2021-06-10 00:00:00.100000001") + t1, _ := time.Parse(layout, "2021-06-10 00:00:00.150000000") + t2, _ := time.Parse(layout, "2021-06-10 00:00:00.299999999") + t3, _ := time.Parse(layout, "2021-06-10 00:00:00.300000000") + t4, _ := time.Parse(layout, "2021-06-10 00:00:00.300000001") + t5, _ := time.Parse(layout, "2021-06-10 00:00:00.999999999") + + e.CheckData2(0, 0, t0, data) + e.CheckData2(1, 0, t1, data) + e.CheckData2(2, 0, t2, data) + e.CheckData2(3, 0, t3, data) + e.CheckData2(4, 0, t4, data) + e.CheckData2(5, 0, t5, data) + e.CheckData2(3, 1, int32(3), data) + e.CheckData2(4, 1, int32(5), data) + e.CheckData2(5, 1, int32(7), data) + + fmt.Println(" start check nano support!") + + data, _ = e.Query("select count(*) from tb where ts > 1623254400100000000 and ts < 1623254400100000002;") + e.CheckData2(0, 0, int64(1), data) + + data, _ = e.Query("select count(*) from tb where ts > \"2021-06-10 0:00:00.100000001\" and ts < \"2021-06-10 0:00:00.160000000\";") + e.CheckData2(0, 0, int64(1), data) + + data, _ = e.Query("select count(*) from tb where ts > 1623254400100000000 and ts < 1623254400150000000;") + e.CheckData2(0, 0, int64(1), data) + data, _ = e.Query("select count(*) from tb where ts > \"2021-06-10 0:00:00.100000000\" and ts < \"2021-06-10 0:00:00.150000000\";") + e.CheckData2(0, 0, int64(1), data) + + data, _ = e.Query("select count(*) from tb where ts > 1623254400400000000;") + e.CheckData2(0, 0, int64(1), data) + data, _ = e.Query("select count(*) from tb where ts < \"2021-06-10 00:00:00.400000000\";") + e.CheckData2(0, 0, int64(5), data) + + data, _ = e.Query("select count(*) from tb where ts < now + 400000000b;") + e.CheckData2(0, 0, int64(6), data) + + data, _ = e.Query("select count(*) from tb where ts >= \"2021-06-10 0:00:00.100000001\";") + e.CheckData2(0, 0, int64(6), data) + + data, _ = e.Query("select count(*) from tb where ts <= 1623254400300000000;") + e.CheckData2(0, 0, int64(4), data) + + data, _ = e.Query("select count(*) from tb where ts = \"2021-06-10 0:00:00.000000000\";") + + data, _ = e.Query("select count(*) from tb where ts = 1623254400150000000;") + e.CheckData2(0, 0, int64(1), data) + + data, _ = e.Query("select count(*) from tb where ts = \"2021-06-10 0:00:00.100000001\";") + e.CheckData2(0, 0, int64(1), data) + + data, _ = e.Query("select count(*) from tb where ts between 1623254400000000000 and 1623254400400000000;") + e.CheckData2(0, 0, int64(5), data) + + data, _ = e.Query("select count(*) from tb where ts between \"2021-06-10 0:00:00.299999999\" and \"2021-06-10 0:00:00.300000001\";") + e.CheckData2(0, 0, int64(3), data) + + data, _ = e.Query("select avg(speed) from tb interval(5000000000b);") + e.CheckRow(1, data) + + data, _ = e.Query("select avg(speed) from tb interval(100000000b)") + e.CheckRow(4, data) + + data, _ = e.Query("select avg(speed) from tb interval(1000b);") + e.CheckRow(5, data) + + data, _ = e.Query("select avg(speed) from tb interval(1u);") + e.CheckRow(5, data) + + data, _ = e.Query("select avg(speed) from tb interval(100000000b) sliding (100000000b);") + e.CheckRow(4, data) + + data, _ = e.Query("select last(*) from tb") + tt, _ := time.Parse(layout, "2021-06-10 0:00:00.999999999") + e.CheckData2(0, 0, tt, data) + + data, _ = e.Query("select first(*) from tb") + tt1, _ := time.Parse(layout, "2021-06-10 0:00:00.100000001") + e.CheckData2(0, 0, tt1, data) + + e.Execute("insert into tb values(now + 500000000b, 6);") + data, _ = e.Query("select * from tb;") + e.CheckRow(7, data) + + e.Execute("create table tb2 (ts timestamp, speed int, ts2 timestamp);") + e.Execute("insert into tb2 values(\"2021-06-10 0:00:00.100000001\", 1, \"2021-06-11 0:00:00.100000001\");") + e.Execute("insert into tb2 values(1623254400150000000, 2, 1623340800150000000);") + e.Execute("import into tb2 values(1623254400300000000, 3, 1623340800300000000);") + e.Execute("import into tb2 values(1623254400299999999, 4, 1623340800299999999);") + e.Execute("insert into tb2 values(1623254400300000001, 5, 1623340800300000001);") + e.Execute("insert into tb2 values(1623254400999999999, 7, 1623513600999999999);") + + data, _ = e.Query("select * from tb2;") + tt2, _ := time.Parse(layout, "2021-06-10 0:00:00.100000001") + tt3, _ := time.Parse(layout, "2021-06-10 0:00:00.150000000") + + e.CheckData2(0, 0, tt2, data) + e.CheckData2(1, 0, tt3, data) + e.CheckData2(2, 1, int32(4), data) + e.CheckData2(3, 1, int32(3), data) + tt4, _ := time.Parse(layout, "2021-06-11 00:00:00.300000001") + e.CheckData2(4, 2, tt4, data) + e.CheckRow(6, data) + + data, _ = e.Query("select count(*) from tb2 where ts2 > 1623340800000000000 and ts2 < 1623340800150000000;") + e.CheckData2(0, 0, int64(1), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 > \"2021-06-11 0:00:00.100000000\" and ts2 < \"2021-06-11 0:00:00.100000002\";") + e.CheckData2(0, 0, int64(1), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 > 1623340800500000000;") + e.CheckData2(0, 0, int64(1), data) + data, _ = e.Query("select count(*) from tb2 where ts2 < \"2021-06-11 0:00:00.400000000\";") + e.CheckData2(0, 0, int64(5), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 < now + 400000000b;") + e.CheckData2(0, 0, int64(6), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 >= \"2021-06-11 0:00:00.100000001\";") + e.CheckData2(0, 0, int64(6), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 <= 1623340800400000000;") + e.CheckData2(0, 0, int64(5), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 = \"2021-06-11 0:00:00.000000000\";") + + data, _ = e.Query("select count(*) from tb2 where ts2 = \"2021-06-11 0:00:00.300000001\";") + e.CheckData2(0, 0, int64(1), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 = 1623340800300000001;") + e.CheckData2(0, 0, int64(1), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 between 1623340800000000000 and 1623340800450000000;") + e.CheckData2(0, 0, int64(5), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 between \"2021-06-11 0:00:00.299999999\" and \"2021-06-11 0:00:00.300000001\";") + e.CheckData2(0, 0, int64(3), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 <> 1623513600999999999;") + e.CheckData2(0, 0, int64(5), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 <> \"2021-06-11 0:00:00.100000001\";") + e.CheckData2(0, 0, int64(5), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 <> \"2021-06-11 0:00:00.100000000\";") + e.CheckData2(0, 0, int64(6), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 != 1623513600999999999;") + e.CheckData2(0, 0, int64(5), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 != \"2021-06-11 0:00:00.100000001\";") + e.CheckData2(0, 0, int64(5), data) + + data, _ = e.Query("select count(*) from tb2 where ts2 != \"2021-06-11 0:00:00.100000000\";") + e.CheckData2(0, 0, int64(6), data) + + e.Execute("insert into tb2 values(now + 500000000b, 6, now +2d);") + data, _ = e.Query("select * from tb2;") + e.CheckRow(7, data) + + e.Execute("create table tb3 (ts timestamp, speed int);") + _, err = e.Execute("insert into tb3 values(16232544001500000, 2);") + if err != nil { + fmt.Println("check pass! ") + } + + e.Execute("insert into tb3 values(\"2021-06-10 0:00:00.123456\", 2);") + data, _ = e.Query("select * from tb3 where ts = \"2021-06-10 0:00:00.123456000\";") + e.CheckRow(1, data) + + e.Execute("insert into tb3 values(\"2021-06-10 0:00:00.123456789000\", 2);") + data, _ = e.Query("select * from tb3 where ts = \"2021-06-10 0:00:00.123456789\";") + e.CheckRow(1, data) + + // check timezone support + + e.Execute("drop database if exists nsdb;") + e.Execute("create database nsdb precision 'ns';") + e.Execute("use nsdb;") + e.Execute("create stable st (ts timestamp ,speed float ) tags(time timestamp ,id int);") + e.Execute("insert into tb1 using st tags('2021-06-10 0:00:00.123456789' , 1 ) values('2021-06-10T0:00:00.123456789+07:00' , 1.0);") + data, _ = e.Query("select first(*) from tb1;") + + ttt, _ := time.Parse(layout, "2021-06-10 01:00:00.123456789") + e.CheckData2(0, 0, ttt, data) + + e.Execute("create database usdb precision 'us';") + e.Execute("use usdb;") + e.Execute("create stable st (ts timestamp ,speed float ) tags(time timestamp ,id int);") + e.Execute("insert into tb1 using st tags('2021-06-10 0:00:00.123456' , 1 ) values('2021-06-10T0:00:00.123456+07:00' , 1.0);") + data, _ = e.Query("select first(*) from tb1;") + ttt2, _ := time.Parse(layout, "2021-06-10 01:00:00.123456") + e.CheckData2(0, 0, ttt2, data) + + e.Execute("drop database if exists msdb;") + e.Execute("create database msdb precision 'ms';") + e.Execute("use msdb;") + e.Execute("create stable st (ts timestamp ,speed float ) tags(time timestamp ,id int);") + e.Execute("insert into tb1 using st tags('2021-06-10 0:00:00.123' , 1 ) values('2021-06-10T0:00:00.123+07:00' , 1.0);") + data, _ = e.Query("select first(*) from tb1;") + ttt3, _ := time.Parse(layout, "2021-06-10 01:00:00.123") + e.CheckData2(0, 0, ttt3, data) + fmt.Println("all test done!") + +} + +func prepareData(e *connector.Executor) { + sqlList := []string{ + "reset query cache;", + "drop database if exists db;", + "create database db;", + "use db;", + "reset query cache;", + "drop database if exists db;", + "create database db precision 'ns';", + "show databases;", + "use db;", + "create table tb (ts timestamp, speed int);", + "insert into tb values('2021-06-10 0:00:00.100000001', 1);", + "insert into tb values(1623254400150000000, 2);", + "import into tb values(1623254400300000000, 3);", + "import into tb values(1623254400299999999, 4);", + "insert into tb values(1623254400300000001, 5);", + "insert into tb values(1623254400999999999, 7);", + } + for _, sql := range sqlList { + err := executeSql(e, sql) + if err != nil { + log.Fatalf("prepare data error:%v, sql:%s", err, sql) + } + } +} + +func executeSql(e *connector.Executor, sql string) error { + _, err := e.Execute(sql) + if err != nil { + return err + } + return nil +} diff --git a/tests/pytest/client/taoshellCheckCase.py b/tests/pytest/client/taoshellCheckCase.py new file mode 100644 index 0000000000000000000000000000000000000000..936f7dfa159d2949ed7f029c3f754f6a039bce2d --- /dev/null +++ b/tests/pytest/client/taoshellCheckCase.py @@ -0,0 +1,202 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys, shutil +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import subprocess + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self) -> str: + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/debug/build/bin")] + break + return buildPath + + def execute_cmd(self,cmd): + out = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.PIPE).stderr.read().decode("utf-8") + if out.find("error:") >=0: + print(cmd) + print(out) + sys.exit() + + + + def run(self): + tdSql.prepare() + build_path = self.getBuildPath() + "/debug/build/bin" + tdLog.info("====== check tables use taos -d -k ========") + + tdSql.execute("drop database if exists test") + tdSql.execute("drop database if exists dumptest") + tdSql.execute("create database if not exists test") + tdLog.info("====== only create database test ==== ") + self.execute_cmd(build_path + "/" + "taos -d test -k 1 > res.txt 2>&1") + + tdSql.execute("use test") + tdSql.execute("create stable st (ts timestamp , id int , val double , str binary(20) ) tags (ind int)") + tdSql.execute("create table tb1 using st tags(1)") + tdLog.info("======= only create one table ==========") + self.execute_cmd(build_path + "/" + "taos -d test -k 1 > res.txt 2>&1") + + tdSql.execute("create table tb2 using st tags(2)") + tdSql.execute("create table tb3 using st tags(3)") + tdLog.info("======= only create three table =======") + self.execute_cmd(build_path + "/" + "taos -d test -k 1 > res.txt 2>&1") + + tdSql.execute("create table tb4 using st tags(4)") + tdSql.execute("create table tb5 using st tags(5)") + tdLog.info("======= only create five table =======") + self.execute_cmd(build_path + "/" + "taos -d test -k 1 > res.txt 2>&1") + + start_time = 1604298064000 + rows = 10 + tb_nums = 5 + tdLog.info("====== start insert rows ========") + + for i in range(1, tb_nums + 1): + for j in range(rows): + start_time += 10 + tdSql.execute( + "insert into tb%d values(%d, %d,%f,%s) " % (i, start_time, j, float(j), "'str" + str(j) + "'")) + tdSql.query("select count(*) from st") + tdSql.checkData(0, 0, 50) + + for i in range(1, tb_nums + 1): + tdSql.execute("select * from test.tb%s" % (str(i))) + + tdLog.info("====== check taos -D filedir ========") + + if not os.path.exists("./dumpdata"): + os.mkdir("./dumpdata") + else: + shutil.rmtree("./dumpdata") + os.mkdir("./dumpdata") + + os.system(build_path + "/" + "taosdump -D test -o ./dumpdata") + sleep(2) + os.system("cd ./dumpdata && mv dbs.sql tables.sql") + os.system('sed -i "s/test/dumptest/g" `grep test -rl ./dumpdata`') + os.system(build_path + "/" + "taos -D ./dumpdata") + tdSql.query("select count(*) from dumptest.st") + tdSql.checkData(0, 0, 50) + + tdLog.info("========test other file name about tables.sql========") + os.system("rm -rf ./dumpdata/*") + os.system(build_path + "/" + "taosdump -D test -o ./dumpdata") + sleep(2) + os.system("cd ./dumpdata && mv dbs.sql table.sql") + os.system('sed -i "s/test/tt/g" `grep test -rl ./dumpdata`') + cmd = build_path + "/" + "taos -D ./dumpdata" + out = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.PIPE).stderr.read().decode("utf-8") + if out.find("error:") >=0: + print("===========expected error occured======") + + + tdLog.info("====== check taos shell params ========") + + tdLog.info("====== step 1 : insert data with some unicode ========") + + sqls = ["drop database if exists dbst", + "create database dbst", + "use dbst", + "create stable dbst.st (ts timestamp , id int , val double , str binary(200) ,char nchar(200) ) tags (ind int)", + "create table dbst.tb1 using dbst.st tags(1)", + "create table dbst.tb2 using dbst.st tags(2)", + "insert into dbst.tb1 values('2021-07-14T10:40:00.006+0800' , 1 , 1.0 , 'binary_1','中文-1') ", + "insert into dbst.tb1 values('2021-07-14T10:40:00.006Z' , 1 , 1.0 , 'binary\\'1','中文?-1')", + "insert into dbst.tb1 values('2021-07-14 10:40:00.000',1,1.0,'！@#¥%……&*', '中文12&%#@!*')", + "insert into dbst.tb1 values(now ,1,1.0,'(){}[];./?&*\n', '中文&%#@!*34')", + "insert into dbst.tb1 values(now ,1,1.0,'\\t\\0', '中文_\\t\\0')", + # "insert into dbst.tb1 values(now ,1,1.0,'\t\"', '中文_\t\\')", + "CREATE STABLE dbst.stb (TS TIMESTAMP , ID INT , VAL DOUBLE , STR BINARY(200) ,CHAR NCHAR(200) ) TAGS (IND INT)", + "CREATE TABLE dbst.tbb1 USING dbst.STB TAGS(1)", + "CREATE TABLE dbst.tbb2 USING dbst.STB TAGS(2)", + "INSERT INTO dbst.TBB1 VALUES('2021-07-14T10:40:00.006+0800' , 1 , 1.0 , 'BINARY_1','中文-1')", + "INSERT INTO dbst.TBB1 VALUES('2021-07-14T10:40:00.006Z' , 1 , 1.0 , 'BINARY1','中文?-1')", + "INSERT INTO dbst.TBB1 VALUES('2021-07-14 10:40:00.000',1,1.0,'！@#¥%……&*', '中文12&%#@!*');"] + for sql in sqls: + cmd = build_path + "/" + "taos -s \""+sql+"\"" + self.execute_cmd(cmd) + + basic_code = ['!' ,'#', '$', '%', '&', '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', + '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'A', + 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M','N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', '\\',']' ,'^', '_', '`', 'a', 'b', 'c', + 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r','s', 't', 'u', + 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~'] + for code in basic_code: + # bug -> : this is a bug need be repaired to support '`' and '\' + if code=='\\': + cmd = build_path + "/" + "taos -s \" insert into dbst.tb2 values(now ,2,2.0," +r'"\\"'+",'中文"+r'\\'+ "')\"" + continue + elif code =='`': + cmd = build_path + "/" + "taos -s \" insert into dbst.tb2 values(now ,2,2.0,'"+code+"','汉字"+code+"\')\"" + continue + else: + cmd = build_path + "/" + "taos -s \" insert into dbst.tb2 values(now ,2,2.0,'"+code+"','汉字"+code+"\')\"" + + self.execute_cmd(cmd) + + + tdLog.info("====== step 2 : query result of results ========") + + querys = ["select count(*) from dbst.tb2", + "show dbst.tables", + "show dbst.tables like tb_", + "show dbst.tables like 't%'", + "select * from dbst.stb", + "select avg(val),max(id),min(id) from dbst.st ", + "select last_row(*) from dbst.st", + "select * from dbst.st where ts >'2021-07-14T10:40:00.006+0800' and ind = 1 ", + "select max(val) from dbst.st where ts >'2021-07-14T10:40:00.006+0800' group by tbname", + "select count(*) from dbst.st interval(1s) group by tbname", + "show queries ", + "show connections", + "show functions", + "select * from dbst.tb2 where str like 'a'", + "select bottom(id, 3) from dbst.st; ", + "select _block_dist() from dbst.st;", + "select 5 from dbst.tb1;", + "select id , val from dbst.st", + "describe dbst.st", + "alter stable dbst.st modify column str binary(205);" ] + + for query in querys: + cmd = build_path + "/" + "taos -s \""+query+"\"" + self.execute_cmd(cmd) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/crash_gen/valgrind_taos.supp b/tests/pytest/crash_gen/valgrind_taos.supp index 8c35778018b9c34789f862f6a728e487694357f4..6b3a04353d30b1109f40db2053d56122bd9df290 100644 --- a/tests/pytest/crash_gen/valgrind_taos.supp +++ b/tests/pytest/crash_gen/valgrind_taos.supp @@ -18213,4 +18213,21 @@ obj:/usr/bin/python3.8 fun:PyVectorcall_Call fun:_PyEval_EvalFrameDefault +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:_buffer_get_info + fun:array_getbuffer + fun:__Pyx__GetBufferAndValidate.constprop.64 + fun:__pyx_f_5numpy_6random_13bit_generator_12SeedSequence_mix_entropy + fun:__pyx_pw_5numpy_6random_13bit_generator_12SeedSequence_1__init__ + obj:/usr/bin/python3.8 + fun:__Pyx__PyObject_CallOneArg + fun:__Pyx_PyObject_CallOneArg + fun:__pyx_pw_5numpy_6random_13bit_generator_12BitGenerator_1__init__ + obj:/usr/bin/python3.8 + obj:/usr/bin/python3.8 } \ No newline at end of file diff --git a/tests/pytest/fulltest.bat b/tests/pytest/fulltest.bat new file mode 100644 index 0000000000000000000000000000000000000000..ad3803b01fc3733d2c17a7706e3dc4dee4d97f04 --- /dev/null +++ b/tests/pytest/fulltest.bat @@ -0,0 +1,21 @@ +python .\test.py -f insert\basic.py +python .\test.py -f insert\int.py +python .\test.py -f insert\float.py +python .\test.py -f insert\bigint.py +python .\test.py -f insert\bool.py +python .\test.py -f insert\double.py +python .\test.py -f insert\smallint.py +python .\test.py -f insert\tinyint.py +python .\test.py -f insert\date.py +python .\test.py -f insert\binary.py +python .\test.py -f insert\nchar.py + +python .\test.py -f query\filter.py +python .\test.py -f query\filterCombo.py +python .\test.py -f query\queryNormal.py +python .\test.py -f query\queryError.py +python .\test.py -f query\filterAllIntTypes.py +python .\test.py -f query\filterFloatAndDouble.py +python .\test.py -f query\filterOtherTypes.py +python .\test.py -f query\querySort.py +python .\test.py -f query\queryJoin.py \ No newline at end of file diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 989d240129aaa5539e86c372f9ffab7818a13ef8..397ff1f748fcaeb96d20bde2af025a2636712547 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -45,7 +45,7 @@ python3 ./test.py -f table/del_stable.py #stable python3 ./test.py -f stable/insert.py -python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJsonStmt.py +# python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJsonStmt.py # tag python3 ./test.py -f tag_lite/filter.py @@ -211,7 +211,7 @@ python3 ./test.py -f perfbenchmark/bug3433.py python3 ./test.py -f perfbenchmark/taosdemoInsert.py #taosdemo -python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py +# python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py #query @@ -263,7 +263,7 @@ python3 ./test.py -f query/nestedQuery/queryInterval.py python3 ./test.py -f query/queryStateWindow.py # python3 ./test.py -f query/nestedQuery/queryWithOrderLimit.py python3 ./test.py -f query/nestquery_last_row.py -#python3 ./test.py -f query/nestedQuery/nestedQuery.py +python3 ./test.py -f query/nestedQuery/nestedQuery.py python3 ./test.py -f query/queryCnameDisplay.py python3 ./test.py -f query/operator_cost.py python3 test.py -f query/nestedQuery/queryWithSpread.py @@ -293,6 +293,7 @@ python3 ./test.py -f client/client.py python3 ./test.py -f client/version.py python3 ./test.py -f client/alterDatabase.py python3 ./test.py -f client/noConnectionErrorTest.py +python3 ./test.py -f client/taoshellCheckCase.py # python3 test.py -f client/change_time_1_1.py # python3 test.py -f client/change_time_1_2.py diff --git a/tests/pytest/insert/binary.py b/tests/pytest/insert/binary.py index 0cbb7876c6194041a160f8fee7271f0c76d3b90c..e91a20e65cd04dd64a88af88259e8e25eebf595c 100644 --- a/tests/pytest/insert/binary.py +++ b/tests/pytest/insert/binary.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import platform import sys from util.log import * from util.cases import * @@ -53,9 +54,10 @@ class TDTestCase: tdLog.info("tdSql.checkData(0, 0, '34567')") tdSql.checkData(0, 0, '34567') tdLog.info("insert into tb values (now+4a, \"'';\")") - config_dir = subprocess.check_output(str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), stderr=subprocess.STDOUT, shell=True).decode('utf-8').replace('\n', '') - result = ''.join(os.popen(r"""taos -s "insert into db.tb values (now+4a, \"'';\")" -c %s"""%(config_dir)).readlines()) - if "Query OK" not in result: tdLog.exit("err:insert '';") + if platform.system() == "Linux": + config_dir = subprocess.check_output(str("ps -ef |grep dnode1|grep -v grep |awk '{print $NF}'"), stderr=subprocess.STDOUT, shell=True).decode('utf-8').replace('\n', '') + result = ''.join(os.popen(r"""taos -s "insert into db.tb values (now+4a, \"'';\")" -c %s"""%(config_dir)).readlines()) + if "Query OK" not in result: tdLog.exit("err:insert '';") tdLog.info('drop database db') tdSql.execute('drop database db') tdLog.info('show databases') diff --git a/tests/pytest/insert/nchar.py b/tests/pytest/insert/nchar.py index 5ad52b96a1555b3ccd622fd4bf88c7a0b26051b5..023da5b014864a2d010e6ec6acc16a33ccb20424 100644 --- a/tests/pytest/insert/nchar.py +++ b/tests/pytest/insert/nchar.py @@ -15,6 +15,7 @@ import sys from util.log import * from util.cases import * from util.sql import * +import platform class TDTestCase: @@ -37,7 +38,7 @@ class TDTestCase: tdSql.error("insert into tb values (now, 'taosdata001')") - tdSql.error("insert into tb(now, 😀)") + if platform.system() == "Linux" : tdSql.error("insert into tb(now, 😀)") tdSql.query("select * from tb") tdSql.checkRows(2) diff --git a/tests/pytest/query/nestedQuery/nestedQuery.py b/tests/pytest/query/nestedQuery/nestedQuery.py index 70158c16b59f9174a21f2f21107d2f0f3d03a0e7..453ee8f53975509c318486242c634d3b60de4992 100755 --- a/tests/pytest/query/nestedQuery/nestedQuery.py +++ b/tests/pytest/query/nestedQuery/nestedQuery.py @@ -217,7 +217,18 @@ class TDTestCase: 't1.q_tinyint >= -127 and t1.q_tinyint <= 127 and t2.q_tinyint >= -127 and t2.q_tinyint <= 127', 't1.q_float >= -100000 and t1.q_float <= 100000 and t2.q_float >= -100000 and t2.q_float <= 100000', 't1.q_double >= -1000000000 and t1.q_double <= 1000000000 and t2.q_double >= -1000000000 and t2.q_double <= 1000000000', - 't1.q_binary like \'binary%\' or t1.q_binary = \'0\' or t2.q_binary like \'binary%\' or t2.q_binary = \'0\' ' , + 't1.q_binary like \'binary%\' and t2.q_binary like \'binary%\' ' , + 't1.q_nchar like \'nchar%\' and t2.q_nchar like \'nchar%\' ' , + 't1.q_bool in (0 , 1) and t2.q_bool in (0 , 1)' , 't1.q_bool in ( true , false) and t2.q_bool in ( true , false)' , + 't1.q_bigint between -9223372036854775807 and 9223372036854775807 and t2.q_bigint between -9223372036854775807 and 9223372036854775807', + 't1.q_int between -2147483647 and 2147483647 and t2.q_int between -2147483647 and 2147483647', + 't1.q_smallint between -32767 and 32767 and t2.q_smallint between -32767 and 32767', + 't1.q_tinyint between -127 and 127 and t2.q_tinyint between -127 and 127 ','t1.q_float between -100000 and 100000 and t2.q_float between -100000 and 100000', + 't1.q_double between -1000000000 and 1000000000 and t2.q_double between -1000000000 and 1000000000'] + #TD-6201 ,'t1.q_bool between 0 and 1 or t2.q_bool between 0 and 1'] + #'t1.q_bool = true and t1.q_bool = false and t2.q_bool = true and t2.q_bool = false' , 't1.q_bool = 0 and t1.q_bool = 1 and t2.q_bool = 0 and t2.q_bool = 1' , + + q_u_or_where = ['t1.q_binary like \'binary%\' or t1.q_binary = \'0\' or t2.q_binary like \'binary%\' or t2.q_binary = \'0\' ' , 't1.q_nchar like \'nchar%\' or t1.q_nchar = \'0\' or t2.q_nchar like \'nchar%\' or t2.q_nchar = \'0\' ' , 't1.q_bool = true or t1.q_bool = false or t2.q_bool = true or t2.q_bool = false' , 't1.q_bool in (0 , 1) or t2.q_bool in (0 , 1)' , 't1.q_bool in ( true , false) or t2.q_bool in ( true , false)' , 't1.q_bool = 0 or t1.q_bool = 1 or t2.q_bool = 0 or t2.q_bool = 1' , 't1.q_bigint between -9223372036854775807 and 9223372036854775807 or t2.q_bigint between -9223372036854775807 and 9223372036854775807', @@ -225,7 +236,6 @@ class TDTestCase: 't1.q_smallint between -32767 and 32767 or t2.q_smallint between -32767 and 32767', 't1.q_tinyint between -127 and 127 or t2.q_tinyint between -127 and 127 ','t1.q_float between -100000 and 100000 or t2.q_float between -100000 and 100000', 't1.q_double between -1000000000 and 1000000000 or t2.q_double between -1000000000 and 1000000000'] - #TD-6201 ,'t1.q_bool between 0 and 1 or t2.q_bool between 0 and 1'] # tag column where t_where = ['ts < now +1s','t_bigint >= -9223372036854775807 and t_bigint <= 9223372036854775807','t_int <= 2147483647 and t_int >= -2147483647', @@ -245,17 +255,28 @@ class TDTestCase: 't1.t_double >= -1000000000 and t1.t_double <= 1000000000 and t2.t_double >= -1000000000 and t2.t_double <= 1000000000', 't1.t_binary like \'binary%\' or t1.t_binary = \'0\' or t2.t_binary like \'binary%\' or t2.t_binary = \'0\' ' , 't1.t_nchar like \'nchar%\' or t1.t_nchar = \'0\' or t2.t_nchar like \'nchar%\' or t2.t_nchar = \'0\' ' , 't1.t_bool = true or t1.t_bool = false or t2.t_bool = true or t2.t_bool = false' , + 't1.t_bool in (0 , 1) and t2.t_bool in (0 , 1)' , 't1.t_bool in ( true , false) and t2.t_bool in ( true , false)' , 't1.t_bool = 0 or t1.t_bool = 1 or t2.t_bool = 0 or t2.t_bool = 1', + 't1.t_bigint between -9223372036854775807 and 9223372036854775807 and t2.t_bigint between -9223372036854775807 and 9223372036854775807', + 't1.t_int between -2147483647 and 2147483647 and t2.t_int between -2147483647 and 2147483647', + 't1.t_smallint between -32767 and 32767 and t2.t_smallint between -32767 and 32767', + 't1.t_tinyint between -127 and 127 and t2.t_tinyint between -127 and 127 ','t1.t_float between -100000 and 100000 and t2.t_float between -100000 and 100000', + 't1.t_double between -1000000000 and 1000000000 and t2.t_double between -1000000000 and 1000000000'] + #TD-6201,'t1.t_bool between 0 and 1 or t2.q_bool between 0 and 1'] + + t_u_or_where = ['t1.t_binary like \'binary%\' or t1.t_binary = \'0\' or t2.t_binary like \'binary%\' or t2.t_binary = \'0\' ' , + 't1.t_nchar like \'nchar%\' or t1.t_nchar = \'0\' or t2.t_nchar like \'nchar%\' or t2.t_nchar = \'0\' ' , 't1.t_bool = true or t1.t_bool = false or t2.t_bool = true or t2.t_bool = false' , 't1.t_bool in (0 , 1) or t2.t_bool in (0 , 1)' , 't1.t_bool in ( true , false) or t2.t_bool in ( true , false)' , 't1.t_bool = 0 or t1.t_bool = 1 or t2.t_bool = 0 or t2.t_bool = 1', 't1.t_bigint between -9223372036854775807 and 9223372036854775807 or t2.t_bigint between -9223372036854775807 and 9223372036854775807', 't1.t_int between -2147483647 and 2147483647 or t2.t_int between -2147483647 and 2147483647', 't1.t_smallint between -32767 and 32767 or t2.t_smallint between -32767 and 32767', 't1.t_tinyint between -127 and 127 or t2.t_tinyint between -127 and 127 ','t1.t_float between -100000 and 100000 or t2.t_float between -100000 and 100000', 't1.t_double between -1000000000 and 1000000000 or t2.t_double between -1000000000 and 1000000000'] - #TD-6201,'t1.t_bool between 0 and 1 or t2.q_bool between 0 and 1'] # regular and tag column where qt_where = q_where + t_where qt_u_where = q_u_where + t_u_where + # now,qt_u_or_where is not support + qt_u_or_where = q_u_or_where + t_u_or_where # tag column where for test super join | this is support , 't1.t_bool = t2.t_bool ' ？？？ t_join_where = ['t1.t_bigint = t2.t_bigint ', 't1.t_int = t2.t_int ', 't1.t_smallint = t2.t_smallint ', 't1.t_tinyint = t2.t_tinyint ', @@ -535,7 +556,22 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) #tdSql.checkData(0,0,'2020-09-13 20:26:40.000') - tdSql.checkRows(6*self.num) + #tdSql.checkRows(6*self.num) + + tdSql.query("select 1-5 from table_0;") + for i in range(self.fornum): + sql = "select ts , * from ( select t1.ts ," + sql += "t1.%s, " % random.choice(s_s_select) + sql += "t1.%s, " % random.choice(q_select) + sql += "t2.%s, " % random.choice(s_s_select) + sql += "t2.%s, " % random.choice(q_select) + sql += "t2.ts from regular_table_1 t1 , regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += ");" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) #2 select column from (select * form regular_table ) where <\>\in\and\or order by dcDB = self.dropandcreateDB(random.randint(1,2)) @@ -672,6 +708,21 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.error(sql) + tdSql.query("select 3-6 from table_0;") + for i in range(self.fornum): + sql = "select ts , * from ( select t1.ts ," + sql += "t1.%s, " % random.choice(s_s_select) + sql += "t1.%s, " % random.choice(q_select) + sql += "t2.%s, " % random.choice(s_s_select) + sql += "t2.%s, " % random.choice(q_select) + sql += "t2.ts from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(t_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += ");" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + #4 select column from (select * form stable where <\>\in\and\or order by ) dcDB = self.dropandcreateDB(random.randint(1,2)) tdSql.query("select 4-1 from table_0;") @@ -775,6 +826,19 @@ class TDTestCase: tdLog.info(sql) tdLog.info(len(sql)) tdSql.query(sql) + + tdSql.query("select 8-3 from table_0;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(calc_select_in_ts_j) + sql += "from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) #9 select * from (select ts,calc form stable where <\>\in\and\or order by ) # TD-5960\TD-6185 @@ -808,6 +872,19 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + tdSql.query("select 9-3 from table_0;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(calc_select_in_ts_j) + sql += "from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit1_where) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + #functions or others can not be mixed up ,calc out select not use with ts #10 select calc from (select * form regualr_table where <\>\in\and\or order by ) @@ -860,6 +937,20 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + tdSql.query("select 10-4 from table_0;") + for i in range(self.fornum): + sql = "select " + sql += "%s as calc10_1 " % random.choice(calc_select_all) + sql += " from ( select * from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s ;" % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + #11 select calc from (select * form stable where <\>\in\and\or order by limit ) dcDB = self.dropandcreateDB(random.randint(1,2)) tdSql.query("select 11-1 from table_0;") @@ -907,6 +998,20 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + tdSql.query("select 11-4 from table_0;") + for i in range(self.fornum): + sql = "select " + sql += "%s " % random.choice(calc_select_all) + sql += "as calc11_1 from ( select * from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s ;" % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + #12 select calc-diff from (select * form regualr_table where <\>\in\and\or order by limit ) dcDB = self.dropandcreateDB(random.randint(1,3)) tdSql.query("select 12-1 from table_0;") @@ -935,7 +1040,20 @@ class TDTestCase: tdLog.info(sql) tdLog.info(len(sql)) tdSql.query(sql) - tdSql.checkRows(1) + #tdSql.checkRows(1) + + tdSql.query("select 12-2.2 from table_0;") + for i in range(self.fornum): + sql = "select " + sql += "%s " % random.choice(calc_calculate_regular) + sql += " from ( select * from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice([limit_where[2] , limit_where[3]] ) + sql += ") ;" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) #12-1 select calc-diff from (select * form stable where <\>\in\and\or order by limit ) tdSql.query("select 12-3 from table_0;") @@ -969,6 +1087,22 @@ class TDTestCase: tdLog.info(sql) tdLog.info(len(sql)) tdSql.error(sql) + + tdSql.query("select 12-5 from table_0;") + #join query does not support group by + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s " % random.choice(calc_calculate_regular_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(group_where) + sql += ") " + sql += "%s " % random.choice(order_desc_where) + sql += "%s " % random.choice([limit_where[2] , limit_where[3]] ) + sql += " ;" + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) #13 select calc-diff as diffns from (select * form stable where <\>\in\and\or order by limit ) @@ -1041,6 +1175,22 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + tdSql.query("select 14-4 from table_0;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc14_1, " % random.choice(calc_aggregate_all_j) + sql += "%s as calc14_2, " % random.choice(calc_aggregate_all_j) + sql += "%s " % random.choice(calc_aggregate_all_j) + sql += " as calc14_3 from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(slimit1_where) + sql += ") " + sql += "%s ;" % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + #15 TD-6320 select * from (select calc_aggregate_regulars as agg from regular_table where <\>\in\and\or order by slimit soffset ) tdSql.query("select 15-1 from table_0;") for i in range(self.fornum): @@ -1074,6 +1224,22 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + tdSql.query("select 15-2.2 from table_0;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc15_1, " % random.choice(calc_aggregate_regular_j) + sql += "%s as calc15_2, " % random.choice(calc_aggregate_regular_j) + sql += "%s " % random.choice(calc_aggregate_regular_j) + sql += " as calc15_3 from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s ;" % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + rsDn = self.restartDnodes() tdSql.query("select 15-3 from table_0;") for i in range(self.fornum): @@ -1111,6 +1277,24 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.error(sql) + tdSql.query("select 15-4.2 from table_0;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc15_1, " % random.choice(calc_aggregate_groupbytbname_j) + sql += "%s as calc15_2, " % random.choice(calc_aggregate_groupbytbname_j) + sql += "%s " % random.choice(calc_aggregate_groupbytbname_j) + sql += " as calc15_3 from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(group_where) + sql += "%s " % random.choice(having_support) + sql += "%s " % random.choice(orders_desc_where) + sql += ") " + sql += "order by calc15_1 " + sql += "%s " % random.choice(limit_u_where) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + tdSql.query("select 15-5 from table_0;") for i in range(self.fornum): sql = "select * from ( select " @@ -1161,6 +1345,20 @@ class TDTestCase: tdLog.info(sql) tdLog.info(len(sql)) tdSql.query(sql) + + tdSql.query("select 16-2.2 from table_0;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc16_0 " % random.choice(calc_calculate_all_j) + sql += ", %s as calc16_1 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += ") " + sql += "order by calc16_0 " + sql += "%s " % random.choice(limit1_where) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) dcDB = self.dropandcreateDB(random.randint(1,3)) tdSql.query("select 16-3 from table_0;") @@ -1188,6 +1386,18 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) tdSql.checkRows(1) + + tdSql.query("select 16-4.2 from table_0;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc16_1 " % random.choice(calc_calculate_regular_j) + sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "limit 2 ) " + sql += "%s " % random.choice(limit1_where) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) tdSql.query("select 16-5 from table_0;") for i in range(self.fornum): @@ -1233,6 +1443,18 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.error(sql) + tdSql.query("select 16-8 from table_0;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s as calc16_1 " % random.choice(calc_calculate_groupbytbname_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "limit 2 ) " + sql += "%s " % random.choice(limit1_where) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + #17 select apercentile from (select calc_aggregate_alls form regualr_table or stable where <\>\in\and\or interval_sliding group by having order by limit offset )interval_sliding # TD-6088 dcDB = self.dropandcreateDB(random.randint(1,2)) @@ -1275,6 +1497,24 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + tdSql.query("select 17-2.2 from table_0;") + for i in range(self.fornum): + #this is having_support , but tag-select cannot mix with last_row,other select can + sql = "select apercentile(cal17_0, %d)/10 ,apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_0 , " % random.choice(calc_calculate_all_j) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + rsDn = self.restartDnodes() tdSql.query("select 17-3 from table_0;") for i in range(self.fornum): @@ -1312,6 +1552,23 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + tdSql.query("select 17-4.2 from table_0;") + for i in range(self.fornum): + #this is having_tagnot_support , because tag-select cannot mix with last_row... + sql = "select apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + tdSql.query("select 17-5 from table_0;") for i in range(self.fornum): #having_not_support @@ -1364,6 +1621,22 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + tdSql.query("select 17-7.2 from table_0;") + for i in range(self.fornum): + sql = "select apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all_j) + sql += " from table_1 t1, table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit1_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + rsDn = self.restartDnodes() tdSql.query("select 17-8 from table_0;") for i in range(self.fornum): @@ -1398,6 +1671,22 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + tdSql.query("select 17-10 from table_0;") + for i in range(self.fornum): + sql = "select apercentile(cal17_1, %d)/1000 ,apercentile(cal17_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal17_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal17_2 " % random.choice(calc_aggregate_all_j) + sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(interval_sliding) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + #18 select apercentile from (select calc_aggregate_alls form regualr_table or stable where <\>\in\and\or session order by limit )interval_sliding tdSql.query("select 18-1 from table_0;") for i in range(self.fornum): @@ -1434,6 +1723,23 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + tdSql.query("select 18-2.2 from table_0;") + for i in range(self.fornum): + sql = "select apercentile(cal18_1, %d)/1000 ,apercentile(cal18_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal18_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal18_2 " % random.choice(calc_aggregate_all_j) + sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(session_u_where) + sql += "%s " % random.choice(fill_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + rsDn = self.restartDnodes() tdSql.query("select 18-3 from table_0;") for i in range(self.fornum): @@ -1470,6 +1776,23 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + tdSql.query("select 18-4.2 from table_0;") + for i in range(self.fornum): + sql = "select apercentile(cal18_1, %d)/1000 ,apercentile(cal18_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal18_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal18_2 " % random.choice(calc_aggregate_all_j) + sql += " from table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(session_u_where) + sql += "%s " % random.choice(fill_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + tdSql.query("select 18-5 from table_0;") for i in range(self.fornum): sql = "select apercentile(cal18_1, %d)/1000 ,apercentile(cal18_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) @@ -1503,6 +1826,23 @@ class TDTestCase: tdLog.info(sql) tdLog.info(len(sql)) tdSql.error(sql) + + tdSql.query("select 18-7 from table_0;") + for i in range(self.fornum): + sql = "select apercentile(cal18_1, %d)/1000 ,apercentile(cal18_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal18_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal18_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(session_u_where) + sql += "%s " % random.choice(fill_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) #19 select apercentile from (select calc_aggregate_alls form regualr_table or stable where <\>\in\and\or session order by limit )interval_sliding dcDB = self.dropandcreateDB(random.randint(1,3)) @@ -1536,6 +1876,21 @@ class TDTestCase: tdLog.info(sql) tdLog.info(len(sql)) tdSql.error(sql) + + tdSql.query("select 19-2.2 from table_0;") + for i in range(self.fornum): + sql = "select apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal19_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal19_2 " % random.choice(calc_aggregate_all_j) + sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(state_u_window) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) dcDB = self.dropandcreateDB(random.randint(1,2)) tdSql.query("select 19-3 from table_0;") @@ -1568,6 +1923,20 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + tdSql.query("select 19-4.2 from table_0;") + for i in range(self.fornum): + sql = "select apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal19_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal19_2 " % random.choice(calc_aggregate_all_j) + sql += " from table_1 t1, table_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(q_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + tdSql.query("select 19-5 from table_0;") for i in range(self.fornum): sql = "select apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) @@ -1601,6 +1970,21 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.error(sql) + tdSql.query("select 19-7 from table_0;") + for i in range(self.fornum): + sql = "select apercentile(cal19_1, %d)/1000 ,apercentile(cal19_2, %d)*10+%d from ( select " %(random.randint(0,100) , random.randint(0,100) ,random.randint(-1000,1000)) + sql += "%s as cal19_1 ," % random.choice(calc_aggregate_all_j) + sql += "%s as cal19_2 " % random.choice(calc_aggregate_all_j) + sql += " from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " + sql += "%s " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + sql += "%s " % random.choice(interval_sliding) + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + #20 select * from (select calc_select_fills form regualr_table or stable where <\>\in\and\or fill_where group by order by limit offset ) dcDB = self.dropandcreateDB(random.randint(1,2)) tdSql.query("select 20-1 from table_0;") @@ -1639,6 +2023,23 @@ class TDTestCase: tdLog.info(len(sql)) tdSql.query(sql) + tdSql.query("select 20-2.2 from table_0;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s , " % random.choice(calc_select_fill_j) + sql += "%s ," % random.choice(calc_select_fill_j) + sql += "%s " % random.choice(calc_select_fill_j) + sql += " from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " + sql += "%s and " % random.choice(qt_u_or_where) + sql += "%s " % random.choice(interp_where_j) + sql += "%s " % random.choice(fill_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) + tdSql.query("select 20-3 from table_0;") for i in range(self.fornum): sql = "select * from ( select " @@ -1671,6 +2072,23 @@ class TDTestCase: tdLog.info(sql) tdLog.info(len(sql)) tdSql.query(sql) + + tdSql.query("select 20-4.2 from table_0;") + for i in range(self.fornum): + sql = "select * from ( select " + sql += "%s , " % random.choice(calc_select_fill_j) + sql += "%s ," % random.choice(calc_select_fill_j) + sql += "%s " % random.choice(calc_select_fill_j) + sql += " from table_0 t1, table_1 t2 where t1.ts = t2.ts and " + sql += "%s and " % random.choice(qt_u_or_where) + sql += "%s " % interp_where_j[random.randint(0,5)] + sql += "%s " % random.choice(fill_where) + sql += "%s " % random.choice(order_u_where) + sql += "%s " % random.choice(limit_u_where) + sql += ") " + tdLog.info(sql) + tdLog.info(len(sql)) + tdSql.error(sql) dcDB = self.dropandcreateDB(random.randint(1,2)) tdSql.query("select 20-5 from table_0;") @@ -1983,4 +2401,4 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/queryNormal.py b/tests/pytest/query/queryNormal.py index 52e49a57c6883f6fe57df887756bbf2d27199806..a1789c8909f542ba3dcae83042ab50cde9e58e32 100644 --- a/tests/pytest/query/queryNormal.py +++ b/tests/pytest/query/queryNormal.py @@ -17,6 +17,7 @@ from util.log import * from util.cases import * from util.sql import * from util.dnodes import * +import platform class TDTestCase: def init(self, conn, logSql): @@ -137,8 +138,9 @@ class TDTestCase: tdSql.checkData(1, 1, 421) tdSql.checkData(1, 2, "tm1") - tdDnodes.stop(1) - tdDnodes.start(1) + if platform.system() == "Linux": + tdDnodes.stop(1) + tdDnodes.start(1) tdSql.query("select last(*) from m1 group by tbname") tdSql.checkData(0, 0, "2020-03-01 01:01:01") diff --git a/tests/pytest/test-all.bat b/tests/pytest/test-all.bat new file mode 100644 index 0000000000000000000000000000000000000000..1f1e2c1727527e91f7632213992607d6221eac85 --- /dev/null +++ b/tests/pytest/test-all.bat @@ -0,0 +1,15 @@ +@echo off +SETLOCAL EnableDelayedExpansion +for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a") +for /F "usebackq tokens=*" %%i in (fulltest.bat) do ( + echo Processing %%i + call %%i ARG1 -w 1 -m %1 > result.txt 2>error.txt + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) +) +exit + +:colorEcho +echo off + "%~2" +findstr /v /a:%1 /R "^$" "%~2" nul +del "%~2" > nul 2>&1i \ No newline at end of file diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 97dca6be1811ee87a31661e018616f469d5fd4ca..a96ac21496431b811f26fa82091c92f6ae8ecb9a 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -18,6 +18,7 @@ import getopt import subprocess import time from distutils.log import warn as printf +from fabric2 import Connection from util.log import * from util.dnodes import * @@ -35,8 +36,9 @@ if __name__ == "__main__": logSql = True stop = 0 restart = False - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghr', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help']) + windows = 0 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrw', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'windows']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -49,6 +51,7 @@ if __name__ == "__main__": tdLog.printNoPrefix('-c Test Cluster Flag') tdLog.printNoPrefix('-g valgrind Test Flag') tdLog.printNoPrefix('-r taosd restart test') + tdLog.printNoPrefix('-w taos on windows') sys.exit(0) if key in ['-r', '--restart']: @@ -81,6 +84,9 @@ if __name__ == "__main__": if key in ['-s', '--stop']: stop = 1 + if key in ['-w', '--windows']: + windows = 1 + if (stop != 0): if (valgrind == 0): toBeKilled = "taosd" @@ -111,66 +117,81 @@ if __name__ == "__main__": tdLog.info('stop All dnodes') - tdDnodes.init(deployPath) - tdDnodes.setTestCluster(testCluster) - tdDnodes.setValgrind(valgrind) - tdDnodes.stopAll() - is_test_framework = 0 - key_word = 'tdCases.addLinux' - try: - if key_word in open(fileName).read(): - is_test_framework = 1 - except: - pass - if is_test_framework: - moduleName = fileName.replace(".py", "").replace("/", ".") - uModule = importlib.import_module(moduleName) - try: - ucase = uModule.TDTestCase() - tdDnodes.deploy(1,ucase.updatecfgDict) - except : - tdDnodes.deploy(1,{}) - else: - tdDnodes.deploy(1,{}) - tdDnodes.start(1) - if masterIp == "": host = '127.0.0.1' else: host = masterIp tdLog.info("Procedures for tdengine deployed in %s" % (host)) - - tdCases.logSql(logSql) - - if testCluster: - tdLog.info("Procedures for testing cluster") - if fileName == "all": - tdCases.runAllCluster() - else: - tdCases.runOneCluster(fileName) - else: + if windows: + tdCases.logSql(logSql) tdLog.info("Procedures for testing self-deployment") + td_clinet = TDSimClient("C:\\TDengine") + td_clinet.deploy() + remote_conn = Connection("root@%s"%host) + with remote_conn.cd('/var/lib/jenkins/workspace/TDinternal/community/tests/pytest'): + remote_conn.run("python3 ./test.py") conn = taos.connect( - host, - config=tdDnodes.getSimCfgPath()) - if fileName == "all": - tdCases.runAllLinux(conn) + host="%s"%(host), + config=td_clinet.cfgDir) + tdCases.runOneWindows(conn, fileName) + else: + tdDnodes.init(deployPath) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + tdDnodes.stopAll() + is_test_framework = 0 + key_word = 'tdCases.addLinux' + try: + if key_word in open(fileName).read(): + is_test_framework = 1 + except: + pass + if is_test_framework: + moduleName = fileName.replace(".py", "").replace("/", ".") + uModule = importlib.import_module(moduleName) + try: + ucase = uModule.TDTestCase() + tdDnodes.deploy(1,ucase.updatecfgDict) + except : + tdDnodes.deploy(1,{}) + else: + pass + tdDnodes.deploy(1,{}) + tdDnodes.start(1) + + + + tdCases.logSql(logSql) + + if testCluster: + tdLog.info("Procedures for testing cluster") + if fileName == "all": + tdCases.runAllCluster() + else: + tdCases.runOneCluster(fileName) else: - tdCases.runOneLinux(conn, fileName) - if restart: - if fileName == "all": - tdLog.info("not need to query ") - else: - sp = fileName.rsplit(".", 1) - if len(sp) == 2 and sp[1] == "py": - tdDnodes.stopAll() - tdDnodes.start(1) - time.sleep(1) - conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) - tdLog.info("Procedures for tdengine deployed in %s" % (host)) - tdLog.info("query test after taosd restart") - tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") + tdLog.info("Procedures for testing self-deployment") + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + if fileName == "all": + tdCases.runAllLinux(conn) else: - tdLog.info("not need to query") + tdCases.runOneWindows(conn, fileName) + if restart: + if fileName == "all": + tdLog.info("not need to query ") + else: + sp = fileName.rsplit(".", 1) + if len(sp) == 2 and sp[1] == "py": + tdDnodes.stopAll() + tdDnodes.start(1) + time.sleep(1) + conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) + tdLog.info("Procedures for tdengine deployed in %s" % (host)) + tdLog.info("query test after taosd restart") + tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") + else: + tdLog.info("not need to query") conn.close() diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdumpTestNanoSupport.py b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdumpTestNanoSupport.py new file mode 100644 index 0000000000000000000000000000000000000000..a2059ec924ad1e2239c2709bc99dd58fbafa1337 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdumpTestNanoSupport.py @@ -0,0 +1,362 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1625068800000000000 # this is timestamp "2021-07-01 00:00:00" + self.numberOfTables = 10 + self.numberOfRecords = 100 + + def checkCommunity(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + return False + else: + return True + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + + + def createdb(self, precision="ns"): + tb_nums = self.numberOfTables + per_tb_rows = self.numberOfRecords + + def build_db(precision, start_time): + tdSql.execute("drop database if exists timedb1") + tdSql.execute( + "create database timedb1 days 10 keep 365 blocks 8 precision "+"\""+precision+"\"") + + tdSql.execute("use timedb1") + tdSql.execute( + "create stable st(ts timestamp, c1 int, c2 nchar(10),c3 timestamp) tags(t1 int, t2 binary(10))") + for tb in range(tb_nums): + tbname = "t"+str(tb) + tdSql.execute("create table " + tbname + + " using st tags(1, 'beijing')") + sql = "insert into " + tbname + " values" + currts = start_time + if precision == "ns": + ts_seed = 1000000000 + elif precision == "us": + ts_seed = 1000000 + else: + ts_seed = 1000 + + for i in range(per_tb_rows): + sql += "(%d, %d, 'nchar%d',%d)" % (currts + i*ts_seed, i % + 100, i % 100, currts + i*100) # currts +1000ms (1000000000ns) + tdSql.execute(sql) + + if precision == "ns": + start_time = 1625068800000000000 + build_db(precision, start_time) + + elif precision == "us": + start_time = 1625068800000000 + build_db(precision, start_time) + + elif precision == "ms": + start_time = 1625068800000 + build_db(precision, start_time) + + else: + print("other time precision not valid , please check! ") + + + def run(self): + + # clear envs + os.system("rm -rf ./taosdumptest/") + tdSql.execute("drop database if exists dumptmp1") + tdSql.execute("drop database if exists dumptmp2") + tdSql.execute("drop database if exists dumptmp3") + + if not os.path.exists("./taosdumptest/tmp1"): + os.makedirs("./taosdumptest/dumptmp1") + else: + print("path exist!") + + if not os.path.exists("./taosdumptest/dumptmp2"): + os.makedirs("./taosdumptest/dumptmp2") + + if not os.path.exists("./taosdumptest/dumptmp3"): + os.makedirs("./taosdumptest/dumptmp3") + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosdump not found!") + else: + tdLog.info("taosdump found in %s" % buildPath) + binPath = buildPath + "/build/bin/" + + # create nano second database + + self.createdb(precision="ns") + + # dump all data + + os.system( + "%staosdump --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath) + + # dump part data with -S -E + os.system( + '%staosdump --databases timedb1 -S 1625068810000000000 -E 1625068860000000000 -C ns -o ./taosdumptest/dumptmp2 ' % + binPath) + os.system( + '%staosdump --databases timedb1 -S 1625068810000000000 -o ./taosdumptest/dumptmp3 ' % + binPath) + + # replace strings to dump in databases + os.system( + "sed -i \"s/timedb1/dumptmp1/g\" `grep timedb1 -rl ./taosdumptest/dumptmp1`") + os.system( + "sed -i \"s/timedb1/dumptmp2/g\" `grep timedb1 -rl ./taosdumptest/dumptmp2`") + os.system( + "sed -i \"s/timedb1/dumptmp3/g\" `grep timedb1 -rl ./taosdumptest/dumptmp3`") + + os.system( "%staosdump -i ./taosdumptest/dumptmp1" %binPath) + os.system( "%staosdump -i ./taosdumptest/dumptmp2" %binPath) + os.system( "%staosdump -i ./taosdumptest/dumptmp3" %binPath) + + # dump data and check for taosdump + tdSql.query("select count(*) from dumptmp1.st") + tdSql.checkData(0,0,1000) + + tdSql.query("select count(*) from dumptmp2.st") + tdSql.checkData(0,0,510) + + tdSql.query("select count(*) from dumptmp3.st") + tdSql.checkData(0,0,900) + + # check data + origin_res = tdSql.getResult("select * from timedb1.st") + dump_res = tdSql.getResult("select * from dumptmp1.st") + if origin_res == dump_res: + tdLog.info("test nano second : dump check data pass for all data!" ) + else: + tdLog.info("test nano second : dump check data failed for all data!" ) + + origin_res = tdSql.getResult("select * from timedb1.st where ts >=1625068810000000000 and ts <= 1625068860000000000") + dump_res = tdSql.getResult("select * from dumptmp2.st") + if origin_res == dump_res: + tdLog.info(" test nano second : dump check data pass for data! " ) + else: + tdLog.info(" test nano second : dump check data failed for data !" ) + + origin_res = tdSql.getResult("select * from timedb1.st where ts >=1625068810000000000 ") + dump_res = tdSql.getResult("select * from dumptmp3.st") + if origin_res == dump_res: + tdLog.info(" test nano second : dump check data pass for data! " ) + else: + tdLog.info(" test nano second : dump check data failed for data !" ) + + + # us second support test case + + os.system("rm -rf ./taosdumptest/") + tdSql.execute("drop database if exists dumptmp1") + tdSql.execute("drop database if exists dumptmp2") + tdSql.execute("drop database if exists dumptmp3") + + if not os.path.exists("./taosdumptest/tmp1"): + os.makedirs("./taosdumptest/dumptmp1") + else: + print("path exits!") + + if not os.path.exists("./taosdumptest/dumptmp2"): + os.makedirs("./taosdumptest/dumptmp2") + + if not os.path.exists("./taosdumptest/dumptmp3"): + os.makedirs("./taosdumptest/dumptmp3") + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosdump not found!") + else: + tdLog.info("taosdump found in %s" % buildPath) + binPath = buildPath + "/build/bin/" + + self.createdb(precision="us") + + os.system( + "%staosdump --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath) + + os.system( + '%staosdump --databases timedb1 -S 1625068810000000 -E 1625068860000000 -C us -o ./taosdumptest/dumptmp2 ' % + binPath) + os.system( + '%staosdump --databases timedb1 -S 1625068810000000 -o ./taosdumptest/dumptmp3 ' % + binPath) + + os.system( + "sed -i \"s/timedb1/dumptmp1/g\" `grep timedb1 -rl ./taosdumptest/dumptmp1`") + os.system( + "sed -i \"s/timedb1/dumptmp2/g\" `grep timedb1 -rl ./taosdumptest/dumptmp2`") + os.system( + "sed -i \"s/timedb1/dumptmp3/g\" `grep timedb1 -rl ./taosdumptest/dumptmp3`") + + os.system( "%staosdump -i ./taosdumptest/dumptmp1" %binPath) + os.system( "%staosdump -i ./taosdumptest/dumptmp2" %binPath) + os.system( "%staosdump -i ./taosdumptest/dumptmp3" %binPath) + + + tdSql.query("select count(*) from dumptmp1.st") + tdSql.checkData(0,0,1000) + + tdSql.query("select count(*) from dumptmp2.st") + tdSql.checkData(0,0,510) + + tdSql.query("select count(*) from dumptmp3.st") + tdSql.checkData(0,0,900) + + + origin_res = tdSql.getResult("select * from timedb1.st") + dump_res = tdSql.getResult("select * from dumptmp1.st") + if origin_res == dump_res: + tdLog.info("test us second : dump check data pass for all data!" ) + else: + tdLog.info("test us second : dump check data failed for all data!" ) + + origin_res = tdSql.getResult("select * from timedb1.st where ts >=1625068810000000 and ts <= 1625068860000000") + dump_res = tdSql.getResult("select * from dumptmp2.st") + if origin_res == dump_res: + tdLog.info(" test us second : dump check data pass for data! " ) + else: + tdLog.info(" test us second : dump check data failed for data!" ) + + origin_res = tdSql.getResult("select * from timedb1.st where ts >=1625068810000000 ") + dump_res = tdSql.getResult("select * from dumptmp3.st") + if origin_res == dump_res: + tdLog.info(" test us second : dump check data pass for data! " ) + else: + tdLog.info(" test us second : dump check data failed for data! " ) + + + # ms second support test case + + os.system("rm -rf ./taosdumptest/") + tdSql.execute("drop database if exists dumptmp1") + tdSql.execute("drop database if exists dumptmp2") + tdSql.execute("drop database if exists dumptmp3") + + if not os.path.exists("./taosdumptest/tmp1"): + os.makedirs("./taosdumptest/dumptmp1") + else: + print("path exits!") + + if not os.path.exists("./taosdumptest/dumptmp2"): + os.makedirs("./taosdumptest/dumptmp2") + + if not os.path.exists("./taosdumptest/dumptmp3"): + os.makedirs("./taosdumptest/dumptmp3") + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosdump not found!") + else: + tdLog.info("taosdump found in %s" % buildPath) + binPath = buildPath + "/build/bin/" + + self.createdb(precision="ms") + + os.system( + "%staosdump --databases timedb1 -o ./taosdumptest/dumptmp1" % binPath) + + os.system( + '%staosdump --databases timedb1 -S 1625068810000 -E 1625068860000 -C ms -o ./taosdumptest/dumptmp2 ' % + binPath) + os.system( + '%staosdump --databases timedb1 -S 1625068810000 -o ./taosdumptest/dumptmp3 ' % + binPath) + + os.system( + "sed -i \"s/timedb1/dumptmp1/g\" `grep timedb1 -rl ./taosdumptest/dumptmp1`") + os.system( + "sed -i \"s/timedb1/dumptmp2/g\" `grep timedb1 -rl ./taosdumptest/dumptmp2`") + os.system( + "sed -i \"s/timedb1/dumptmp3/g\" `grep timedb1 -rl ./taosdumptest/dumptmp3`") + + os.system( "%staosdump -i ./taosdumptest/dumptmp1" %binPath) + os.system( "%staosdump -i ./taosdumptest/dumptmp2" %binPath) + os.system( "%staosdump -i ./taosdumptest/dumptmp3" %binPath) + + + tdSql.query("select count(*) from dumptmp1.st") + tdSql.checkData(0,0,1000) + + tdSql.query("select count(*) from dumptmp2.st") + tdSql.checkData(0,0,510) + + tdSql.query("select count(*) from dumptmp3.st") + tdSql.checkData(0,0,900) + + + origin_res = tdSql.getResult("select * from timedb1.st") + dump_res = tdSql.getResult("select * from dumptmp1.st") + if origin_res == dump_res: + tdLog.info("test ms second : dump check data pass for all data!" ) + else: + tdLog.info("test ms second : dump check data failed for all data!" ) + + origin_res = tdSql.getResult("select * from timedb1.st where ts >=1625068810000 and ts <= 1625068860000") + dump_res = tdSql.getResult("select * from dumptmp2.st") + if origin_res == dump_res: + tdLog.info(" test ms second : dump check data pass for data! " ) + else: + tdLog.info(" test ms second : dump check data failed for data!" ) + + origin_res = tdSql.getResult("select * from timedb1.st where ts >=1625068810000 ") + dump_res = tdSql.getResult("select * from dumptmp3.st") + if origin_res == dump_res: + tdLog.info(" test ms second : dump check data pass for data! " ) + else: + tdLog.info(" test ms second : dump check data failed for data! " ) + + + os.system("rm -rf ./taosdumptest/") + os.system("rm -rf ./dump_result.txt") + os.system("rm -rf *.py.sql") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_data2_update0.py b/tests/pytest/update/merge_commit_data2_update0.py index def50e04661b1752668202359eec7dd89df9b6f0..7e3c65a0a2f2e3c0b01977b0b28cb0ec8a2530ea 100644 --- a/tests/pytest/update/merge_commit_data2_update0.py +++ b/tests/pytest/update/merge_commit_data2_update0.py @@ -27,7 +27,7 @@ class TDTestCase: def restart_taosd(self,db): tdDnodes.stop(1) - tdDnodes.startWithoutSleep(1) + tdDnodes.start(1) tdSql.execute("use %s;" % db) def date_to_timestamp_microseconds(self, date): diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 2fc1ac8515e47f9354483ebb590897eea96dcc57..fd3926a6f1bc79fee81c7d438dceb8eedcb7803d 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -34,7 +34,7 @@ class TDCases: self.clusterCases = [] def __dynamicLoadModule(self, fileName): - moduleName = fileName.replace(".py", "").replace("/", ".") + moduleName = fileName.replace(".py", "").replace(os.sep, ".") return importlib.import_module(moduleName, package='..') def logSql(self, logSql): @@ -80,7 +80,7 @@ class TDCases: runNum += 1 continue - def runAllWindows(self, conn): + def runAllWindows(self, conn, fileName): # TODO: load all Windows cases here runNum = 0 for tmp in self.windowsCases: @@ -101,12 +101,17 @@ class TDCases: for tmp in self.windowsCases: if tmp.name.find(fileName) != -1: case = testModule.TDTestCase() - case.init(conn) - case.run() + case.init(conn, self._logSql) + try: + case.run() + except Exception as e: + tdLog.notice(repr(e)) + tdLog.exit("%s failed" % (fileName)) case.stop() runNum += 1 continue tdLog.notice("total %d Windows case(s) executed" % (runNum)) + def runAllCluster(self): # TODO: load all cluster case module here diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index cd720ab62d4d71661a214ef24df8df81164fd71b..ff3c271cd8ab1ea2480f3d122513badab09016fc 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -14,15 +14,18 @@ import sys import os import os.path +import platform +import pathlib +import shutil import subprocess from time import sleep from util.log import * class TDSimClient: - def __init__(self): + def __init__(self, path): self.testCluster = False - + self.path = path self.cfgDict = { "numOfLogLines": "100000000", "numOfThreadsPerCore": "2.0", @@ -41,10 +44,7 @@ class TDSimClient: "jnidebugFlag": "135", "qdebugFlag": "135", "telemetryReporting": "0", - } - def init(self, path): - self.__init__() - self.path = path + } def getLogDir(self): self.logDir = "%s/sim/psim/log" % (self.path) @@ -61,35 +61,48 @@ class TDSimClient: self.cfgDict.update({option: value}) def cfg(self, option, value): - cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) - + def os_string(self,path): + os_path = path.replace("/",os.sep) + return os_path def deploy(self): - self.logDir = "%s/sim/psim/log" % (self.path) - self.cfgDir = "%s/sim/psim/cfg" % (self.path) - self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - + self.logDir = self.os_string("%s/sim/psim/log" % (self.path)) + self.cfgDir = self.os_string("%s/sim/psim/cfg" % (self.path)) + self.cfgPath = self.os_string("%s/sim/psim/cfg/taos.cfg" % (self.path)) + + # cmd = "rm -rf " + self.logDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + if os.path.exists(self.logDir): + try: + shutil.rmtree(self.logDir) + except: + tdLog.exit("del %s failed"%self.logDir) + # cmd = "mkdir -p " + self.logDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + os.makedirs(self.logDir) + # cmd = "rm -rf " + self.cfgDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + if os.path.exists(self.cfgDir): + try: + shutil.rmtree(self.cfgDir) + except: + tdLog.exit("del %s failed"%self.cfgDir) + # cmd = "mkdir -p " + self.cfgDir + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + os.makedirs(self.cfgDir) + # cmd = "touch " + self.cfgPath + # if os.system(cmd) != 0: + # tdLog.exit(cmd) + try: + pathlib.Path(self.cfgPath).touch() + except: + tdLog.exit("create %s failed"%self.cfgPath) if self.testCluster: self.cfg("masterIp", "192.168.0.1") self.cfg("secondIp", "192.168.0.2") @@ -246,7 +259,7 @@ class TDDnode: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if (("taosd") in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root)-len("/build/bin")] @@ -404,7 +417,7 @@ class TDDnode: tdLog.exit(cmd) def cfg(self, option, value): - cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: tdLog.exit(cmd) @@ -480,8 +493,7 @@ class TDDnodes: for i in range(len(self.dnodes)): self.dnodes[i].init(self.path) - self.sim = TDSimClient() - self.sim.init(self.path) + self.sim = TDSimClient(self.path) def setTestCluster(self, value): self.testCluster = value