diff --git a/cmake/define.inc b/cmake/define.inc index 598957baa808d27fd61ae955950af3dc99815ff4..c4f9396cf1ccac443505d957eb53b1d1ea567745 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -41,6 +41,10 @@ IF (TD_POWER) ADD_DEFINITIONS(-D_TD_POWER_) ENDIF () +IF (TD_TQ) + ADD_DEFINITIONS(-D_TD_TQ_) +ENDIF () + IF (TD_MEM_CHECK) ADD_DEFINITIONS(-DTAOS_MEM_CHECK) ENDIF () diff --git a/cmake/input.inc b/cmake/input.inc index c073bbbc037d105dbacea1e14981537cd31b7202..04160b47756b420247bfa7477e2fb67d0346ff3f 100755 --- a/cmake/input.inc +++ b/cmake/input.inc @@ -46,6 +46,9 @@ ENDIF () IF (${DBNAME} MATCHES "power") SET(TD_POWER TRUE) MESSAGE(STATUS "power is true") +ELSEIF (${DBNAME} MATCHES "tq") + SET(TD_TQ TRUE) + MESSAGE(STATUS "tq is true") ENDIF () IF (${DLLTYPE} MATCHES "go") diff --git a/cmake/version.inc b/cmake/version.inc index 34e8f1f83a39327aa5f579c3b9dad0aca00acced..7338b4593631550861774dbc12633a260830c3ea 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -10,7 +10,7 @@ ENDIF () IF (DEFINED VERCOMPATIBLE) SET(TD_VER_COMPATIBLE ${VERCOMPATIBLE}) ELSE () - SET(TD_VER_COMPATIBLE "2.0.0.0") + SET(TD_VER_COMPATIBLE "1.0.0.0") ENDIF () find_program(HAVE_GIT NAMES git) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 764560dbebd2f716a135895b3f1a45ebbb3cd32e..34b9aba12be36a504d380050f41e7a063aee732c 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -218,7 +218,7 @@ TDengine 缺省的时间戳是毫秒精度，但通过在 CREATE DATABASE 时传 说明：可在like中使用通配符进行名称的匹配，这一通配符字符串最长不能超过24字节。 - 通配符匹配：1）’%’ (百分号)匹配0到任意个字符；2）’\_’下划线匹配一个字符。 + 通配符匹配：1）'%'（百分号）匹配0到任意个字符；2）'\_'下划线匹配单个任意字符。（如果希望匹配表名中带有的下划线，那么这里可以用反斜线进行转义，也就是说 '\\\_' 会被用于匹配表名中原始带有的下划线符号） - **显示一个数据表的创建语句** @@ -696,9 +696,9 @@ Query OK, 1 row(s) in set (0.001091s) * 暂不支持含列名的四则运算表达式作为 SQL 函数的应用对象（例如，不支持 `select min(2*a) from t;`，但可以写 `select 2*min(a) from t;`）。 - WHERE 语句可以使用各种逻辑判断来过滤数字值，或使用通配符来过滤字符串。 - 输出结果缺省按首列时间戳升序排序，但可以指定按降序排序( _c0 指首列时间戳)。使用 ORDER BY 对其他字段进行排序为非法操作。 -- 参数 LIMIT 控制输出条数，OFFSET 指定从第几条开始输出。LIMIT/OFFSET 对结果集的执行顺序在 ORDER BY 之后。 +- 参数 LIMIT 控制输出条数，OFFSET 指定从第几条开始输出。LIMIT/OFFSET 对结果集的执行顺序在 ORDER BY 之后。且 `LIMIT 5 OFFSET 2` 可以简写为 `LIMIT 2, 5`。 * 在有 GROUP BY 子句的情况下，LIMIT 参数控制的是每个分组中至多允许输出的条数。 -- 参数 SLIMIT 控制由 GROUP BY 指令划分的分组中，至多允许输出几个分组的数据。 +- 参数 SLIMIT 控制由 GROUP BY 指令划分的分组中，至多允许输出几个分组的数据。且 `SLIMIT 5 OFFSET 2` 可以简写为 `SLIMIT 2, 5`。 - 通过 “>>” 输出结果可以导出到指定文件。 ### 支持的条件过滤操作 diff --git a/packaging/release.sh b/packaging/release.sh index 1d81f818a9b386e5520c5fa6b7499a7990cbe23a..34fda44243684cb0ae86071df1bff7a8b919075f 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -11,7 +11,7 @@ set -e # -V [stable | beta] # -l [full | lite] # -s [static | dynamic] -# -d [taos | power] +# -d [taos | power | tq ] # -n [2.0.0.3] # -m [2.0.0.0] @@ -22,10 +22,10 @@ cpuType=x64 # [aarch32 | aarch64 | x64 | x86 | mips64 ...] osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] pagMode=full # [full | lite] soMode=dynamic # [static | dynamic] +dbName=taos # [taos | power | tq] allocator=glibc # [glibc | jemalloc] -dbName=taos # [taos | power] verNumber="" -verNumberComp="2.0.0.0" +verNumberComp="1.0.0.0" while getopts "hv:V:c:o:l:s:d:a:n:m:" arg do @@ -78,7 +78,7 @@ do echo " -l [full | lite] " echo " -a [glibc | jemalloc] " echo " -s [static | dynamic] " - echo " -d [taos | power] " + echo " -d [taos | power | tq ] " echo " -n [version number] " echo " -m [compatible version number] " exit 0 @@ -204,7 +204,7 @@ else exit 1 fi -make +make -j8 cd ${curr_dir} @@ -249,6 +249,10 @@ if [ "$osType" != "Darwin" ]; then ${csudo} ./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${csudo} ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${csudo} ./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + elif [[ "$dbName" == "tq" ]]; then + ${csudo} ./makepkg_tq.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} + ${csudo} ./makeclient_tq.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} + ${csudo} ./makearbi_tq.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} else ${csudo} ./makepkg_power.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ${csudo} ./makeclient_power.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 325ac810539385f8a43fb655b76a8e211d65c872..3911b1bc727fcb317f3841e00bc28f910f126038 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -896,6 +896,7 @@ function install_TDengine() { install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh index f47c3672cb0f806ee429209462c2a001be6090de..cdfeef0543313dd94fe3c9068ff86a4f19729884 100755 --- a/packaging/tools/install_arbi.sh +++ b/packaging/tools/install_arbi.sh @@ -38,11 +38,11 @@ initd_mod=0 service_mod=2 if pidof systemd &> /dev/null; then service_mod=0 -elif $(which service &> /dev/null); then +elif $(which service &> /dev/null); then service_mod=1 - service_config_dir="/etc/init.d" + service_config_dir="/etc/init.d" if $(which chkconfig &> /dev/null); then - initd_mod=1 + initd_mod=1 elif $(which insserv &> /dev/null); then initd_mod=2 elif $(which update-rc.d &> /dev/null); then @@ -50,7 +50,7 @@ elif $(which service &> /dev/null); then else service_mod=2 fi -else +else service_mod=2 fi @@ -82,7 +82,7 @@ elif echo $osinfo | grep -qwi "fedora" ; then os_type=2 else echo " osinfo: ${osinfo}" - echo " This is an officially unverified linux system," + echo " This is an officially unverified linux system," echo " if there are any problems with the installation and operation, " echo " please feel free to contact taosdata.com for support." os_type=1 @@ -99,7 +99,7 @@ function install_main_path() { #create install main dir and all sub dir ${csudo} rm -rf ${install_main_dir} || : ${csudo} mkdir -p ${install_main_dir} - ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/bin #${csudo} mkdir -p ${install_main_dir}/include ${csudo} mkdir -p ${install_main_dir}/init.d } @@ -117,21 +117,67 @@ function install_bin() { function install_header() { ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : - ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo} /usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo} /usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo} ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo} /usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + fi +} + function clean_service_on_sysvinit() { #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" - #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : - + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + if pidof tarbitrator &> /dev/null; then ${csudo} service tarbitratord stop || : fi if ((${initd_mod}==1)); then - if [ -e ${service_config_dir}/tarbitratord ]; then + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} chkconfig --del tarbitratord || : fi elif ((${initd_mod}==2)); then @@ -142,10 +188,10 @@ function clean_service_on_sysvinit() { if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} update-rc.d -f tarbitratord remove || : fi - fi + fi ${csudo} rm -f ${service_config_dir}/tarbitratord || : - + if $(which init &> /dev/null); then ${csudo} init q || : fi @@ -164,10 +210,10 @@ function install_service_on_sysvinit() { ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord fi - + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" - + if ((${initd_mod}==1)); then ${csudo} chkconfig --add tarbitratord || : ${csudo} chkconfig --level 2345 tarbitratord on || : @@ -245,12 +291,13 @@ function update_TDengine() { fi sleep 1 fi - + install_main_path #install_header install_bin install_service - + install_jemalloc + echo #echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then @@ -259,7 +306,7 @@ function update_TDengine() { echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" else echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" - fi + fi echo echo -e "\033[44;32;1mTDengine's arbitrator is updated successfully!${NC}" } @@ -267,11 +314,13 @@ function update_TDengine() { function install_TDengine() { # Start to install echo -e "${GREEN}Start to install TDengine's arbitrator ...${NC}" - - install_main_path + + install_main_path #install_header install_bin install_service + install_jemalloc + echo #echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then diff --git a/packaging/tools/install_arbi_power.sh b/packaging/tools/install_arbi_power.sh index 3f271751511a124994a0f1833b59ff406dd9b227..161b916b4ca103f9c2ce4a8d2e90db91cfa3487f 100755 --- a/packaging/tools/install_arbi_power.sh +++ b/packaging/tools/install_arbi_power.sh @@ -38,11 +38,11 @@ initd_mod=0 service_mod=2 if pidof systemd &> /dev/null; then service_mod=0 -elif $(which service &> /dev/null); then +elif $(which service &> /dev/null); then service_mod=1 - service_config_dir="/etc/init.d" + service_config_dir="/etc/init.d" if $(which chkconfig &> /dev/null); then - initd_mod=1 + initd_mod=1 elif $(which insserv &> /dev/null); then initd_mod=2 elif $(which update-rc.d &> /dev/null); then @@ -50,7 +50,7 @@ elif $(which service &> /dev/null); then else service_mod=2 fi -else +else service_mod=2 fi @@ -82,7 +82,7 @@ elif echo $osinfo | grep -qwi "fedora" ; then os_type=2 else echo " osinfo: ${osinfo}" - echo " This is an officially unverified linux system," + echo " This is an officially unverified linux system," echo " if there are any problems with the installation and operation, " echo " please feel free to contact taosdata.com for support." os_type=1 @@ -99,7 +99,7 @@ function install_main_path() { #create install main dir and all sub dir ${csudo} rm -rf ${install_main_dir} || : ${csudo} mkdir -p ${install_main_dir} - ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/bin #${csudo} mkdir -p ${install_main_dir}/include ${csudo} mkdir -p ${install_main_dir}/init.d } @@ -115,23 +115,69 @@ function install_bin() { [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : } +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo} /usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo} /usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo} ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo} /usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + fi +} + function install_header() { ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : - ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } function clean_service_on_sysvinit() { #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" - #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : - + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + if pidof tarbitrator &> /dev/null; then ${csudo} service tarbitratord stop || : fi if ((${initd_mod}==1)); then - if [ -e ${service_config_dir}/tarbitratord ]; then + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} chkconfig --del tarbitratord || : fi elif ((${initd_mod}==2)); then @@ -142,10 +188,10 @@ function clean_service_on_sysvinit() { if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} update-rc.d -f tarbitratord remove || : fi - fi + fi ${csudo} rm -f ${service_config_dir}/tarbitratord || : - + if $(which init &> /dev/null); then ${csudo} init q || : fi @@ -164,10 +210,10 @@ function install_service_on_sysvinit() { ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord fi - + #restart_config_str="power:2345:respawn:${service_config_dir}/powerd start" #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" - + if ((${initd_mod}==1)); then ${csudo} chkconfig --add tarbitratord || : ${csudo} chkconfig --level 2345 tarbitratord on || : @@ -245,12 +291,13 @@ function update_PowerDB() { fi sleep 1 fi - + install_main_path #install_header install_bin install_service - + install_jemalloc + echo #echo -e "${GREEN_DARK}To configure PowerDB ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then @@ -259,7 +306,7 @@ function update_PowerDB() { echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" else echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" - fi + fi echo echo -e "\033[44;32;1mPowerDB's arbitrator is updated successfully!${NC}" } @@ -267,11 +314,13 @@ function update_PowerDB() { function install_PowerDB() { # Start to install echo -e "${GREEN}Start to install PowerDB's arbitrator ...${NC}" - - install_main_path + + install_main_path #install_header install_bin install_service + install_jemalloc + echo #echo -e "${GREEN_DARK}To configure PowerDB ${NC}: edit /etc/taos/taos.cfg" if ((${service_mod}==0)); then diff --git a/packaging/tools/install_arbi_tq.sh b/packaging/tools/install_arbi_tq.sh new file mode 100755 index 0000000000000000000000000000000000000000..bd852dd0ad2c9114f2424193adccf56b0cb40412 --- /dev/null +++ b/packaging/tools/install_arbi_tq.sh @@ -0,0 +1,298 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) + +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/tarbitrator" + +# old bin dir +bin_dir="/usr/local/tarbitrator/bin" + +service_config_dir="/etc/systemd/system" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu" ; then +# echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian" ; then +# echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin" ; then +# echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos" ; then +# echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora" ; then +# echo "This is fedora system" + os_type=2 +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact taosdata.com for support." + os_type=1 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/bin + #${csudo} mkdir -p ${install_main_dir}/include + ${csudo} mkdir -p ${install_main_dir}/init.d +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/rmtarbitrator || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/remove_arbi_tq.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_arbi_tq.sh ${bin_link_dir}/rmtarbitrator || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function clean_service_on_sysvinit() { + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + + if pidof tarbitrator &> /dev/null; then + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + # Install tqd service + + if ((${os_type}==1)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type}==2)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + fi + + #restart_config_str="tq:2345:respawn:${service_config_dir}/tqd start" + #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" + + if ((${initd_mod}==1)); then + ${csudo} chkconfig --add tarbitratord || : + ${csudo} chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod}==2)); then + ${csudo} insserv tarbitratord || : + ${csudo} insserv -d tarbitratord || : + elif ((${initd_mod}==3)); then + ${csudo} update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +# tq:2345:respawn:/etc/init.d/tarbitratord start + +function install_service_on_systemd() { + clean_service_on_systemd + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=TQ arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + ${csudo} systemctl enable tarbitratord +} + +function install_service() { + if ((${service_mod}==0)); then + install_service_on_systemd + elif ((${service_mod}==1)); then + install_service_on_sysvinit + else + # must manual stop taosd + kill_tarbitrator + fi +} + +function update_tq() { + # Start to update + echo -e "${GREEN}Start to update TQ's arbitrator ...${NC}" + # Stop the service if running + if pidof tarbitrator &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop tarbitratord || : + elif ((${service_mod}==1)); then + ${csudo} service tarbitratord stop || : + else + kill_tarbitrator + fi + sleep 1 + fi + + install_main_path + #install_header + install_bin + install_service + + echo + #echo -e "${GREEN_DARK}To configure TQ ${NC}: edit /etc/taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" + fi + echo + echo -e "\033[44;32;1mTQ's arbitrator is updated successfully!${NC}" +} + +function install_tq() { + # Start to install + echo -e "${GREEN}Start to install TQ's arbitrator ...${NC}" + + install_main_path + #install_header + install_bin + install_service + echo + #echo -e "${GREEN_DARK}To configure TQ ${NC}: edit /etc/taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: tarbitrator${NC}" + fi + + echo -e "\033[44;32;1mTQ's arbitrator is installed successfully!${NC}" + echo +} + + +## ==============================Main program starts from here============================ +# Install server and client +if [ -x ${bin_dir}/tarbitrator ]; then + update_flag=1 + update_tq +else + install_tq +fi + diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 0a0a6633e376d084532abb5f490917abd1a173f2..2969a3cc98fff7e9c15c9338db29dc180f866804 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -119,16 +119,16 @@ function install_lib() { if [ "$osType" != "Darwin" ]; then ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - + if [ -d "${lib64_link_dir}" ]; then - ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : - ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : - fi + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi else ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib ${csudo} ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib fi - + ${csudo} ldconfig } @@ -139,6 +139,52 @@ function install_header() { ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo} /usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo} /usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo} ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo} /usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + fi +} + function install_config() { #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : @@ -194,6 +240,7 @@ function update_TDengine() { install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi @@ -217,10 +264,11 @@ function install_TDengine() { echo -e "${GREEN}Start to install TDengine client...${NC}" - install_main_path + install_main_path install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi diff --git a/packaging/tools/install_client_power.sh b/packaging/tools/install_client_power.sh index 8d7463366ff46bcae2822ee3e76dbc9b588f2a89..d56527ecdae320d5e58b54620f7ff2d021e02d11 100755 --- a/packaging/tools/install_client_power.sh +++ b/packaging/tools/install_client_power.sh @@ -119,16 +119,16 @@ function install_lib() { if [ "$osType" != "Darwin" ]; then ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - + if [ -d "${lib64_link_dir}" ]; then - ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : - ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : - fi + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi else ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib ${csudo} ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib fi - + ${csudo} ldconfig } @@ -139,6 +139,52 @@ function install_header() { ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo} /usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo} /usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo} ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo} /usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + fi +} + function install_config() { #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : @@ -194,6 +240,7 @@ function update_PowerDB() { install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi @@ -217,10 +264,11 @@ function install_PowerDB() { echo -e "${GREEN}Start to install PowerDB client...${NC}" - install_main_path + install_main_path install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi diff --git a/packaging/tools/install_client_tq.sh b/packaging/tools/install_client_tq.sh new file mode 100755 index 0000000000000000000000000000000000000000..2537442ee264e9aeb4eb6b3d25a17faf60f4df9a --- /dev/null +++ b/packaging/tools/install_client_tq.sh @@ -0,0 +1,251 @@ +#!/bin/bash +# +# This file is used to install TQ client on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- + +osType=Linux +pagMode=full + +if [ "$osType" != "Darwin" ]; then + script_dir=$(dirname $(readlink -f "$0")) + # Dynamic directory + data_dir="/var/lib/tq" + log_dir="/var/log/tq" +else + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + data_dir="/var/lib/tq" + log_dir="~/TQLog" +fi + +log_link_dir="/usr/local/tq/log" + +cfg_install_dir="/etc/tq" + +if [ "$osType" != "Darwin" ]; then + bin_link_dir="/usr/bin" + lib_link_dir="/usr/lib" + lib64_link_dir="/usr/lib64" + inc_link_dir="/usr/include" +else + bin_link_dir="/usr/local/bin" + lib_link_dir="/usr/local/lib" + inc_link_dir="/usr/local/include" +fi + +#install main path +install_main_dir="/usr/local/tq" + +# old bin dir +bin_dir="/usr/local/tq/bin" + +# v1.5 jar dir +#v15_java_app_dir="/usr/local/lib/tq" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +function kill_client() { + pid=$(ps -ef | grep "tq" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/cfg + ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/connector + ${csudo} mkdir -p ${install_main_dir}/driver + ${csudo} mkdir -p ${install_main_dir}/examples + ${csudo} mkdir -p ${install_main_dir}/include +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/tq || : + if [ "$osType" != "Darwin" ]; then + ${csudo} rm -f ${bin_link_dir}/tqdemo || : + ${csudo} rm -f ${bin_link_dir}/tqdump || : + fi + ${csudo} rm -f ${bin_link_dir}/rmtq || : + ${csudo} rm -f ${bin_link_dir}/set_core || : + + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/tq ] && ${csudo} ln -s ${install_main_dir}/bin/tq ${bin_link_dir}/tq || : + if [ "$osType" != "Darwin" ]; then + [ -x ${install_main_dir}/bin/tqdemo ] && ${csudo} ln -s ${install_main_dir}/bin/tqdemo ${bin_link_dir}/tqdemo || : + [ -x ${install_main_dir}/bin/tqdump ] && ${csudo} ln -s ${install_main_dir}/bin/tqdump ${bin_link_dir}/tqdump || : + fi + [ -x ${install_main_dir}/bin/remove_client_tq.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client_tq.sh ${bin_link_dir}/rmtq || : + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : +} + +function clean_lib() { + sudo rm -f /usr/lib/libtaos.* || : + sudo rm -rf ${lib_dir} || : +} + +function install_lib() { + # Remove links + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo} rm -rf ${v15_java_app_dir} || : + + ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + + if [ "$osType" != "Darwin" ]; then + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + + if [ -d "${lib64_link_dir}" ]; then + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi + else + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib + ${csudo} ln -s ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib + fi + + ${csudo} ldconfig +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function install_config() { + #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : + + if [ ! -f ${cfg_install_dir}/taos.cfg ]; then + ${csudo} mkdir -p ${cfg_install_dir} + [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} + ${csudo} chmod 644 ${cfg_install_dir}/* + fi + + ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org + ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg +} + + +function install_log() { + ${csudo} rm -rf ${log_dir} || : + + if [ "$osType" != "Darwin" ]; then + ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} + else + mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} + fi + ${csudo} ln -s ${log_dir} ${install_main_dir}/log +} + +function install_connector() { + ${csudo} cp -rf ${script_dir}/connector/* ${install_main_dir}/connector +} + +function install_examples() { + if [ -d ${script_dir}/examples ]; then + ${csudo} cp -rf ${script_dir}/examples/* ${install_main_dir}/examples + fi +} + +function update_tq() { + # Start to update + if [ ! -e tq.tar.gz ]; then + echo "File tq.tar.gz does not exist" + exit 1 + fi + tar -zxf tq.tar.gz + + echo -e "${GREEN}Start to update TQ client...${NC}" + # Stop the client shell if running + if pidof tq &> /dev/null; then + kill_client + sleep 1 + fi + + install_main_path + + install_log + install_header + install_lib + if [ "$pagMode" != "lite" ]; then + install_connector + fi + install_examples + install_bin + install_config + + echo + echo -e "\033[44;32;1mTQ client is updated successfully!${NC}" + + rm -rf $(tar -tf tq.tar.gz) +} + +function install_tq() { + # Start to install + if [ ! -e tq.tar.gz ]; then + echo "File tq.tar.gz does not exist" + exit 1 + fi + tar -zxf tq.tar.gz + + echo -e "${GREEN}Start to install TQ client...${NC}" + + install_main_path + install_log + install_header + install_lib + if [ "$pagMode" != "lite" ]; then + install_connector + fi + install_examples + install_bin + install_config + + echo + echo -e "\033[44;32;1mTQ client is installed successfully!${NC}" + + rm -rf $(tar -tf tq.tar.gz) +} + + +## ==============================Main program starts from here============================ +# Install or updata client and client +# if server is already install, don't install client + if [ -e ${bin_dir}/tqd ]; then + echo -e "\033[44;32;1mThere are already installed TQ server, so don't need install client!${NC}" + exit 0 + fi + + if [ -x ${bin_dir}/tq ]; then + update_flag=1 + update_tq + else + install_tq + fi diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index 9f28435cb5f1cc43ae3bf1433074920889ccee3b..b1c4167e945c6553f169ff34f82af9c2f7fa6f3c 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -58,11 +58,11 @@ initd_mod=0 service_mod=2 if pidof systemd &> /dev/null; then service_mod=0 -elif $(which service &> /dev/null); then +elif $(which service &> /dev/null); then service_mod=1 - service_config_dir="/etc/init.d" + service_config_dir="/etc/init.d" if $(which chkconfig &> /dev/null); then - initd_mod=1 + initd_mod=1 elif $(which insserv &> /dev/null); then initd_mod=2 elif $(which update-rc.d &> /dev/null); then @@ -70,7 +70,7 @@ elif $(which service &> /dev/null); then else service_mod=2 fi -else +else service_mod=2 fi @@ -102,7 +102,7 @@ elif echo $osinfo | grep -qwi "fedora" ; then os_type=2 else echo " osinfo: ${osinfo}" - echo " This is an officially unverified linux system," + echo " This is an officially unverified linux system," echo " if there are any problems with the installation and operation, " echo " please feel free to contact taosdata.com for support." os_type=1 @@ -137,7 +137,7 @@ do echo "Usage: `basename $0` -v [server | client] -e [yes | no]" exit 0 ;; - ?) #unknow option + ?) #unknow option echo "unkonw argument" exit 1 ;; @@ -156,9 +156,9 @@ function kill_process() { function install_main_path() { #create install main dir and all sub dir ${csudo} rm -rf ${install_main_dir} || : - ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir} ${csudo} mkdir -p ${install_main_dir}/cfg - ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/bin ${csudo} mkdir -p ${install_main_dir}/connector ${csudo} mkdir -p ${install_main_dir}/driver ${csudo} mkdir -p ${install_main_dir}/examples @@ -200,29 +200,75 @@ function install_lib() { ${csudo} rm -f ${lib_link_dir}/libtaos.* || : ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : #${csudo} rm -rf ${v15_java_app_dir} || : - ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* - + ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : fi - - #if [ "$verMode" == "cluster" ]; then + + #if [ "$verMode" == "cluster" ]; then # # Compatible with version 1.5 # ${csudo} mkdir -p ${v15_java_app_dir} # ${csudo} ln -s ${install_main_dir}/connector/taos-jdbcdriver-1.0.2-dist.jar ${v15_java_app_dir}/JDBCDriver-1.0.2-dist.jar # ${csudo} chmod 777 ${v15_java_app_dir} || : #fi - + ${csudo} ldconfig } +function install_jemalloc() { + jemalloc_dir=${script_dir}/jemalloc + + if [ -d ${jemalloc_dir} ]; then + ${csudo} /usr/bin/install -c -d /usr/local/bin + + if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin + fi + if [ -f ${jemalloc_dir}/bin/jeprof ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin + fi + if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then + ${csudo} /usr/bin/install -c -d /usr/local/include/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib + ${csudo} ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo} /usr/bin/install -c -d /usr/local/lib + if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib + fi + if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then + ${csudo} /usr/bin/install -c -d /usr/local/lib/pkgconfig + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig + fi + fi + if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/doc/jemalloc + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc + fi + if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then + ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3 + ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 + fi + fi +} + function install_header() { ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : - ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h } @@ -239,13 +285,13 @@ function add_newHostname_to_hosts() { if [[ "$s" == "$localIp" ]]; then return fi - done + done ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: } function set_hostname() { echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" - read newHostname + read newHostname while true; do if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then break @@ -259,25 +305,25 @@ function set_hostname() { if [[ $retval != 0 ]]; then echo echo "set hostname fail!" - return + return fi #echo -e -n "$(hostnamectl status --static)" #echo -e -n "$(hostnamectl status --transient)" #echo -e -n "$(hostnamectl status --pretty)" - + #ubuntu/centos /etc/hostname if [[ -e /etc/hostname ]]; then ${csudo} echo $newHostname > /etc/hostname ||: fi - + #debian: #HOSTNAME=yourname if [[ -e /etc/sysconfig/network ]]; then ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: fi ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg - serverFqdn=$newHostname - + serverFqdn=$newHostname + if [[ -e /etc/hosts ]]; then add_newHostname_to_hosts $newHostname fi @@ -295,7 +341,7 @@ function is_correct_ipaddr() { return 0 fi done - + return 1 } @@ -309,13 +355,13 @@ function set_ipAsFqdn() { echo echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" localFqdn="127.0.0.1" - # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg serverFqdn=$localFqdn echo return - fi - + fi + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" echo echo -e -n "${GREEN}$iplist${NC}" @@ -324,15 +370,15 @@ function set_ipAsFqdn() { echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" read localFqdn while true; do - if [ ! -z "$localFqdn" ]; then + if [ ! -z "$localFqdn" ]; then # Check if correct ip address is_correct_ipaddr $localFqdn retval=`echo $?` if [[ $retval != 0 ]]; then read -p "Please choose an IP from local IP list:" localFqdn else - # Write the local FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg serverFqdn=$localFqdn break fi @@ -347,59 +393,59 @@ function local_fqdn_check() { echo echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" echo - if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then - echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" - echo - - while true - do - read -r -p "Set hostname now? [Y/n] " input - if [ ! -n "$input" ]; then - set_hostname - break - else - case $input in - [yY][eE][sS]|[yY]) - set_hostname - break - ;; - - [nN][oO]|[nN]) - set_ipAsFqdn - break - ;; - - *) - echo "Invalid input..." - ;; - esac - fi - done + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done fi } function install_config() { #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : - + if [ ! -f ${cfg_install_dir}/taos.cfg ]; then ${csudo} mkdir -p ${cfg_install_dir} [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* - fi - + fi + ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg [ ! -z $1 ] && return 0 || : # only install client - + if ((${update_flag}==1)); then return 0 fi - + if [ "$interactiveFqdn" == "no" ]; then return 0 fi - + local_fqdn_check #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" @@ -417,8 +463,8 @@ function install_config() { if [ ! -z "$firstEp" ]; then # check the format of the firstEp #if [[ $firstEp == $FQDN_PATTERN ]]; then - # Write the first FQDN to configuration file - ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg + # Write the first FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg break #else # read -p "Please enter the correct FQDN:port: " firstEp @@ -426,21 +472,21 @@ function install_config() { else break fi - done + done } function install_log() { ${csudo} rm -rf ${log_dir} || : ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} - + ${csudo} ln -s ${log_dir} ${install_main_dir}/log } function install_data() { ${csudo} mkdir -p ${data_dir} - - ${csudo} ln -s ${data_dir} ${install_main_dir}/data + + ${csudo} ln -s ${data_dir} ${install_main_dir}/data } function install_connector() { @@ -455,26 +501,26 @@ function install_examples() { function clean_service_on_sysvinit() { #restart_config_str="power:2345:respawn:${service_config_dir}/powerd start" - #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : - + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + if pidof powerd &> /dev/null; then ${csudo} service powerd stop || : fi - + if pidof tarbitrator &> /dev/null; then ${csudo} service tarbitratord stop || : fi if ((${initd_mod}==1)); then - if [ -e ${service_config_dir}/powerd ]; then + if [ -e ${service_config_dir}/powerd ]; then ${csudo} chkconfig --del powerd || : fi - if [ -e ${service_config_dir}/tarbitratord ]; then + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} chkconfig --del tarbitratord || : fi elif ((${initd_mod}==2)); then - if [ -e ${service_config_dir}/powerd ]; then + if [ -e ${service_config_dir}/powerd ]; then ${csudo} insserv -r powerd || : fi if [ -e ${service_config_dir}/tarbitratord ]; then @@ -488,10 +534,10 @@ function clean_service_on_sysvinit() { ${csudo} update-rc.d -f tarbitratord remove || : fi fi - + ${csudo} rm -f ${service_config_dir}/powerd || : ${csudo} rm -f ${service_config_dir}/tarbitratord || : - + if $(which init &> /dev/null); then ${csudo} init q || : fi @@ -514,10 +560,10 @@ function install_service_on_sysvinit() { ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord fi - + #restart_config_str="power:2345:respawn:${service_config_dir}/powerd start" #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" - + if ((${initd_mod}==1)); then ${csudo} chkconfig --add powerd || : ${csudo} chkconfig --level 2345 powerd on || : @@ -542,7 +588,7 @@ function clean_service_on_systemd() { fi ${csudo} systemctl disable powerd &> /dev/null || echo &> /dev/null ${csudo} rm -f ${powerd_service_config} - + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" if systemctl is-active --quiet tarbitratord; then echo "tarbitrator is running, stopping it..." @@ -550,7 +596,7 @@ function clean_service_on_systemd() { fi ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null ${csudo} rm -f ${tarbitratord_service_config} - + if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/nginxd.service" if systemctl is-active --quiet nginxd; then @@ -558,8 +604,8 @@ function clean_service_on_systemd() { ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${nginx_service_config} - fi + ${csudo} rm -f ${nginx_service_config} + fi } # power:2345:respawn:/etc/init.d/powerd start @@ -590,7 +636,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Install]' >> ${powerd_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${powerd_service_config}" ${csudo} systemctl enable powerd - + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" @@ -612,9 +658,9 @@ function install_service_on_systemd() { ${csudo} bash -c "echo >> ${tarbitratord_service_config}" ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" - #${csudo} systemctl enable tarbitratord - - if [ "$verMode" == "cluster" ]; then + #${csudo} systemctl enable tarbitratord + + if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/nginxd.service" ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" ${csudo} bash -c "echo 'Description=Nginx For PowrDB Service' >> ${nginx_service_config}" @@ -643,7 +689,7 @@ function install_service_on_systemd() { ${csudo} systemctl enable nginxd fi ${csudo} systemctl start nginxd - fi + fi } function install_service() { @@ -725,8 +771,8 @@ function update_PowerDB() { kill_process powerd fi sleep 1 - fi - if [ "$verMode" == "cluster" ]; then + fi + if [ "$verMode" == "cluster" ]; then if pidof nginx &> /dev/null; then if ((${service_mod}==0)); then ${csudo} systemctl stop nginxd || : @@ -738,12 +784,13 @@ function update_PowerDB() { sleep 1 fi fi - + install_main_path install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi @@ -751,10 +798,10 @@ function update_PowerDB() { if [ -z $1 ]; then install_bin install_service - install_config - + install_config + openresty_work=false - if [ "$verMode" == "cluster" ]; then + if [ "$verMode" == "cluster" ]; then # Check if openresty is installed # Check if nginx is installed successfully if type curl &> /dev/null; then @@ -765,7 +812,7 @@ function update_PowerDB() { echo -e "\033[44;31;5mNginx for PowerDB does not work! Please try again!\033[0m" fi fi - fi + fi #echo #echo -e "\033[44;32;1mPowerDB is updated successfully!${NC}" @@ -784,7 +831,7 @@ function update_PowerDB() { else echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power -h $serverFqdn${NC} in shell${NC}" fi - + echo echo -e "\033[44;32;1mPowerDB is updated successfully!${NC}" else @@ -807,16 +854,17 @@ function install_PowerDB() { tar -zxf power.tar.gz echo -e "${GREEN}Start to install PowerDB...${NC}" - - install_main_path - + + install_main_path + if [ -z $1 ]; then install_data - fi - - install_log + fi + + install_log install_header install_lib + install_jemalloc if [ "$pagMode" != "lite" ]; then install_connector fi @@ -839,8 +887,8 @@ function install_PowerDB() { fi fi fi - - install_config + + install_config # Ask if to start the service #echo @@ -853,35 +901,35 @@ function install_PowerDB() { echo -e "${GREEN_DARK}To start PowerDB ${NC}: ${csudo} service powerd start${NC}" else echo -e "${GREEN_DARK}To start PowerDB ${NC}: powerd${NC}" - fi + fi #if [ ${openresty_work} = 'true' ]; then # echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" #else # echo -e "${GREEN_DARK}To access PowerDB ${NC}: use ${GREEN_UNDERLINE}power${NC} in shell${NC}" #fi - + if [ ! -z "$firstEp" ]; then - tmpFqdn=${firstEp%%:*} - substr=":" - if [[ $firstEp =~ $substr ]];then - tmpPort=${firstEp#*:} - else - tmpPort="" - fi - if [[ "$tmpPort" != "" ]];then - echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" - else - echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" - fi - echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" - echo + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo elif [ ! -z "$serverFqdn" ]; then - echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $serverFqdn${GREEN_DARK} to login into PowerDB server${NC}" - echo + echo -e "${GREEN_DARK}To access PowerDB ${NC}: power -h $serverFqdn${GREEN_DARK} to login into PowerDB server${NC}" + echo fi echo -e "\033[44;32;1mPowerDB is installed successfully!${NC}" - echo + echo else # Only install client install_bin install_config @@ -913,6 +961,6 @@ elif [ "$verType" == "client" ]; then else install_PowerDB client fi -else - echo "please input correct verType" +else + echo "please input correct verType" fi diff --git a/packaging/tools/install_tq.sh b/packaging/tools/install_tq.sh new file mode 100755 index 0000000000000000000000000000000000000000..6579592fdf0544cc71b7a25871995eab59873bce --- /dev/null +++ b/packaging/tools/install_tq.sh @@ -0,0 +1,918 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +verMode=edge +pagMode=full + +iplist="" +serverFqdn="" +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) +# Dynamic directory +data_dir="/var/lib/tq" +log_dir="/var/log/tq" + +data_link_dir="/usr/local/tq/data" +log_link_dir="/usr/local/tq/log" + +cfg_install_dir="/etc/tq" + +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/tq" + +# old bin dir +bin_dir="/usr/local/tq/bin" + +# v1.5 jar dir +#v15_java_app_dir="/usr/local/lib/tq" + +service_config_dir="/etc/systemd/system" +nginx_port=6060 +nginx_dir="/usr/local/nginxd" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu" ; then +# echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian" ; then +# echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin" ; then +# echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos" ; then +# echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora" ; then +# echo "This is fedora system" + os_type=2 +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact taosdata.com for support." + os_type=1 +fi + + +# ============================= get input parameters ================================================= + +# install.sh -v [server | client] -e [yes | no] -i [systemd | service | ...] + +# set parameters by default value +interactiveFqdn=yes # [yes | no] +verType=server # [server | client] +initType=systemd # [systemd | service | ...] + +while getopts "hv:e:i:" arg +do + case $arg in + e) + #echo "interactiveFqdn=$OPTARG" + interactiveFqdn=$( echo $OPTARG ) + ;; + v) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; + i) + #echo "initType=$OPTARG" + initType=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -v [server | client] -e [yes | no]" + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +#echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" + +function kill_process() { + pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/cfg + ${csudo} mkdir -p ${install_main_dir}/bin + ${csudo} mkdir -p ${install_main_dir}/connector + ${csudo} mkdir -p ${install_main_dir}/driver + ${csudo} mkdir -p ${install_main_dir}/examples + ${csudo} mkdir -p ${install_main_dir}/include + ${csudo} mkdir -p ${install_main_dir}/init.d + if [ "$verMode" == "cluster" ]; then + ${csudo} mkdir -p ${nginx_dir} + fi +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/tq || : + ${csudo} rm -f ${bin_link_dir}/tqd || : + ${csudo} rm -f ${bin_link_dir}/tqdemo || : + ${csudo} rm -f ${bin_link_dir}/rmtq || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} rm -f ${bin_link_dir}/set_core || : + + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/tq ] && ${csudo} ln -s ${install_main_dir}/bin/tq ${bin_link_dir}/tq || : + [ -x ${install_main_dir}/bin/tqd ] && ${csudo} ln -s ${install_main_dir}/bin/tqd ${bin_link_dir}/tqd || : + [ -x ${install_main_dir}/bin/tqdemo ] && ${csudo} ln -s ${install_main_dir}/bin/tqdemo ${bin_link_dir}/tqdemo || : + [ -x ${install_main_dir}/bin/remove_tq.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_tq.sh ${bin_link_dir}/rmtq || : + [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : + + if [ "$verMode" == "cluster" ]; then + ${csudo} cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo} chmod 0555 ${nginx_dir}/* + ${csudo} mkdir -p ${nginx_dir}/logs + ${csudo} chmod 777 ${nginx_dir}/sbin/nginx + fi +} + +function install_lib() { + # Remove links + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo} rm -rf ${v15_java_app_dir} || : + ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 + ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then + ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : + ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + fi + + #if [ "$verMode" == "cluster" ]; then + # # Compatible with version 1.5 + # ${csudo} mkdir -p ${v15_java_app_dir} + # ${csudo} ln -s ${install_main_dir}/connector/taos-jdbcdriver-1.0.2-dist.jar ${v15_java_app_dir}/JDBCDriver-1.0.2-dist.jar + # ${csudo} chmod 777 ${v15_java_app_dir} || : + #fi + + ${csudo} ldconfig +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function add_newHostname_to_hosts() { + localIp="127.0.0.1" + OLD_IFS="$IFS" + IFS=" " + iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}') + arr=($iphost) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$localIp" ]]; then + return + fi + done + ${csudo} echo "127.0.0.1 $1" >> /etc/hosts ||: +} + +function set_hostname() { + echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" + read newHostname + while true; do + if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then + break + else + read -p "Please enter one hostname(must not be 'localhost'):" newHostname + fi + done + + ${csudo} hostname $newHostname ||: + retval=`echo $?` + if [[ $retval != 0 ]]; then + echo + echo "set hostname fail!" + return + fi + #echo -e -n "$(hostnamectl status --static)" + #echo -e -n "$(hostnamectl status --transient)" + #echo -e -n "$(hostnamectl status --pretty)" + + #ubuntu/centos /etc/hostname + if [[ -e /etc/hostname ]]; then + ${csudo} echo $newHostname > /etc/hostname ||: + fi + + #debian: #HOSTNAME=yourname + if [[ -e /etc/sysconfig/network ]]; then + ${csudo} sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network ||: + fi + + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/taos.cfg + serverFqdn=$newHostname + + if [[ -e /etc/hosts ]]; then + add_newHostname_to_hosts $newHostname + fi +} + +function is_correct_ipaddr() { + newIp=$1 + OLD_IFS="$IFS" + IFS=" " + arr=($iplist) + IFS="$OLD_IFS" + for s in ${arr[@]} + do + if [[ "$s" == "$newIp" ]]; then + return 0 + fi + done + + return 1 +} + +function set_ipAsFqdn() { + iplist=$(ip address |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F "/" '{print $1}') ||: + if [ -z "$iplist" ]; then + iplist=$(ifconfig |grep inet |grep -v inet6 |grep -v 127.0.0.1 |awk '{print $2}' |awk -F ":" '{print $2}') ||: + fi + + if [ -z "$iplist" ]; then + echo + echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" + localFqdn="127.0.0.1" + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + echo + return + fi + + echo -e -n "${GREEN}Please choose an IP from local IP list${NC}:" + echo + echo -e -n "${GREEN}$iplist${NC}" + echo + echo + echo -e -n "${GREEN}Notes: if IP is used as the node name, data can NOT be migrated to other machine directly${NC}:" + read localFqdn + while true; do + if [ ! -z "$localFqdn" ]; then + # Check if correct ip address + is_correct_ipaddr $localFqdn + retval=`echo $?` + if [[ $retval != 0 ]]; then + read -p "Please choose an IP from local IP list:" localFqdn + else + # Write the local FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/taos.cfg + serverFqdn=$localFqdn + break + fi + else + read -p "Please choose an IP from local IP list:" localFqdn + fi + done +} + +function local_fqdn_check() { + #serverFqdn=$(hostname) + echo + echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" + echo + if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then + echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" + echo + + while true + do + read -r -p "Set hostname now? [Y/n] " input + if [ ! -n "$input" ]; then + set_hostname + break + else + case $input in + [yY][eE][sS]|[yY]) + set_hostname + break + ;; + + [nN][oO]|[nN]) + set_ipAsFqdn + break + ;; + + *) + echo "Invalid input..." + ;; + esac + fi + done + fi +} + +function install_config() { + #${csudo} rm -f ${install_main_dir}/cfg/taos.cfg || : + + if [ ! -f ${cfg_install_dir}/taos.cfg ]; then + ${csudo} mkdir -p ${cfg_install_dir} + [ -f ${script_dir}/cfg/taos.cfg ] && ${csudo} cp ${script_dir}/cfg/taos.cfg ${cfg_install_dir} + ${csudo} chmod 644 ${cfg_install_dir}/* + fi + + ${csudo} cp -f ${script_dir}/cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org + ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + + [ ! -z $1 ] && return 0 || : # only install client + + if ((${update_flag}==1)); then + return 0 + fi + + if [ "$interactiveFqdn" == "no" ]; then + return 0 + fi + + local_fqdn_check + + #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" + #FQDN_FORMAT="(:[1-6][0-9][0-9][0-9][0-9]$)" + #PORT_FORMAT="(/[1-6][0-9][0-9][0-9][0-9]?/)" + #FQDN_PATTERN=":[0-9]{1,5}$" + + # first full-qualified domain name (FQDN) for TQ cluster system + echo + echo -e -n "${GREEN}Enter FQDN:port (like h1.taosdata.com:6030) of an existing TQ cluster node to join${NC}" + echo + echo -e -n "${GREEN}OR leave it blank to build one${NC}:" + read firstEp + while true; do + if [ ! -z "$firstEp" ]; then + # check the format of the firstEp + #if [[ $firstEp == $FQDN_PATTERN ]]; then + # Write the first FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/taos.cfg + break + #else + # read -p "Please enter the correct FQDN:port: " firstEp + #fi + else + break + fi + done +} + + +function install_log() { + ${csudo} rm -rf ${log_dir} || : + ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} + + ${csudo} ln -s ${log_dir} ${install_main_dir}/log +} + +function install_data() { + ${csudo} mkdir -p ${data_dir} + + ${csudo} ln -s ${data_dir} ${install_main_dir}/data +} + +function install_connector() { + ${csudo} cp -rf ${script_dir}/connector/* ${install_main_dir}/connector +} + +function install_examples() { + if [ -d ${script_dir}/examples ]; then + ${csudo} cp -rf ${script_dir}/examples/* ${install_main_dir}/examples + fi +} + +function clean_service_on_sysvinit() { + #restart_config_str="tq:2345:respawn:${service_config_dir}/tqd start" + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + + if pidof tqd &> /dev/null; then + ${csudo} service tqd stop || : + fi + + if pidof tarbitrator &> /dev/null; then + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tqd ]; then + ${csudo} chkconfig --del tqd || : + fi + + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tqd ]; then + ${csudo} insserv -r tqd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tqd ]; then + ${csudo} update-rc.d -f tqd remove || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tqd || : + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + # Install tqd service + + if ((${os_type}==1)); then + ${csudo} cp -f ${script_dir}/init.d/tqd.deb ${install_main_dir}/init.d/tqd + ${csudo} cp ${script_dir}/init.d/tqd.deb ${service_config_dir}/tqd && ${csudo} chmod a+x ${service_config_dir}/tqd + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type}==2)); then + ${csudo} cp -f ${script_dir}/init.d/tqd.rpm ${install_main_dir}/init.d/tqd + ${csudo} cp ${script_dir}/init.d/tqd.rpm ${service_config_dir}/tqd && ${csudo} chmod a+x ${service_config_dir}/tqd + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + fi + + #restart_config_str="tq:2345:respawn:${service_config_dir}/tqd start" + #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" + + if ((${initd_mod}==1)); then + ${csudo} chkconfig --add tqd || : + ${csudo} chkconfig --level 2345 tqd on || : + ${csudo} chkconfig --add tarbitratord || : + ${csudo} chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod}==2)); then + ${csudo} insserv tqd || : + ${csudo} insserv -d tqd || : + ${csudo} insserv tarbitratord || : + ${csudo} insserv -d tarbitratord || : + elif ((${initd_mod}==3)); then + ${csudo} update-rc.d tqd defaults || : + ${csudo} update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + tqd_service_config="${service_config_dir}/tqd.service" + if systemctl is-active --quiet tqd; then + echo "TQ is running, stopping it..." + ${csudo} systemctl stop tqd &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tqd &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tqd_service_config} + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tarbitratord_service_config} + + if [ "$verMode" == "cluster" ]; then + nginx_service_config="${service_config_dir}/nginxd.service" + if systemctl is-active --quiet nginxd; then + echo "Nginx for TDengine is running, stopping it..." + ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${nginx_service_config} + fi +} + +# tq:2345:respawn:/etc/init.d/tqd start + +function install_service_on_systemd() { + clean_service_on_systemd + + tqd_service_config="${service_config_dir}/tqd.service" + ${csudo} bash -c "echo '[Unit]' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'Description=TQ server service' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tqd_service_config}" + ${csudo} bash -c "echo >> ${tqd_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tqd' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'ExecStartPre=/usr/local/tq/bin/startPre.sh' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tqd_service_config}" + ${csudo} bash -c "echo >> ${tqd_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tqd_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tqd_service_config}" + ${csudo} systemctl enable tqd + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + #${csudo} systemctl enable tarbitratord + + if [ "$verMode" == "cluster" ]; then + nginx_service_config="${service_config_dir}/nginxd.service" + ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Description=Nginx For PowrDB Service' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${nginx_service_config}" + ${csudo} bash -c "echo >> ${nginx_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Type=forking' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'PIDFile=/usr/local/nginxd/logs/nginx.pid' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/local/nginxd/sbin/nginx' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'ExecStop=/usr/local/nginxd/sbin/nginx -s stop' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'TimeoutStopSec=1000000s' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${nginx_service_config}" + ${csudo} bash -c "echo >> ${nginx_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${nginx_service_config}" + if ! ${csudo} systemctl enable nginxd &> /dev/null; then + ${csudo} systemctl daemon-reexec + ${csudo} systemctl enable nginxd + fi + ${csudo} systemctl start nginxd + fi +} + +function install_service() { + if ((${service_mod}==0)); then + install_service_on_systemd + elif ((${service_mod}==1)); then + install_service_on_sysvinit + else + # must manual stop tqd + kill_process tqd + fi +} + +vercomp () { + if [[ $1 == $2 ]]; then + return 0 + fi + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do + ver1[i]=0 + done + + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + return 1 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + return 2 + fi + done + return 0 +} + +function is_version_compatible() { + + curr_version=$(${bin_dir}/tqd -V | head -1 | cut -d ' ' -f 3) + + min_compatible_version=$(${script_dir}/bin/tqd -V | head -1 | cut -d ' ' -f 5) + + vercomp $curr_version $min_compatible_version + case $? in + 0) return 0;; + 1) return 0;; + 2) return 1;; + esac +} + +function update_tq() { + # Start to update + if [ ! -e tq.tar.gz ]; then + echo "File tq.tar.gz does not exist" + exit 1 + fi + tar -zxf tq.tar.gz + + # Check if version compatible + if ! is_version_compatible; then + echo -e "${RED}Version incompatible${NC}" + return 1 + fi + + echo -e "${GREEN}Start to update TQ...${NC}" + # Stop the service if running + if pidof tqd &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop tqd || : + elif ((${service_mod}==1)); then + ${csudo} service tqd stop || : + else + kill_process tqd + fi + sleep 1 + fi + if [ "$verMode" == "cluster" ]; then + if pidof nginx &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop nginxd || : + elif ((${service_mod}==1)); then + ${csudo} service nginxd stop || : + else + kill_process nginx + fi + sleep 1 + fi + fi + + install_main_path + + install_log + install_header + install_lib + if [ "$pagMode" != "lite" ]; then + install_connector + fi + install_examples + if [ -z $1 ]; then + install_bin + install_service + install_config + + openresty_work=false + if [ "$verMode" == "cluster" ]; then + # Check if openresty is installed + # Check if nginx is installed successfully + if type curl &> /dev/null; then + if curl -sSf http://127.0.0.1:${nginx_port} &> /dev/null; then + echo -e "\033[44;32;1mNginx for TQ is updated successfully!${NC}" + openresty_work=true + else + echo -e "\033[44;31;5mNginx for TQ does not work! Please try again!\033[0m" + fi + fi + fi + + #echo + #echo -e "\033[44;32;1mTQ is updated successfully!${NC}" + echo + echo -e "${GREEN_DARK}To configure TQ ${NC}: edit /etc/tq/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start TQ ${NC}: ${csudo} systemctl start tqd${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start TQ ${NC}: ${csudo} service tqd start${NC}" + else + echo -e "${GREEN_DARK}To start TQ ${NC}: ./tqd${NC}" + fi + + if [ ${openresty_work} = 'true' ]; then + echo -e "${GREEN_DARK}To access TQ ${NC}: use ${GREEN_UNDERLINE}tq -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + else + echo -e "${GREEN_DARK}To access TQ ${NC}: use ${GREEN_UNDERLINE}tq -h $serverFqdn${NC} in shell${NC}" + fi + + echo + echo -e "\033[44;32;1mTQ is updated successfully!${NC}" + else + install_bin + install_config + + echo + echo -e "\033[44;32;1mTQ client is updated successfully!${NC}" + fi + + rm -rf $(tar -tf tq.tar.gz) +} + +function install_tq() { + # Start to install + if [ ! -e tq.tar.gz ]; then + echo "File tq.tar.gz does not exist" + exit 1 + fi + tar -zxf tq.tar.gz + + echo -e "${GREEN}Start to install TQ...${NC}" + + install_main_path + + if [ -z $1 ]; then + install_data + fi + + install_log + install_header + install_lib + if [ "$pagMode" != "lite" ]; then + install_connector + fi + install_examples + + if [ -z $1 ]; then # install service and client + # For installing new + install_bin + install_service + + openresty_work=false + if [ "$verMode" == "cluster" ]; then + # Check if nginx is installed successfully + if type curl &> /dev/null; then + if curl -sSf http://127.0.0.1:${nginx_port} &> /dev/null; then + echo -e "\033[44;32;1mNginx for TQ is installed successfully!${NC}" + openresty_work=true + else + echo -e "\033[44;31;5mNginx for TQ does not work! Please try again!\033[0m" + fi + fi + fi + + install_config + + # Ask if to start the service + #echo + #echo -e "\033[44;32;1mTQ is installed successfully!${NC}" + echo + echo -e "${GREEN_DARK}To configure TQ ${NC}: edit /etc/tq/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start TQ ${NC}: ${csudo} systemctl start tqd${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start TQ ${NC}: ${csudo} service tqd start${NC}" + else + echo -e "${GREEN_DARK}To start TQ ${NC}: tqd${NC}" + fi + + #if [ ${openresty_work} = 'true' ]; then + # echo -e "${GREEN_DARK}To access TQ ${NC}: use ${GREEN_UNDERLINE}tq${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" + #else + # echo -e "${GREEN_DARK}To access TQ ${NC}: use ${GREEN_UNDERLINE}tq${NC} in shell${NC}" + #fi + + if [ ! -z "$firstEp" ]; then + tmpFqdn=${firstEp%%:*} + substr=":" + if [[ $firstEp =~ $substr ]];then + tmpPort=${firstEp#*:} + else + tmpPort="" + fi + if [[ "$tmpPort" != "" ]];then + echo -e "${GREEN_DARK}To access TQ ${NC}: tq -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + else + echo -e "${GREEN_DARK}To access TQ ${NC}: tq -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + fi + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo + elif [ ! -z "$serverFqdn" ]; then + echo -e "${GREEN_DARK}To access TQ ${NC}: tq -h $serverFqdn${GREEN_DARK} to login into TQ server${NC}" + echo + fi + echo -e "\033[44;32;1mTQ is installed successfully!${NC}" + echo + else # Only install client + install_bin + install_config + + echo + echo -e "\033[44;32;1mTQ client is installed successfully!${NC}" + fi + + rm -rf $(tar -tf tq.tar.gz) +} + + +## ==============================Main program starts from here============================ +serverFqdn=$(hostname) +if [ "$verType" == "server" ]; then + # Install server and client + if [ -x ${bin_dir}/tqd ]; then + update_flag=1 + update_tq + else + install_tq + fi +elif [ "$verType" == "client" ]; then + interactiveFqdn=no + # Only install client + if [ -x ${bin_dir}/tq ]; then + update_flag=1 + update_tq client + else + install_tq client + fi +else + echo "please input correct verType" +fi diff --git a/packaging/tools/makearbi_tq.sh b/packaging/tools/makearbi_tq.sh new file mode 100755 index 0000000000000000000000000000000000000000..c10dfec255d411965a3887942e5d2aded4635979 --- /dev/null +++ b/packaging/tools/makearbi_tq.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Generate arbitrator's tar.gz setup package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/TQ-enterprise-arbitrator-${version}" +else + install_dir="${release_dir}/TQ-arbitrator-${version}" +fi + +# Directories and files. +bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi_tq.sh" +install_files="${script_dir}/install_arbi_tq.sh" + +#header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" +init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord + +# make directories. +mkdir -p ${install_dir} && cp ${install_files} ${install_dir} && chmod a+x ${install_dir}/install_arbi_tq.sh || : +#mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc || : +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/packaging/tools/makeclient_tq.sh b/packaging/tools/makeclient_tq.sh new file mode 100755 index 0000000000000000000000000000000000000000..51fd064e1b769191c2baaf27c3a45f73a475cabd --- /dev/null +++ b/packaging/tools/makeclient_tq.sh @@ -0,0 +1,203 @@ +#!/bin/bash +# +# Generate tar.gz package for linux client in all os system +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +if [ "$osType" != "Darwin" ]; then + script_dir="$(dirname $(readlink -f $0))" + top_dir="$(readlink -f ${script_dir}/../..)" +else + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + top_dir=${script_dir}/../.. +fi + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' + +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/TQ-enterprise-client-${version}" +else + install_dir="${release_dir}/TQ-client-${version}" +fi + +# Directories and files. + +if [ "$osType" != "Darwin" ]; then +# if [ "$pagMode" == "lite" ]; then +# strip ${build_dir}/bin/tqd +# strip ${build_dir}/bin/tq +# bin_files="${build_dir}/bin/tq ${script_dir}/remove_client_tq.sh" +# else +# bin_files="${build_dir}/bin/tq ${build_dir}/bin/tqdemo ${script_dir}/remove_client_tq.sh ${script_dir}/set_core.sh" +# fi + lib_files="${build_dir}/lib/libtaos.so.${version}" +else + bin_files="${build_dir}/bin/tq ${script_dir}/remove_client_tq.sh" + lib_files="${build_dir}/lib/libtaos.${version}.dylib" +fi + +header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi + +install_files="${script_dir}/install_client_tq.sh" + +# make directories. +mkdir -p ${install_dir} +mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg + +sed -i '/dataDir/ {s/taos/tq/g}' ${install_dir}/cfg/taos.cfg +sed -i '/logDir/ {s/taos/tq/g}' ${install_dir}/cfg/taos.cfg +sed -i "s/TDengine/TQ/g" ${install_dir}/cfg/taos.cfg + +mkdir -p ${install_dir}/bin +if [ "$osType" != "Darwin" ]; then + if [ "$pagMode" == "lite" ]; then + strip ${build_dir}/bin/taos + cp ${build_dir}/bin/taos ${install_dir}/bin/tq + cp ${script_dir}/remove_tq.sh ${install_dir}/bin + else + cp ${build_dir}/bin/taos ${install_dir}/bin/tq + cp ${script_dir}/remove_tq.sh ${install_dir}/bin + cp ${build_dir}/bin/taosdemo ${install_dir}/bin/tqdemo + cp ${build_dir}/bin/taosdump ${install_dir}/bin/tqdump + cp ${script_dir}/set_core.sh ${install_dir}/bin + cp ${script_dir}/get_client.sh ${install_dir}/bin + cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin + fi +else + cp ${bin_files} ${install_dir}/bin +fi +chmod a+x ${install_dir}/bin/* || : + +cd ${install_dir} + +if [ "$osType" != "Darwin" ]; then + tar -zcv -f tq.tar.gz * --remove-files || : +else + tar -zcv -f tq.tar.gz * || : + mv tq.tar.gz .. + rm -rf ./* + mv ../tq.tar.gz . +fi + +cd ${curr_dir} +cp ${install_files} ${install_dir} +if [ "$osType" == "Darwin" ]; then + sed 's/osType=Linux/osType=Darwin/g' ${install_dir}/install_client_tq.sh >> install_client_tq_temp.sh + mv install_client_tq_temp.sh ${install_dir}/install_client_tq.sh +fi +if [ "$pagMode" == "lite" ]; then + sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install_client_tq.sh >> install_client_tq_temp.sh + mv install_client_tq_temp.sh ${install_dir}/install_client_tq.sh +fi +chmod a+x ${install_dir}/install_client_tq.sh + +# Copy example code +mkdir -p ${install_dir}/examples +examples_dir="${top_dir}/tests/examples" +cp -r ${examples_dir}/c ${install_dir}/examples +sed -i '/passwd/ {s/taosdata/tqueue/g}' ${install_dir}/examples/c/*.c +sed -i '/root/ {s/taosdata/tqueue/g}' ${install_dir}/examples/c/*.c + +if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + cp -r ${examples_dir}/JDBC ${install_dir}/examples + cp -r ${examples_dir}/matlab ${install_dir}/examples + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/examples/matlab/TDengineDemo.m + cp -r ${examples_dir}/python ${install_dir}/examples + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/examples/python/read_example.py + cp -r ${examples_dir}/R ${install_dir}/examples + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/examples/R/command.txt + cp -r ${examples_dir}/go ${install_dir}/examples + sed -i '/root/ {s/taosdata/tqueue/g}' ${install_dir}/examples/go/taosdemo.go +fi +# Copy driver +mkdir -p ${install_dir}/driver +cp ${lib_files} ${install_dir}/driver + +# Copy connector +connector_dir="${code_dir}/connector" +mkdir -p ${install_dir}/connector + +if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + if [ "$osType" != "Darwin" ]; then + cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: + fi + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bunlded dir not found, please check if want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector + + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/cinterface.py + + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/subscription.py + + sed -i '/self._password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/connection.py +fi +# Copy release note +# cp ${script_dir}/release_note ${install_dir} + +# exit 1 + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$pagMode" == "lite" ]; then + pkg_name=${pkg_name}-Lite +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stable or beta" + exit 1 +fi + +if [ "$osType" != "Darwin" ]; then + tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +else + tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) || : + mv "$(basename ${pkg_name}).tar.gz" .. + rm -rf ./* + mv ../"$(basename ${pkg_name}).tar.gz" . +fi + +cd ${curr_dir} diff --git a/packaging/tools/makepkg_tq.sh b/packaging/tools/makepkg_tq.sh new file mode 100755 index 0000000000000000000000000000000000000000..086092ea33b7cfbb06b9f14e4af7224f6b686f73 --- /dev/null +++ b/packaging/tools/makepkg_tq.sh @@ -0,0 +1,224 @@ +#!/bin/bash +# +# Generate tar.gz package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/TQ-enterprise-server-${version}" +else + install_dir="${release_dir}/TQ-server-${version}" +fi + +# Directories and files. +#if [ "$pagMode" == "lite" ]; then +# strip ${build_dir}/bin/taosd +# strip ${build_dir}/bin/taos +# bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${script_dir}/remove_tq.sh" +#else +# bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${build_dir}/bin/tqdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_tq.sh\ +# ${script_dir}/set_core.sh ${script_dir}/startPre.sh ${script_dir}/taosd-dump-cfg.gdb" +#fi + +lib_files="${build_dir}/lib/libtaos.so.${version}" +header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi +install_files="${script_dir}/install_tq.sh" +nginx_dir="${code_dir}/../../enterprise/src/plugins/web" + +# Init file +#init_dir=${script_dir}/deb +#if [ $package_type = "centos" ]; then +# init_dir=${script_dir}/rpm +#fi +#init_files=${init_dir}/tqd +# temp use rpm's tqd. TODO: later modify according to os type +#init_file_deb=${script_dir}/../deb/tqd +#init_file_rpm=${script_dir}/../rpm/tqd +#init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +#init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord + +# make directories. +mkdir -p ${install_dir} +mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc +mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg + +#mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +mkdir -p ${install_dir}/bin +if [ "$pagMode" == "lite" ]; then + strip ${build_dir}/bin/taosd + strip ${build_dir}/bin/taos +# bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${script_dir}/remove_tq.sh" + cp ${build_dir}/bin/taos ${install_dir}/bin/tq + cp ${build_dir}/bin/taosd ${install_dir}/bin/tqd + cp ${script_dir}/remove_tq.sh ${install_dir}/bin +else +# bin_files="${build_dir}/bin/tqd ${build_dir}/bin/tq ${build_dir}/bin/tqdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_tq.sh ${script_dir}/set_core.sh" + cp ${build_dir}/bin/taos ${install_dir}/bin/tq + cp ${build_dir}/bin/taosd ${install_dir}/bin/tqd + cp ${script_dir}/remove_tq.sh ${install_dir}/bin + cp ${build_dir}/bin/taosdemo ${install_dir}/bin/tqdemo + cp ${build_dir}/bin/taosdump ${install_dir}/bin/tqdump + cp ${build_dir}/bin/tarbitrator ${install_dir}/bin + cp ${script_dir}/set_core.sh ${install_dir}/bin + cp ${script_dir}/get_client.sh ${install_dir}/bin + cp ${script_dir}/startPre.sh ${install_dir}/bin + cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin +fi +chmod a+x ${install_dir}/bin/* || : + +#mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/tqd.deb +#mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/tqd.rpm +#mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +#mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : + +if [ "$verMode" == "cluster" ]; then + sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove_tq.sh >> remove_tq_temp.sh + mv remove_tq_temp.sh ${install_dir}/bin/remove_tq.sh + + mkdir -p ${install_dir}/nginxd && cp -r ${nginx_dir}/* ${install_dir}/nginxd + cp ${nginx_dir}/png/taos.png ${install_dir}/nginxd/admin/images/taos.png + rm -rf ${install_dir}/nginxd/png + + sed -i "s/TDengine/TQ/g" ${install_dir}/nginxd/admin/*.html + sed -i "s/TDengine/TQ/g" ${install_dir}/nginxd/admin/js/*.js + + sed -i '/dataDir/ {s/taos/tq/g}' ${install_dir}/cfg/taos.cfg + sed -i '/logDir/ {s/taos/tq/g}' ${install_dir}/cfg/taos.cfg + sed -i "s/TDengine/TQ/g" ${install_dir}/cfg/taos.cfg + + if [ "$cpuType" == "aarch64" ]; then + cp -f ${install_dir}/nginxd/sbin/arm/64bit/nginx ${install_dir}/nginxd/sbin/ + elif [ "$cpuType" == "aarch32" ]; then + cp -f ${install_dir}/nginxd/sbin/arm/32bit/nginx ${install_dir}/nginxd/sbin/ + fi + rm -rf ${install_dir}/nginxd/sbin/arm +fi + +cd ${install_dir} +tar -zcv -f tq.tar.gz * --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar tq.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} +cp ${install_files} ${install_dir} +if [ "$verMode" == "cluster" ]; then + sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/install_tq.sh >> install_tq_temp.sh + mv install_tq_temp.sh ${install_dir}/install_tq.sh +fi +if [ "$pagMode" == "lite" ]; then + sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install.sh >> install_tq_temp.sh + mv install_tq_temp.sh ${install_dir}/install_tq.sh +fi +chmod a+x ${install_dir}/install_tq.sh + +# Copy example code +mkdir -p ${install_dir}/examples +examples_dir="${top_dir}/tests/examples" +cp -r ${examples_dir}/c ${install_dir}/examples +sed -i '/passwd/ {s/taosdata/tqueue/g}' ${install_dir}/examples/c/*.c +sed -i '/root/ {s/taosdata/tqueue/g}' ${install_dir}/examples/c/*.c + +if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + cp -r ${examples_dir}/JDBC ${install_dir}/examples + cp -r ${examples_dir}/matlab ${install_dir}/examples + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/examples/matlab/TDengineDemo.m + cp -r ${examples_dir}/python ${install_dir}/examples + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/examples/python/read_example.py + cp -r ${examples_dir}/R ${install_dir}/examples + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/examples/R/command.txt + cp -r ${examples_dir}/go ${install_dir}/examples + sed -i '/root/ {s/taosdata/tqueue/g}' ${install_dir}/examples/go/taosdemo.go +fi +# Copy driver +mkdir -p ${install_dir}/driver +cp ${lib_files} ${install_dir}/driver + +# Copy connector +connector_dir="${code_dir}/connector" +mkdir -p ${install_dir}/connector +if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then + cp ${build_dir}/lib/*.jar ${install_dir}/connector ||: + + if [ -d "${connector_dir}/grafanaplugin/dist" ]; then + cp -r ${connector_dir}/grafanaplugin/dist ${install_dir}/connector/grafanaplugin + else + echo "WARNING: grafanaplugin bundled dir not found, please check if want to use it!" + fi + if find ${connector_dir}/go -mindepth 1 -maxdepth 1 | read; then + cp -r ${connector_dir}/go ${install_dir}/connector + else + echo "WARNING: go connector not found, please check if want to use it!" + fi + cp -r ${connector_dir}/python ${install_dir}/connector/ + + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/cinterface.py + + sed -i '/password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/subscription.py + + sed -i '/self._password/ {s/taosdata/tqueue/g}' ${install_dir}/connector/python/taos/connection.py +fi +# Copy release note +# cp ${script_dir}/release_note ${install_dir} + +# exit 1 + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$pagMode" == "lite" ]; then + pkg_name=${pkg_name}-Lite +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/packaging/tools/remove_arbi_tq.sh b/packaging/tools/remove_arbi_tq.sh new file mode 100755 index 0000000000000000000000000000000000000000..3d99b6d41a74938d74383df3d8cdfc75c2ebb7c8 --- /dev/null +++ b/packaging/tools/remove_arbi_tq.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# +# Script to stop the service and uninstall TQ's arbitrator + +set -e +#set -x + +verMode=edge + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/tarbitrator" +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +service_config_dir="/etc/systemd/system" +tarbitrator_service_name="tarbitratord" +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf /arbitrator.log || : +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "TQ tarbitrator is running, stopping it..." + ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +function clean_service_on_sysvinit() { + if pidof tarbitrator &> /dev/null; then + echo "TQ's tarbitrator is running, stopping it..." + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + # must manual stop + kill_tarbitrator + fi +} + +# Stop service and disable booting start. +clean_service +# Remove binary file and links +clean_bin +# Remove header file. +##clean_header +# Remove log file +clean_log + +${csudo} rm -rf ${install_main_dir} + +echo -e "${GREEN}TQ's arbitrator is removed successfully!${NC}" +echo \ No newline at end of file diff --git a/packaging/tools/remove_client_tq.sh b/packaging/tools/remove_client_tq.sh new file mode 100755 index 0000000000000000000000000000000000000000..ad8056c18cc32623edb8b77bf6aa17070acc1cbc --- /dev/null +++ b/packaging/tools/remove_client_tq.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# +# Script to stop the client and uninstall database, but retain the config and log files. +set -e +# set -x + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/tq" + +log_link_dir="/usr/local/tq/log" +cfg_link_dir="/usr/local/tq/cfg" +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" + + +# v1.5 jar dir +#v15_java_app_dir="/usr/local/lib/tq" + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +function kill_client() { + #pid=$(ps -ef | grep "tq" | grep -v "grep" | awk '{print $2}') + if [ -n "$(pidof tq)" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/tq || : + ${csudo} rm -f ${bin_link_dir}/tqdemo || : + ${csudo} rm -f ${bin_link_dir}/tqdump || : + ${csudo} rm -f ${bin_link_dir}/rmtq || : + ${csudo} rm -f ${bin_link_dir}/set_core || : +} + +function clean_lib() { + # Remove link + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo} rm -rf ${v15_java_app_dir} || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_config() { + # Remove link + ${csudo} rm -f ${cfg_link_dir}/* || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf ${log_link_dir} || : +} + +# Stop client. +kill_client +# Remove binary file and links +clean_bin +# Remove header file. +clean_header +# Remove lib file +clean_lib +# Remove link log directory +clean_log +# Remove link configuration file +clean_config + +${csudo} rm -rf ${install_main_dir} + +echo -e "${GREEN}TQ client is removed successfully!${NC}" +echo diff --git a/packaging/tools/remove_tq.sh b/packaging/tools/remove_tq.sh new file mode 100755 index 0000000000000000000000000000000000000000..211eed4dff09ab5da00d5c475cd93148b5ce1b24 --- /dev/null +++ b/packaging/tools/remove_tq.sh @@ -0,0 +1,227 @@ +#!/bin/bash +# +# Script to stop the service and uninstall TDengine, but retain the config, data and log files. + +set -e +#set -x + +verMode=edge + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/tq" +data_link_dir="/usr/local/tq/data" +log_link_dir="/usr/local/tq/log" +cfg_link_dir="/usr/local/tq/cfg" +bin_link_dir="/usr/bin" +lib_link_dir="/usr/lib" +lib64_link_dir="/usr/lib64" +inc_link_dir="/usr/include" +install_nginxd_dir="/usr/local/nginxd" + +# v1.5 jar dir +#v15_java_app_dir="/usr/local/lib/tq" + +service_config_dir="/etc/systemd/system" +tq_service_name="tqd" +tarbitrator_service_name="tarbitratord" +nginx_service_name="nginxd" +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_tqd() { + pid=$(ps -ef | grep "tqd" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/tq || : + ${csudo} rm -f ${bin_link_dir}/tqd || : + ${csudo} rm -f ${bin_link_dir}/tqdemo || : + ${csudo} rm -f ${bin_link_dir}/tqdump || : + ${csudo} rm -f ${bin_link_dir}/rmtq || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} rm -f ${bin_link_dir}/set_core || : +} + +function clean_lib() { + # Remove link + ${csudo} rm -f ${lib_link_dir}/libtaos.* || : + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + #${csudo} rm -rf ${v15_java_app_dir} || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_config() { + # Remove link + ${csudo} rm -f ${cfg_link_dir}/* || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf ${log_link_dir} || : +} + +function clean_service_on_systemd() { + tq_service_config="${service_config_dir}/${tq_service_name}.service" + if systemctl is-active --quiet ${tq_service_name}; then + echo "TQ tqd is running, stopping it..." + ${csudo} systemctl stop ${tq_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tq_service_name} &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tq_service_config} + + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "TDengine tarbitrator is running, stopping it..." + ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tarbitratord_service_config} + + if [ "$verMode" == "cluster" ]; then + nginx_service_config="${service_config_dir}/${nginx_service_name}.service" + if [ -d ${bin_dir}/web ]; then + if systemctl is-active --quiet ${nginx_service_name}; then + echo "Nginx for TDengine is running, stopping it..." + ${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${nginx_service_name} &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${nginx_service_config} + fi + fi +} + +function clean_service_on_sysvinit() { + #restart_config_str="tq:2345:respawn:${service_config_dir}/tqd start" + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + + if pidof tqd &> /dev/null; then + echo "TQ tqd is running, stopping it..." + ${csudo} service tqd stop || : + fi + + if pidof tarbitrator &> /dev/null; then + echo "TQ tarbitrator is running, stopping it..." + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tqd ]; then + ${csudo} chkconfig --del tqd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tqd ]; then + ${csudo} insserv -r tqd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tqd ]; then + ${csudo} update-rc.d -f tqd remove || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tqd || : + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + # must manual stop taosd + kill_tqd + kill_tarbitrator + fi +} + +# Stop service and disable booting start. +clean_service +# Remove binary file and links +clean_bin +# Remove header file. +clean_header +# Remove lib file +clean_lib +# Remove link log directory +clean_log +# Remove link configuration file +clean_config +# Remove data link directory +${csudo} rm -rf ${data_link_dir} || : + +${csudo} rm -rf ${install_main_dir} +${csudo} rm -rf ${install_nginxd_dir} +if [[ -e /etc/os-release ]]; then + osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +else + osinfo="" +fi + +#if echo $osinfo | grep -qwi "ubuntu" ; then +## echo "this is ubuntu system" +# ${csudo} rm -f /var/lib/dpkg/info/tdengine* || : +#elif echo $osinfo | grep -qwi "debian" ; then +## echo "this is debian system" +# ${csudo} rm -f /var/lib/dpkg/info/tdengine* || : +#elif echo $osinfo | grep -qwi "centos" ; then +## echo "this is centos system" +# ${csudo} rpm -e --noscripts tdengine || : +#fi + +echo -e "${GREEN}TQ is removed successfully!${NC}" +echo diff --git a/packaging/tools/startPre.sh b/packaging/tools/startPre.sh old mode 100644 new mode 100755 diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 668a9e940657f0cde8d8406dbe4a18fdb9c72f7e..d1a325be3592a1789ac62661e61a84e6ccb969d3 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -920,7 +920,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { } else if (pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE) { pRes->code = tscProcessShowCreateDatabase(pSql); } else if (pCmd->command == TSDB_SQL_RESET_CACHE) { - taosHashEmpty(tscTableMetaInfo); + taosHashClear(tscTableMetaInfo); pRes->code = TSDB_CODE_SUCCESS; } else if (pCmd->command == TSDB_SQL_SERV_VERSION) { pRes->code = tscProcessServerVer(pSql); diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index b75088ba282b29c64ae72ccc4b52928b3b9bd1ee..08d3cc599ee0d66df11da5b22ebdf3437a58295f 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1163,7 +1163,7 @@ static void insertBatchClean(STscStmt* pStmt) { pCmd->insertParam.pDataBlocks = tscDestroyBlockArrayList(pCmd->insertParam.pDataBlocks); pCmd->insertParam.numOfTables = 0; - taosHashEmpty(pCmd->insertParam.pTableBlockHashList); + taosHashClear(pCmd->insertParam.pTableBlockHashList); tscFreeSqlResult(pSql); tscFreeSubobj(pSql); tfree(pSql->pSubs); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4d04d4184ff1aa9f2c63cb9999a0750b981a9f9e..f0889d20b06fc8b0fa523a78028a3268b5cd9b11 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -916,7 +916,8 @@ static int32_t checkInvalidExprForTimeWindow(SSqlCmd* pCmd, SQueryInfo* pQueryIn int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode) { const char* msg1 = "sliding cannot be used without interval"; - const char* msg2 = "interval cannot be less than 10 ms"; + const char* msg2 = "interval cannot be less than 1 us"; + const char* msg3 = "interval value is too small"; SSqlCmd* pCmd = &pSql->cmd; @@ -943,6 +944,10 @@ int32_t validateIntervalNode(SSqlObj* pSql, SQueryInfo* pQueryInfo, SSqlNode* pS return TSDB_CODE_TSC_INVALID_OPERATION; } + if (pQueryInfo->interval.interval <= 0) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') { // interval cannot be less than 10 milliseconds diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 029fa853aca50c9d66597b7ece7d8bfc7468305f..ac4243d5b4e3302d500cd54bba5a160c94b26a93 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2309,7 +2309,7 @@ int tscProcessDropDbRsp(SSqlObj *pSql) { //TODO LOCK DB WHEN MODIFY IT //pSql->pTscObj->db[0] = 0; - taosHashEmpty(tscTableMetaInfo); + taosHashClear(tscTableMetaInfo); return 0; } @@ -2340,7 +2340,7 @@ int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { tfree(pTableMetaInfo->pTableMeta); if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta - taosHashEmpty(tscTableMetaInfo); + taosHashClear(tscTableMetaInfo); } return 0; diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 365f24e12608ad99f6c317df7f4103fc4b07282f..d5c13885d55052346d0e25f9d2dea116ec13b7a8 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -81,6 +81,8 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_DEFAULT_USER "root" #ifdef _TD_POWER_ #define TSDB_DEFAULT_PASS "powerdb" +#elif (_TD_TQ_ == true) +#define TSDB_DEFAULT_PASS "tqueue" #else #define TSDB_DEFAULT_PASS "taosdata" #endif diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index f6cb135dd11b20ff2b04b6b56e48c878b993b7e0..58f4b7ff02b673288878aa44671ba2a544556cc5 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -37,6 +37,13 @@ char PROMPT_HEADER[] = "power> "; char CONTINUE_PROMPT[] = " -> "; int prompt_size = 7; +#elif (_TD_TQ_ == true) +char CLIENT_VERSION[] = "Welcome to the TQ shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by TQ, Inc. All rights reserved.\n\n"; +char PROMPT_HEADER[] = "tq> "; + +char CONTINUE_PROMPT[] = " -> "; +int prompt_size = 4; #else char CLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 17e417d3f7fa11789b1736f7ac29b2ef06f2bdd4..6513f3e214d4c16ebf86ecdbb9ff1ca3debe0f59 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -571,6 +571,8 @@ SArguments g_args = { "root", // user #ifdef _TD_POWER_ "powerdb", // password + #elif (_TD_TQ_ == true) + "tqueue", // password #else "taosdata", // password #endif @@ -681,6 +683,11 @@ static void printHelp() { "The password to use when connecting to the server. Default is 'powerdb'."); printf("%s%s%s%s\n", indent, "-c", indent, "Configuration directory. Default is '/etc/power/'."); +#elif (_TD_TQ_ == true) + printf("%s%s%s%s\n", indent, "-P", indent, + "The password to use when connecting to the server. Default is 'tqueue'."); + printf("%s%s%s%s\n", indent, "-c", indent, + "Configuration directory. Default is '/etc/tq/'."); #else printf("%s%s%s%s\n", indent, "-P", indent, "The password to use when connecting to the server. Default is 'taosdata'."); diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index d6feef4ccc8ebc92574de7534b2ea8ad145060da..c2b5e11e67c240d6e1211bf1446aad0b4658c58d 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -27,7 +27,9 @@ #include "tutil.h" #include -#define COMMAND_SIZE 65536 +#define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255 +#define COMMAND_SIZE 65536 +#define MAX_RECORDS_PER_REQ 32766 //#define DEFAULT_DUMP_FILE "taosdump.sql" // for strncpy buffer overflow @@ -61,120 +63,120 @@ typedef struct { // -------------------------- SHOW DATABASE INTERFACE----------------------- enum _show_db_index { - TSDB_SHOW_DB_NAME_INDEX, - TSDB_SHOW_DB_CREATED_TIME_INDEX, - TSDB_SHOW_DB_NTABLES_INDEX, - TSDB_SHOW_DB_VGROUPS_INDEX, - TSDB_SHOW_DB_REPLICA_INDEX, - TSDB_SHOW_DB_QUORUM_INDEX, - TSDB_SHOW_DB_DAYS_INDEX, - TSDB_SHOW_DB_KEEP_INDEX, - TSDB_SHOW_DB_CACHE_INDEX, - TSDB_SHOW_DB_BLOCKS_INDEX, - TSDB_SHOW_DB_MINROWS_INDEX, - TSDB_SHOW_DB_MAXROWS_INDEX, - TSDB_SHOW_DB_WALLEVEL_INDEX, - TSDB_SHOW_DB_FSYNC_INDEX, - TSDB_SHOW_DB_COMP_INDEX, - TSDB_SHOW_DB_CACHELAST_INDEX, - TSDB_SHOW_DB_PRECISION_INDEX, - TSDB_SHOW_DB_UPDATE_INDEX, - TSDB_SHOW_DB_STATUS_INDEX, - TSDB_MAX_SHOW_DB + TSDB_SHOW_DB_NAME_INDEX, + TSDB_SHOW_DB_CREATED_TIME_INDEX, + TSDB_SHOW_DB_NTABLES_INDEX, + TSDB_SHOW_DB_VGROUPS_INDEX, + TSDB_SHOW_DB_REPLICA_INDEX, + TSDB_SHOW_DB_QUORUM_INDEX, + TSDB_SHOW_DB_DAYS_INDEX, + TSDB_SHOW_DB_KEEP_INDEX, + TSDB_SHOW_DB_CACHE_INDEX, + TSDB_SHOW_DB_BLOCKS_INDEX, + TSDB_SHOW_DB_MINROWS_INDEX, + TSDB_SHOW_DB_MAXROWS_INDEX, + TSDB_SHOW_DB_WALLEVEL_INDEX, + TSDB_SHOW_DB_FSYNC_INDEX, + TSDB_SHOW_DB_COMP_INDEX, + TSDB_SHOW_DB_CACHELAST_INDEX, + TSDB_SHOW_DB_PRECISION_INDEX, + TSDB_SHOW_DB_UPDATE_INDEX, + TSDB_SHOW_DB_STATUS_INDEX, + TSDB_MAX_SHOW_DB }; // -----------------------------------------SHOW TABLES CONFIGURE ------------------------------------- enum _show_tables_index { - TSDB_SHOW_TABLES_NAME_INDEX, - TSDB_SHOW_TABLES_CREATED_TIME_INDEX, - TSDB_SHOW_TABLES_COLUMNS_INDEX, - TSDB_SHOW_TABLES_METRIC_INDEX, - TSDB_SHOW_TABLES_UID_INDEX, - TSDB_SHOW_TABLES_TID_INDEX, - TSDB_SHOW_TABLES_VGID_INDEX, - TSDB_MAX_SHOW_TABLES + TSDB_SHOW_TABLES_NAME_INDEX, + TSDB_SHOW_TABLES_CREATED_TIME_INDEX, + TSDB_SHOW_TABLES_COLUMNS_INDEX, + TSDB_SHOW_TABLES_METRIC_INDEX, + TSDB_SHOW_TABLES_UID_INDEX, + TSDB_SHOW_TABLES_TID_INDEX, + TSDB_SHOW_TABLES_VGID_INDEX, + TSDB_MAX_SHOW_TABLES }; // ---------------------------------- DESCRIBE METRIC CONFIGURE ------------------------------ enum _describe_table_index { - TSDB_DESCRIBE_METRIC_FIELD_INDEX, - TSDB_DESCRIBE_METRIC_TYPE_INDEX, - TSDB_DESCRIBE_METRIC_LENGTH_INDEX, - TSDB_DESCRIBE_METRIC_NOTE_INDEX, - TSDB_MAX_DESCRIBE_METRIC + TSDB_DESCRIBE_METRIC_FIELD_INDEX, + TSDB_DESCRIBE_METRIC_TYPE_INDEX, + TSDB_DESCRIBE_METRIC_LENGTH_INDEX, + TSDB_DESCRIBE_METRIC_NOTE_INDEX, + TSDB_MAX_DESCRIBE_METRIC }; #define COL_NOTE_LEN 128 typedef struct { - char field[TSDB_COL_NAME_LEN + 1]; - char type[16]; - int length; - char note[COL_NOTE_LEN]; + char field[TSDB_COL_NAME_LEN + 1]; + char type[16]; + int length; + char note[COL_NOTE_LEN]; } SColDes; typedef struct { - char name[TSDB_TABLE_NAME_LEN]; - SColDes cols[]; + char name[TSDB_TABLE_NAME_LEN]; + SColDes cols[]; } STableDef; extern char version[]; typedef struct { - char name[TSDB_DB_NAME_LEN]; - char create_time[32]; - int32_t ntables; - int32_t vgroups; - int16_t replica; - int16_t quorum; - int16_t days; - char keeplist[32]; - //int16_t daysToKeep; - //int16_t daysToKeep1; - //int16_t daysToKeep2; - int32_t cache; //MB - int32_t blocks; - int32_t minrows; - int32_t maxrows; - int8_t wallevel; - int32_t fsync; - int8_t comp; - int8_t cachelast; - char precision[8]; // time resolution - int8_t update; - char status[16]; + char name[TSDB_DB_NAME_LEN]; + char create_time[32]; + int32_t ntables; + int32_t vgroups; + int16_t replica; + int16_t quorum; + int16_t days; + char keeplist[32]; + //int16_t daysToKeep; + //int16_t daysToKeep1; + //int16_t daysToKeep2; + int32_t cache; //MB + int32_t blocks; + int32_t minrows; + int32_t maxrows; + int8_t wallevel; + int32_t fsync; + int8_t comp; + int8_t cachelast; + char precision[8]; // time resolution + int8_t update; + char status[16]; } SDbInfo; typedef struct { - char name[TSDB_TABLE_NAME_LEN]; - char metric[TSDB_TABLE_NAME_LEN]; + char name[TSDB_TABLE_NAME_LEN]; + char metric[TSDB_TABLE_NAME_LEN]; } STableRecord; typedef struct { - bool isMetric; - STableRecord tableRecord; + bool isMetric; + STableRecord tableRecord; } STableRecordInfo; typedef struct { - pthread_t threadID; - int32_t threadIndex; - int32_t totalThreads; - char dbName[TSDB_DB_NAME_LEN]; - void *taosCon; - int64_t rowsOfDumpOut; - int64_t tablesOfDumpOut; + pthread_t threadID; + int32_t threadIndex; + int32_t totalThreads; + char dbName[TSDB_DB_NAME_LEN]; + void *taosCon; + int64_t rowsOfDumpOut; + int64_t tablesOfDumpOut; } SThreadParaObj; typedef struct { - int64_t totalRowsOfDumpOut; - int64_t totalChildTblsOfDumpOut; - int32_t totalSuperTblsOfDumpOut; - int32_t totalDatabasesOfDumpOut; + int64_t totalRowsOfDumpOut; + int64_t totalChildTblsOfDumpOut; + int32_t totalSuperTblsOfDumpOut; + int32_t totalDatabasesOfDumpOut; } resultStatistics; -static int64_t totalDumpOutRows = 0; +static int64_t g_totalDumpOutRows = 0; -SDbInfo **dbInfos = NULL; +SDbInfo **g_dbInfos = NULL; const char *argp_program_version = version; const char *argp_program_bug_address = ""; @@ -195,1277 +197,1377 @@ static char args_doc[] = "dbname [tbname ...]\n--databases dbname ...\n--all-dat /* The options we understand. */ static struct argp_option options[] = { - // connection option - {"host", 'h', "HOST", 0, "Server host dumping data from. Default is localhost.", 0}, - {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, - #ifdef _TD_POWER_ - {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is powerdb.", 0}, - #else - {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0}, - #endif - {"port", 'P', "PORT", 0, "Port to connect", 0}, - {"cversion", 'v', "CVERION", 0, "client version", 0}, - {"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0}, - // input/output file - {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, - {"inpath", 'i', "INPATH", 0, "Input file path.", 1}, - {"resultFile", 'r', "RESULTFILE", 0, "DumpOut/In Result file path and name.", 1}, - #ifdef _TD_POWER_ - {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, - #else - {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, - #endif - {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, - // dump unit options - {"all-databases", 'A', 0, 0, "Dump all databases.", 2}, - {"databases", 'D', 0, 0, "Dump assigned databases", 2}, - // dump format options - {"schemaonly", 's', 0, 0, "Only dump schema.", 3}, - {"without-property", 'N', 0, 0, "Dump schema without properties.", 3}, - {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, - {"end-time", 'E', "END_TIME", 0, "End time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, - {"data-batch", 'B', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3}, - {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3}, - {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, - {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, - {"allow-sys", 'a', 0, 0, "Allow to dump sys database", 3}, - {"debug", 'g', 0, 0, "Print debug info.", 1}, - {"verbose", 'v', 0, 0, "Print verbose debug info.", 1}, - {0}}; + // connection option + {"host", 'h', "HOST", 0, "Server host dumping data from. Default is localhost.", 0}, + {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, +#ifdef _TD_POWER_ + {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is powerdb.", 0}, +#else + {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0}, +#endif + {"port", 'P', "PORT", 0, "Port to connect", 0}, + {"cversion", 'v', "CVERION", 0, "client version", 0}, + {"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0}, + // input/output file + {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, + {"inpath", 'i', "INPATH", 0, "Input file path.", 1}, + {"resultFile", 'r', "RESULTFILE", 0, "DumpOut/In Result file path and name.", 1}, +#ifdef _TD_POWER_ + {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, +#else + {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, +#endif + {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, + // dump unit options + {"all-databases", 'A', 0, 0, "Dump all databases.", 2}, + {"databases", 'D', 0, 0, "Dump assigned databases", 2}, + {"allow-sys", 'a', 0, 0, "Allow to dump sys database", 2}, + // dump format options + {"schemaonly", 's', 0, 0, "Only dump schema.", 2}, + {"without-property", 'N', 0, 0, "Dump schema without properties.", 2}, + {"avro", 'V', 0, 0, "Dump apache avro format data file. By default, dump sql command sequence.", 2}, + {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, + {"end-time", 'E', "END_TIME", 0, "End time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, + {"data-batch", 'B', "DATA_BATCH", 0, "Number of data point per insert statement. Max value is 32766. Default is 1.", 3}, + {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3}, + {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, + {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, + {"debug", 'g', 0, 0, "Print debug info.", 4}, + {"verbose", 'b', 0, 0, "Print verbose debug info.", 5}, + {"performanceprint", 'm', 0, 0, "Print performance debug info.", 5}, + {0} +}; /* Used by main to communicate with parse_opt. */ typedef struct arguments { - // connection option - char *host; - char *user; - char *password; - uint16_t port; - char cversion[12]; - uint16_t mysqlFlag; - // output file - char outpath[TSDB_FILENAME_LEN+1]; - char inpath[TSDB_FILENAME_LEN+1]; - // result file - char *resultFile; - char *encode; - // dump unit option - bool all_databases; - bool databases; - // dump format option - bool schemaonly; - bool with_property; - int64_t start_time; - int64_t end_time; - int32_t data_batch; - int32_t max_sql_len; - int32_t table_batch; // num of table which will be dump into one output file. - bool allow_sys; - // other options - int32_t thread_num; - int abort; - char **arg_list; - int arg_list_len; - bool isDumpIn; - bool debug_print; - bool verbose_print; - bool performance_print; -} SArguments; - -/* Parse a single option. */ -static error_t parse_opt(int key, char *arg, struct argp_state *state) { - /* Get the input argument from argp_parse, which we - know is a pointer to our arguments structure. */ - struct arguments *arguments = state->input; - wordexp_t full_path; - - switch (key) { // connection option - case 'a': - arguments->allow_sys = true; - break; - case 'h': - arguments->host = arg; - break; - case 'u': - arguments->user = arg; - break; - case 'p': - arguments->password = arg; - break; - case 'P': - arguments->port = atoi(arg); - break; - case 'q': - arguments->mysqlFlag = atoi(arg); - break; - case 'v': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid client vesion %s\n", arg); - return -1; - } - tstrncpy(arguments->cversion, full_path.we_wordv[0], 11); - wordfree(&full_path); - break; - // output file path - case 'o': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - tstrncpy(arguments->outpath, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'g': - arguments->debug_print = true; - break; - case 'i': - arguments->isDumpIn = true; - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - tstrncpy(arguments->inpath, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'r': - arguments->resultFile = arg; - break; - case 'c': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'e': - arguments->encode = arg; - break; + char *host; + char *user; + char *password; + uint16_t port; + char cversion[12]; + uint16_t mysqlFlag; + // output file + char outpath[MAX_FILE_NAME_LEN]; + char inpath[MAX_FILE_NAME_LEN]; + // result file + char *resultFile; + char *encode; // dump unit option - case 'A': - arguments->all_databases = true; - break; - case 'D': - arguments->databases = true; - break; + bool all_databases; + bool databases; // dump format option - case 's': - arguments->schemaonly = true; - break; - case 'N': - arguments->with_property = false; - break; - case 'S': - // parse time here. - arguments->start_time = atol(arg); - break; - case 'E': - arguments->end_time = atol(arg); - break; - case 'B': - arguments->data_batch = atoi(arg); - if (arguments->data_batch >= INT16_MAX) { - arguments->data_batch = INT16_MAX - 1; - } - break; - case 'L': - { - int32_t len = atoi(arg); - if (len > TSDB_MAX_ALLOWED_SQL_LEN) { - len = TSDB_MAX_ALLOWED_SQL_LEN; - } else if (len < TSDB_MAX_SQL_LEN) { - len = TSDB_MAX_SQL_LEN; - } - arguments->max_sql_len = len; - break; - } - case 't': - arguments->table_batch = atoi(arg); - break; - case 'T': - arguments->thread_num = atoi(arg); - break; - case OPT_ABORT: - arguments->abort = 1; - break; - case ARGP_KEY_ARG: - arguments->arg_list = &state->argv[state->next - 1]; - arguments->arg_list_len = state->argc - state->next + 1; - state->next = state->argc; - break; - - default: - return ARGP_ERR_UNKNOWN; - } - return 0; -} + bool schemaonly; + bool with_property; + bool avro; + int64_t start_time; + int64_t end_time; + int32_t data_batch; + int32_t max_sql_len; + int32_t table_batch; // num of table which will be dump into one output file. + bool allow_sys; + // other options + int32_t thread_num; + int abort; + char **arg_list; + int arg_list_len; + bool isDumpIn; + bool debug_print; + bool verbose_print; + bool performance_print; +} SArguments; /* Our argp parser. */ +static error_t parse_opt(int key, char *arg, struct argp_state *state); + static struct argp argp = {options, parse_opt, args_doc, doc}; static resultStatistics g_resultStatistics = {0}; static FILE *g_fpOfResult = NULL; static int g_numOfCores = 1; -static int taosDumpOut(struct arguments *arguments); -static int taosDumpIn(struct arguments *arguments); -static void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp); -static int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon); -static int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, char* dbName); -static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, char* dbName); -static void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName); -static int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName); -static int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName); +static int taosDumpOut(); +static int taosDumpIn(); +static void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, + FILE *fp); +static int taosDumpDb(SDbInfo *dbInfo, FILE *fp, TAOS *taosCon); +static int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, + char* dbName); +static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, + FILE *fp, char* dbName); +static void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, + int numOfCols, FILE *fp, char* dbName); +static int32_t taosDumpTable(char *table, char *metric, + FILE *fp, TAOS* taosCon, char* dbName); +static int taosDumpTableData(FILE *fp, char *tbName, + TAOS* taosCon, char* dbName, + char *jsonAvroSchema); static int taosCheckParam(struct arguments *arguments); static void taosFreeDbInfos(); -static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName); +static void taosStartDumpOutWorkThreads(int32_t numOfThread, char *dbName); struct arguments g_args = { - // connection option - NULL, - "root", - #ifdef _TD_POWER_ - "powerdb", - #else - "taosdata", - #endif - 0, - "", - 0, - // outpath and inpath - "", - "", - "./dump_result.txt", - NULL, - // dump unit option - false, - false, - // dump format option - false, // schemeonly - true, // with_property - 0, - INT64_MAX, - 1, - TSDB_MAX_SQL_LEN, - 1, - false, - // other options - 5, - 0, - NULL, - 0, - false, - false, // debug_print - false, // verbose_print - false // performance_print + // connection option + NULL, + "root", +#ifdef _TD_POWER_ + "powerdb", +#else + "taosdata", +#endif + 0, + "", + 0, + // outpath and inpath + "", + "", + "./dump_result.txt", + NULL, + // dump unit option + false, + false, + // dump format option + false, // schemeonly + true, // with_property + false, // avro format + 0, // start_time + INT64_MAX, // end_time + 1, // data_batch + TSDB_MAX_SQL_LEN, // max_sql_len + 1, // table_batch + false, // allow_sys + // other options + 5, // thread_num + 0, // abort + NULL, // arg_list + 0, // arg_list_len + false, // isDumpIn + false, // debug_print + false, // verbose_print + false // performance_print }; +/* Parse a single option. */ +static error_t parse_opt(int key, char *arg, struct argp_state *state) { + /* Get the input argument from argp_parse, which we + know is a pointer to our arguments structure. */ + wordexp_t full_path; + + switch (key) { + // connection option + case 'a': + g_args.allow_sys = true; + break; + case 'h': + g_args.host = arg; + break; + case 'u': + g_args.user = arg; + break; + case 'p': + g_args.password = arg; + break; + case 'P': + g_args.port = atoi(arg); + break; + case 'q': + g_args.mysqlFlag = atoi(arg); + break; + case 'v': + if (wordexp(arg, &full_path, 0) != 0) { + errorPrint("Invalid client vesion %s\n", arg); + return -1; + } + tstrncpy(g_args.cversion, full_path.we_wordv[0], 11); + wordfree(&full_path); + break; + // output file path + case 'o': + if (wordexp(arg, &full_path, 0) != 0) { + errorPrint("Invalid path %s\n", arg); + return -1; + } + tstrncpy(g_args.outpath, full_path.we_wordv[0], + MAX_FILE_NAME_LEN); + wordfree(&full_path); + break; + case 'g': + g_args.debug_print = true; + break; + case 'i': + g_args.isDumpIn = true; + if (wordexp(arg, &full_path, 0) != 0) { + errorPrint("Invalid path %s\n", arg); + return -1; + } + tstrncpy(g_args.inpath, full_path.we_wordv[0], + MAX_FILE_NAME_LEN); + wordfree(&full_path); + break; + case 'r': + g_args.resultFile = arg; + break; + case 'c': + if (wordexp(arg, &full_path, 0) != 0) { + errorPrint("Invalid path %s\n", arg); + return -1; + } + tstrncpy(configDir, full_path.we_wordv[0], MAX_FILE_NAME_LEN); + wordfree(&full_path); + break; + case 'e': + g_args.encode = arg; + break; + // dump unit option + case 'A': + g_args.all_databases = true; + break; + case 'D': + g_args.databases = true; + break; + // dump format option + case 's': + g_args.schemaonly = true; + break; + case 'N': + g_args.with_property = false; + break; + case 'V': + g_args.avro = true; + break; + case 'S': + // parse time here. + g_args.start_time = atol(arg); + break; + case 'E': + g_args.end_time = atol(arg); + break; + case 'B': + g_args.data_batch = atoi(arg); + if (g_args.data_batch > MAX_RECORDS_PER_REQ) { + g_args.data_batch = MAX_RECORDS_PER_REQ; + } + break; + case 'L': + { + int32_t len = atoi(arg); + if (len > TSDB_MAX_ALLOWED_SQL_LEN) { + len = TSDB_MAX_ALLOWED_SQL_LEN; + } else if (len < TSDB_MAX_SQL_LEN) { + len = TSDB_MAX_SQL_LEN; + } + g_args.max_sql_len = len; + break; + } + case 't': + g_args.table_batch = atoi(arg); + break; + case 'T': + g_args.thread_num = atoi(arg); + break; + case OPT_ABORT: + g_args.abort = 1; + break; + case ARGP_KEY_ARG: + g_args.arg_list = &state->argv[state->next - 1]; + g_args.arg_list_len = state->argc - state->next + 1; + state->next = state->argc; + break; + + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + static int queryDbImpl(TAOS *taos, char *command) { - int i; - TAOS_RES *res = NULL; - int32_t code = -1; + int i; + TAOS_RES *res = NULL; + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != res) { + taos_free_result(res); + res = NULL; + } - for (i = 0; i < 5; i++) { - if (NULL != res) { - taos_free_result(res); - res = NULL; + res = taos_query(taos, command); + code = taos_errno(res); + if (0 == code) { + break; + } } - res = taos_query(taos, command); - code = taos_errno(res); - if (0 == code) { - break; + if (code != 0) { + errorPrint("Failed to run <%s>, reason: %s\n", command, taos_errstr(res)); + taos_free_result(res); + //taos_close(taos); + return -1; } - } - if (code != 0) { - fprintf(stderr, "Failed to run <%s>, reason: %s\n", command, taos_errstr(res)); taos_free_result(res); - //taos_close(taos); - return -1; - } - - taos_free_result(res); - return 0; + return 0; } static void parse_args(int argc, char *argv[], SArguments *arguments) { - for (int i = 1; i < argc; i++) { - if ((strcmp(argv[i], "-S") == 0) - || (strcmp(argv[i], "-E") == 0)) { - if (argv[i+1]) { - char *tmp = strdup(argv[++i]); - - if (tmp) { - int64_t tmpEpoch; - if (strchr(tmp, ':') && strchr(tmp, '-')) { - if (TSDB_CODE_SUCCESS != taosParseTime( - tmp, &tmpEpoch, strlen(tmp), TSDB_TIME_PRECISION_MILLI, 0)) { - fprintf(stderr, "Input end time error!\n"); - free(tmp); - return; + for (int i = 1; i < argc; i++) { + if ((strcmp(argv[i], "-S") == 0) + || (strcmp(argv[i], "-E") == 0)) { + if (argv[i+1]) { + char *tmp = strdup(argv[++i]); + + if (tmp) { + int64_t tmpEpoch; + if (strchr(tmp, ':') && strchr(tmp, '-')) { + if (TSDB_CODE_SUCCESS != taosParseTime( + tmp, &tmpEpoch, strlen(tmp), TSDB_TIME_PRECISION_MILLI, 0)) { + errorPrint("Input %s, end time error!\n", tmp); + free(tmp); + return; + } + } else { + tmpEpoch = atoll(tmp); + } + + sprintf(argv[i], "%"PRId64"", tmpEpoch); + debugPrint("%s() LN%d, tmp is: %s, argv[%d]: %s\n", + __func__, __LINE__, tmp, i, argv[i]); + + free(tmp); + } else { + errorPrint("%s() LN%d, strdup() cannot allocate memory\n", __func__, __LINE__); + exit(-1); + } + } else { + errorPrint("%s need a valid value following!\n", argv[i]); + exit(-1); } - } else { - tmpEpoch = atoll(tmp); - } - - sprintf(argv[i], "%"PRId64"", tmpEpoch); - debugPrint("%s() LN%d, tmp is: %s, argv[%d]: %s\n", - __func__, __LINE__, tmp, i, argv[i]); - - free(tmp); - } else { - errorPrint("%s() LN%d, strdup() cannot allocate memory\n", __func__, __LINE__); - exit(-1); + } else if (strcmp(argv[i], "-g") == 0) { + g_args.debug_print = true; } - } else { - errorPrint("%s need a valid value following!\n", argv[i]); - exit(-1); - } - } else if (strcmp(argv[i], "-g") == 0) { - arguments->debug_print = true; } - } } int main(int argc, char *argv[]) { - /* Parse our arguments; every option seen by parse_opt will be - reflected in arguments. */ - if (argc > 2) - parse_args(argc, argv, &g_args); + int ret = 0; + /* Parse our arguments; every option seen by parse_opt will be + reflected in arguments. */ + if (argc > 2) + parse_args(argc, argv, &g_args); - argp_parse(&argp, argc, argv, 0, 0, &g_args); + argp_parse(&argp, argc, argv, 0, 0, &g_args); - if (g_args.abort) { - #ifndef _ALPINE - error(10, 0, "ABORTED"); - #else - abort(); - #endif - } + if (g_args.abort) { +#ifndef _ALPINE + error(10, 0, "ABORTED"); +#else + abort(); +#endif + } - printf("====== arguments config ======\n"); - { - printf("host: %s\n", g_args.host); - printf("user: %s\n", g_args.user); - printf("password: %s\n", g_args.password); - printf("port: %u\n", g_args.port); - printf("cversion: %s\n", g_args.cversion); - printf("mysqlFlag: %d\n", g_args.mysqlFlag); - printf("outpath: %s\n", g_args.outpath); - printf("inpath: %s\n", g_args.inpath); - printf("resultFile: %s\n", g_args.resultFile); - printf("encode: %s\n", g_args.encode); - printf("all_databases: %d\n", g_args.all_databases); - printf("databases: %d\n", g_args.databases); - printf("schemaonly: %d\n", g_args.schemaonly); - printf("with_property: %d\n", g_args.with_property); - printf("start_time: %" PRId64 "\n", g_args.start_time); - printf("end_time: %" PRId64 "\n", g_args.end_time); - printf("data_batch: %d\n", g_args.data_batch); - printf("max_sql_len: %d\n", g_args.max_sql_len); - printf("table_batch: %d\n", g_args.table_batch); - printf("thread_num: %d\n", g_args.thread_num); - printf("allow_sys: %d\n", g_args.allow_sys); - printf("abort: %d\n", g_args.abort); - printf("isDumpIn: %d\n", g_args.isDumpIn); - printf("arg_list_len: %d\n", g_args.arg_list_len); - printf("debug_print: %d\n", g_args.debug_print); - - for (int32_t i = 0; i < g_args.arg_list_len; i++) { - printf("arg_list[%d]: %s\n", i, g_args.arg_list[i]); + printf("====== arguments config ======\n"); + { + printf("host: %s\n", g_args.host); + printf("user: %s\n", g_args.user); + printf("password: %s\n", g_args.password); + printf("port: %u\n", g_args.port); + printf("cversion: %s\n", g_args.cversion); + printf("mysqlFlag: %d\n", g_args.mysqlFlag); + printf("outpath: %s\n", g_args.outpath); + printf("inpath: %s\n", g_args.inpath); + printf("resultFile: %s\n", g_args.resultFile); + printf("encode: %s\n", g_args.encode); + printf("all_databases: %s\n", g_args.all_databases?"true":"false"); + printf("databases: %d\n", g_args.databases); + printf("schemaonly: %s\n", g_args.schemaonly?"true":"false"); + printf("with_property: %s\n", g_args.with_property?"true":"false"); + printf("avro format: %s\n", g_args.avro?"true":"false"); + printf("start_time: %" PRId64 "\n", g_args.start_time); + printf("end_time: %" PRId64 "\n", g_args.end_time); + printf("data_batch: %d\n", g_args.data_batch); + printf("max_sql_len: %d\n", g_args.max_sql_len); + printf("table_batch: %d\n", g_args.table_batch); + printf("thread_num: %d\n", g_args.thread_num); + printf("allow_sys: %d\n", g_args.allow_sys); + printf("abort: %d\n", g_args.abort); + printf("isDumpIn: %d\n", g_args.isDumpIn); + printf("arg_list_len: %d\n", g_args.arg_list_len); + printf("debug_print: %d\n", g_args.debug_print); + + for (int32_t i = 0; i < g_args.arg_list_len; i++) { + printf("arg_list[%d]: %s\n", i, g_args.arg_list[i]); + } } - } - printf("==============================\n"); + printf("==============================\n"); - if (g_args.cversion[0] != 0){ - tstrncpy(version, g_args.cversion, 11); - } + if (g_args.cversion[0] != 0){ + tstrncpy(version, g_args.cversion, 11); + } - if (taosCheckParam(&g_args) < 0) { - exit(EXIT_FAILURE); - } + if (taosCheckParam(&g_args) < 0) { + exit(EXIT_FAILURE); + } - g_fpOfResult = fopen(g_args.resultFile, "a"); - if (NULL == g_fpOfResult) { - fprintf(stderr, "Failed to open %s for save result\n", g_args.resultFile); - return 1; - }; - - fprintf(g_fpOfResult, "#############################################################################\n"); - fprintf(g_fpOfResult, "============================== arguments config =============================\n"); - { - fprintf(g_fpOfResult, "host: %s\n", g_args.host); - fprintf(g_fpOfResult, "user: %s\n", g_args.user); - fprintf(g_fpOfResult, "password: %s\n", g_args.password); - fprintf(g_fpOfResult, "port: %u\n", g_args.port); - fprintf(g_fpOfResult, "cversion: %s\n", g_args.cversion); - fprintf(g_fpOfResult, "mysqlFlag: %d\n", g_args.mysqlFlag); - fprintf(g_fpOfResult, "outpath: %s\n", g_args.outpath); - fprintf(g_fpOfResult, "inpath: %s\n", g_args.inpath); - fprintf(g_fpOfResult, "resultFile: %s\n", g_args.resultFile); - fprintf(g_fpOfResult, "encode: %s\n", g_args.encode); - fprintf(g_fpOfResult, "all_databases: %s\n", g_args.all_databases?"true":"false"); - fprintf(g_fpOfResult, "databases: %d\n", g_args.databases); - fprintf(g_fpOfResult, "schemaonly: %s\n", g_args.schemaonly?"true":"false"); - fprintf(g_fpOfResult, "with_property: %s\n", g_args.with_property?"true":"false"); - fprintf(g_fpOfResult, "start_time: %" PRId64 "\n", g_args.start_time); - fprintf(g_fpOfResult, "end_time: %" PRId64 "\n", g_args.end_time); - fprintf(g_fpOfResult, "data_batch: %d\n", g_args.data_batch); - fprintf(g_fpOfResult, "max_sql_len: %d\n", g_args.max_sql_len); - fprintf(g_fpOfResult, "table_batch: %d\n", g_args.table_batch); - fprintf(g_fpOfResult, "thread_num: %d\n", g_args.thread_num); - fprintf(g_fpOfResult, "allow_sys: %d\n", g_args.allow_sys); - fprintf(g_fpOfResult, "abort: %d\n", g_args.abort); - fprintf(g_fpOfResult, "isDumpIn: %d\n", g_args.isDumpIn); - fprintf(g_fpOfResult, "arg_list_len: %d\n", g_args.arg_list_len); - - for (int32_t i = 0; i < g_args.arg_list_len; i++) { - fprintf(g_fpOfResult, "arg_list[%d]: %s\n", i, g_args.arg_list[i]); + g_fpOfResult = fopen(g_args.resultFile, "a"); + if (NULL == g_fpOfResult) { + errorPrint("Failed to open %s for save result\n", g_args.resultFile); + exit(-1); + }; + + fprintf(g_fpOfResult, "#############################################################################\n"); + fprintf(g_fpOfResult, "============================== arguments config =============================\n"); + { + fprintf(g_fpOfResult, "host: %s\n", g_args.host); + fprintf(g_fpOfResult, "user: %s\n", g_args.user); + fprintf(g_fpOfResult, "password: %s\n", g_args.password); + fprintf(g_fpOfResult, "port: %u\n", g_args.port); + fprintf(g_fpOfResult, "cversion: %s\n", g_args.cversion); + fprintf(g_fpOfResult, "mysqlFlag: %d\n", g_args.mysqlFlag); + fprintf(g_fpOfResult, "outpath: %s\n", g_args.outpath); + fprintf(g_fpOfResult, "inpath: %s\n", g_args.inpath); + fprintf(g_fpOfResult, "resultFile: %s\n", g_args.resultFile); + fprintf(g_fpOfResult, "encode: %s\n", g_args.encode); + fprintf(g_fpOfResult, "all_databases: %s\n", g_args.all_databases?"true":"false"); + fprintf(g_fpOfResult, "databases: %d\n", g_args.databases); + fprintf(g_fpOfResult, "schemaonly: %s\n", g_args.schemaonly?"true":"false"); + fprintf(g_fpOfResult, "with_property: %s\n", g_args.with_property?"true":"false"); + fprintf(g_fpOfResult, "avro format: %s\n", g_args.avro?"true":"false"); + fprintf(g_fpOfResult, "start_time: %" PRId64 "\n", g_args.start_time); + fprintf(g_fpOfResult, "end_time: %" PRId64 "\n", g_args.end_time); + fprintf(g_fpOfResult, "data_batch: %d\n", g_args.data_batch); + fprintf(g_fpOfResult, "max_sql_len: %d\n", g_args.max_sql_len); + fprintf(g_fpOfResult, "table_batch: %d\n", g_args.table_batch); + fprintf(g_fpOfResult, "thread_num: %d\n", g_args.thread_num); + fprintf(g_fpOfResult, "allow_sys: %d\n", g_args.allow_sys); + fprintf(g_fpOfResult, "abort: %d\n", g_args.abort); + fprintf(g_fpOfResult, "isDumpIn: %d\n", g_args.isDumpIn); + fprintf(g_fpOfResult, "arg_list_len: %d\n", g_args.arg_list_len); + + for (int32_t i = 0; i < g_args.arg_list_len; i++) { + fprintf(g_fpOfResult, "arg_list[%d]: %s\n", i, g_args.arg_list[i]); + } } - } - g_numOfCores = (int32_t)sysconf(_SC_NPROCESSORS_ONLN); - - time_t tTime = time(NULL); - struct tm tm = *localtime(&tTime); - - if (g_args.isDumpIn) { - fprintf(g_fpOfResult, "============================== DUMP IN ============================== \n"); - fprintf(g_fpOfResult, "# DumpIn start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - if (taosDumpIn(&g_args) < 0) { - fprintf(g_fpOfResult, "\n"); - fclose(g_fpOfResult); - return -1; - } - } else { - fprintf(g_fpOfResult, "============================== DUMP OUT ============================== \n"); - fprintf(g_fpOfResult, "# DumpOut start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - if (taosDumpOut(&g_args) < 0) { - fprintf(g_fpOfResult, "\n"); - fclose(g_fpOfResult); - return -1; - } - - fprintf(g_fpOfResult, "\n============================== TOTAL STATISTICS ============================== \n"); - fprintf(g_fpOfResult, "# total database count: %d\n", g_resultStatistics.totalDatabasesOfDumpOut); - fprintf(g_fpOfResult, "# total super table count: %d\n", g_resultStatistics.totalSuperTblsOfDumpOut); - fprintf(g_fpOfResult, "# total child table count: %"PRId64"\n", g_resultStatistics.totalChildTblsOfDumpOut); - fprintf(g_fpOfResult, "# total row count: %"PRId64"\n", g_resultStatistics.totalRowsOfDumpOut); - } + g_numOfCores = (int32_t)sysconf(_SC_NPROCESSORS_ONLN); + + time_t tTime = time(NULL); + struct tm tm = *localtime(&tTime); + + if (g_args.isDumpIn) { + fprintf(g_fpOfResult, "============================== DUMP IN ============================== \n"); + fprintf(g_fpOfResult, "# DumpIn start time: %d-%02d-%02d %02d:%02d:%02d\n", + tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + if (taosDumpIn() < 0) { + ret = -1; + } + } else { + fprintf(g_fpOfResult, "============================== DUMP OUT ============================== \n"); + fprintf(g_fpOfResult, "# DumpOut start time: %d-%02d-%02d %02d:%02d:%02d\n", + tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + if (taosDumpOut() < 0) { + ret = -1; + } else { + fprintf(g_fpOfResult, "\n============================== TOTAL STATISTICS ============================== \n"); + fprintf(g_fpOfResult, "# total database count: %d\n", + g_resultStatistics.totalDatabasesOfDumpOut); + fprintf(g_fpOfResult, "# total super table count: %d\n", + g_resultStatistics.totalSuperTblsOfDumpOut); + fprintf(g_fpOfResult, "# total child table count: %"PRId64"\n", + g_resultStatistics.totalChildTblsOfDumpOut); + fprintf(g_fpOfResult, "# total row count: %"PRId64"\n", + g_resultStatistics.totalRowsOfDumpOut); + } + } - fprintf(g_fpOfResult, "\n"); - fclose(g_fpOfResult); + fprintf(g_fpOfResult, "\n"); + fclose(g_fpOfResult); - return 0; + return ret; } static void taosFreeDbInfos() { - if (dbInfos == NULL) return; - for (int i = 0; i < 128; i++) tfree(dbInfos[i]); - tfree(dbInfos); + if (g_dbInfos == NULL) return; + for (int i = 0; i < 128; i++) tfree(g_dbInfos[i]); + tfree(g_dbInfos); } // check table is normal table or super table -static int taosGetTableRecordInfo(char *table, STableRecordInfo *pTableRecordInfo, TAOS *taosCon) { - TAOS_ROW row = NULL; - bool isSet = false; - TAOS_RES *result = NULL; +static int taosGetTableRecordInfo( + char *table, STableRecordInfo *pTableRecordInfo, TAOS *taosCon) { + TAOS_ROW row = NULL; + bool isSet = false; + TAOS_RES *result = NULL; + + memset(pTableRecordInfo, 0, sizeof(STableRecordInfo)); + + char* tempCommand = (char *)malloc(COMMAND_SIZE); + if (tempCommand == NULL) { + errorPrint("%s() LN%d, failed to allocate memory\n", + __func__, __LINE__); + return -1; + } - memset(pTableRecordInfo, 0, sizeof(STableRecordInfo)); + sprintf(tempCommand, "show tables like %s", table); - char* tempCommand = (char *)malloc(COMMAND_SIZE); - if (tempCommand == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return -1; - } + result = taos_query(taosCon, tempCommand); + int32_t code = taos_errno(result); - sprintf(tempCommand, "show tables like %s", table); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command %s\n", + __func__, __LINE__, tempCommand); + free(tempCommand); + taos_free_result(result); + return -1; + } - result = taos_query(taosCon, tempCommand); - int32_t code = taos_errno(result); + TAOS_FIELD *fields = taos_fetch_fields(result); + + while ((row = taos_fetch_row(result)) != NULL) { + isSet = true; + pTableRecordInfo->isMetric = false; + strncpy(pTableRecordInfo->tableRecord.name, + (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], + fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); + strncpy(pTableRecordInfo->tableRecord.metric, + (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], + fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); + break; + } - if (code != 0) { - fprintf(stderr, "failed to run command %s\n", tempCommand); - free(tempCommand); taos_free_result(result); - return -1; - } - - TAOS_FIELD *fields = taos_fetch_fields(result); + result = NULL; - while ((row = taos_fetch_row(result)) != NULL) { - isSet = true; - pTableRecordInfo->isMetric = false; - strncpy(pTableRecordInfo->tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], - fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); - strncpy(pTableRecordInfo->tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], - fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); - break; - } + if (isSet) { + free(tempCommand); + return 0; + } - taos_free_result(result); - result = NULL; + sprintf(tempCommand, "show stables like %s", table); - if (isSet) { - free(tempCommand); - return 0; - } + result = taos_query(taosCon, tempCommand); + code = taos_errno(result); - sprintf(tempCommand, "show stables like %s", table); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command %s\n", + __func__, __LINE__, tempCommand); + free(tempCommand); + taos_free_result(result); + return -1; + } - result = taos_query(taosCon, tempCommand); - code = taos_errno(result); + while ((row = taos_fetch_row(result)) != NULL) { + isSet = true; + pTableRecordInfo->isMetric = true; + tstrncpy(pTableRecordInfo->tableRecord.metric, table, + TSDB_TABLE_NAME_LEN); + break; + } - if (code != 0) { - errorPrint("%s() LN%d, failed to run command %s\n", - __func__, __LINE__, tempCommand); - free(tempCommand); taos_free_result(result); - return -1; - } - - while ((row = taos_fetch_row(result)) != NULL) { - isSet = true; - pTableRecordInfo->isMetric = true; - tstrncpy(pTableRecordInfo->tableRecord.metric, table, TSDB_TABLE_NAME_LEN); - break; - } + result = NULL; - taos_free_result(result); - result = NULL; - - if (isSet) { + if (isSet) { + free(tempCommand); + return 0; + } + errorPrint("%s() LN%d, invalid table/metric %s\n", + __func__, __LINE__, table); free(tempCommand); - return 0; - } - errorPrint("%s() LN%d, invalid table/metric %s\n", - __func__, __LINE__, table); - free(tempCommand); - return -1; + return -1; } static int32_t taosSaveAllNormalTableToTempFile(TAOS *taosCon, char*meter, char* metric, int* fd) { - STableRecord tableRecord; - - if (-1 == *fd) { - *fd = open(".tables.tmp.0", O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (*fd == -1) { - fprintf(stderr, "failed to open temp file: .tables.tmp.0\n"); - return -1; + STableRecord tableRecord; + + if (-1 == *fd) { + *fd = open(".tables.tmp.0", + O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (*fd == -1) { + errorPrint("%s() LN%d, failed to open temp file: .tables.tmp.0\n", + __func__, __LINE__); + return -1; + } } - } - memset(&tableRecord, 0, sizeof(STableRecord)); - tstrncpy(tableRecord.name, meter, TSDB_TABLE_NAME_LEN); - tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN); + memset(&tableRecord, 0, sizeof(STableRecord)); + tstrncpy(tableRecord.name, meter, TSDB_TABLE_NAME_LEN); + tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN); - taosWrite(*fd, &tableRecord, sizeof(STableRecord)); - return 0; + taosWrite(*fd, &tableRecord, sizeof(STableRecord)); + return 0; } +static int32_t taosSaveTableOfMetricToTempFile( + TAOS *taosCon, char* metric, + int32_t* totalNumOfThread) { + TAOS_ROW row; + int fd = -1; + STableRecord tableRecord; -static int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, - struct arguments *arguments, int32_t* totalNumOfThread) { - TAOS_ROW row; - int fd = -1; - STableRecord tableRecord; - - char* tmpCommand = (char *)malloc(COMMAND_SIZE); - if (tmpCommand == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return -1; - } + char* tmpCommand = (char *)malloc(COMMAND_SIZE); + if (tmpCommand == NULL) { + errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__); + return -1; + } - sprintf(tmpCommand, "select tbname from %s", metric); + sprintf(tmpCommand, "select tbname from %s", metric); - TAOS_RES *res = taos_query(taosCon, tmpCommand); - int32_t code = taos_errno(res); - if (code != 0) { - fprintf(stderr, "failed to run command %s\n", tmpCommand); + TAOS_RES *res = taos_query(taosCon, tmpCommand); + int32_t code = taos_errno(res); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command %s\n", + __func__, __LINE__, tmpCommand); + free(tmpCommand); + taos_free_result(res); + return -1; + } free(tmpCommand); - taos_free_result(res); - return -1; - } - free(tmpCommand); - - char tmpBuf[TSDB_FILENAME_LEN + 1]; - memset(tmpBuf, 0, TSDB_FILENAME_LEN); - sprintf(tmpBuf, ".select-tbname.tmp"); - fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); - taos_free_result(res); - return -1; - } - TAOS_FIELD *fields = taos_fetch_fields(res); + char tmpBuf[MAX_FILE_NAME_LEN]; + memset(tmpBuf, 0, MAX_FILE_NAME_LEN); + sprintf(tmpBuf, ".select-tbname.tmp"); + fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + errorPrint("%s() LN%d, failed to open temp file: %s\n", + __func__, __LINE__, tmpBuf); + taos_free_result(res); + return -1; + } + + TAOS_FIELD *fields = taos_fetch_fields(res); - int32_t numOfTable = 0; - while ((row = taos_fetch_row(res)) != NULL) { + int32_t numOfTable = 0; + while ((row = taos_fetch_row(res)) != NULL) { - memset(&tableRecord, 0, sizeof(STableRecord)); - tstrncpy(tableRecord.name, (char *)row[0], fields[0].bytes); - tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN); + memset(&tableRecord, 0, sizeof(STableRecord)); + tstrncpy(tableRecord.name, (char *)row[0], fields[0].bytes); + tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN); - taosWrite(fd, &tableRecord, sizeof(STableRecord)); - numOfTable++; - } - taos_free_result(res); - lseek(fd, 0, SEEK_SET); - - int maxThreads = arguments->thread_num; - int tableOfPerFile ; - if (numOfTable <= arguments->thread_num) { - tableOfPerFile = 1; - maxThreads = numOfTable; - } else { - tableOfPerFile = numOfTable / arguments->thread_num; - if (0 != numOfTable % arguments->thread_num) { - tableOfPerFile += 1; + taosWrite(fd, &tableRecord, sizeof(STableRecord)); + numOfTable++; } - } + taos_free_result(res); + lseek(fd, 0, SEEK_SET); - char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord)); - if (NULL == tblBuf){ - fprintf(stderr, "failed to calloc %" PRIzu "\n", tableOfPerFile * sizeof(STableRecord)); - close(fd); - return -1; - } + int maxThreads = g_args.thread_num; + int tableOfPerFile ; + if (numOfTable <= g_args.thread_num) { + tableOfPerFile = 1; + maxThreads = numOfTable; + } else { + tableOfPerFile = numOfTable / g_args.thread_num; + if (0 != numOfTable % g_args.thread_num) { + tableOfPerFile += 1; + } + } - int32_t numOfThread = *totalNumOfThread; - int subFd = -1; - for (; numOfThread < maxThreads; numOfThread++) { - memset(tmpBuf, 0, TSDB_FILENAME_LEN); - sprintf(tmpBuf, ".tables.tmp.%d", numOfThread); - subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (subFd == -1) { - fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); - for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { - sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); - (void)remove(tmpBuf); - } - sprintf(tmpBuf, ".select-tbname.tmp"); - (void)remove(tmpBuf); - free(tblBuf); - close(fd); - return -1; + char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord)); + if (NULL == tblBuf){ + errorPrint("%s() LN%d, failed to calloc %" PRIzu "\n", + __func__, __LINE__, tableOfPerFile * sizeof(STableRecord)); + close(fd); + return -1; } - // read tableOfPerFile for fd, write to subFd - ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord)); - if (readLen <= 0) { - close(subFd); - break; + int32_t numOfThread = *totalNumOfThread; + int subFd = -1; + for (; numOfThread < maxThreads; numOfThread++) { + memset(tmpBuf, 0, MAX_FILE_NAME_LEN); + sprintf(tmpBuf, ".tables.tmp.%d", numOfThread); + subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (subFd == -1) { + errorPrint("%s() LN%d, failed to open temp file: %s\n", + __func__, __LINE__, tmpBuf); + for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { + sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); + (void)remove(tmpBuf); + } + sprintf(tmpBuf, ".select-tbname.tmp"); + (void)remove(tmpBuf); + free(tblBuf); + close(fd); + return -1; + } + + // read tableOfPerFile for fd, write to subFd + ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord)); + if (readLen <= 0) { + close(subFd); + break; + } + taosWrite(subFd, tblBuf, readLen); + close(subFd); } - taosWrite(subFd, tblBuf, readLen); - close(subFd); - } - sprintf(tmpBuf, ".select-tbname.tmp"); - (void)remove(tmpBuf); + sprintf(tmpBuf, ".select-tbname.tmp"); + (void)remove(tmpBuf); - if (fd >= 0) { - close(fd); - fd = -1; - } + if (fd >= 0) { + close(fd); + fd = -1; + } - *totalNumOfThread = numOfThread; + *totalNumOfThread = numOfThread; - free(tblBuf); - return 0; + free(tblBuf); + return 0; } -static int taosDumpOut(struct arguments *arguments) { - TAOS *taos = NULL; - TAOS_RES *result = NULL; - char *command = NULL; - - TAOS_ROW row; - FILE *fp = NULL; - int32_t count = 0; - STableRecordInfo tableRecordInfo; - - char tmpBuf[TSDB_FILENAME_LEN+9] = {0}; - if (arguments->outpath[0] != 0) { - sprintf(tmpBuf, "%s/dbs.sql", arguments->outpath); - } else { - sprintf(tmpBuf, "dbs.sql"); - } +static int taosDumpOut() { + TAOS *taos = NULL; + TAOS_RES *result = NULL; + char *command = NULL; - fp = fopen(tmpBuf, "w"); - if (fp == NULL) { - fprintf(stderr, "failed to open file %s\n", tmpBuf); - return -1; - } + TAOS_ROW row; + FILE *fp = NULL; + int32_t count = 0; + STableRecordInfo tableRecordInfo; - dbInfos = (SDbInfo **)calloc(128, sizeof(SDbInfo *)); - if (dbInfos == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - goto _exit_failure; - } + char tmpBuf[4096] = {0}; + if (g_args.outpath[0] != 0) { + sprintf(tmpBuf, "%s/dbs.sql", g_args.outpath); + } else { + sprintf(tmpBuf, "dbs.sql"); + } - command = (char *)malloc(COMMAND_SIZE); - if (command == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - goto _exit_failure; - } + fp = fopen(tmpBuf, "w"); + if (fp == NULL) { + errorPrint("%s() LN%d, failed to open file %s\n", + __func__, __LINE__, tmpBuf); + return -1; + } - /* Connect to server */ - taos = taos_connect(arguments->host, arguments->user, arguments->password, NULL, arguments->port); - if (taos == NULL) { - fprintf(stderr, "failed to connect to TDengine server\n"); - goto _exit_failure; - } + g_dbInfos = (SDbInfo **)calloc(128, sizeof(SDbInfo *)); + if (g_dbInfos == NULL) { + errorPrint("%s() LN%d, failed to allocate memory\n", + __func__, __LINE__); + goto _exit_failure; + } - /* --------------------------------- Main Code -------------------------------- */ - /* if (arguments->databases || arguments->all_databases) { // dump part of databases or all databases */ - /* */ - taosDumpCharset(fp); + command = (char *)malloc(COMMAND_SIZE); + if (command == NULL) { + errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__); + goto _exit_failure; + } - sprintf(command, "show databases"); - result = taos_query(taos, command); - int32_t code = taos_errno(result); + /* Connect to server */ + taos = taos_connect(g_args.host, g_args.user, g_args.password, + NULL, g_args.port); + if (taos == NULL) { + errorPrint("Failed to connect to TDengine server %s\n", g_args.host); + goto _exit_failure; + } - if (code != 0) { - errorPrint("%s() LN%d, failed to run command: %s, reason: %s\n", - __func__, __LINE__, command, taos_errstr(result)); - goto _exit_failure; - } + /* --------------------------------- Main Code -------------------------------- */ + /* if (g_args.databases || g_args.all_databases) { // dump part of databases or all databases */ + /* */ + taosDumpCharset(fp); - TAOS_FIELD *fields = taos_fetch_fields(result); + sprintf(command, "show databases"); + result = taos_query(taos, command); + int32_t code = taos_errno(result); - while ((row = taos_fetch_row(result)) != NULL) { - // sys database name : 'log', but subsequent version changed to 'log' - if ((strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", - fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) - && (!arguments->allow_sys)) { - continue; - } + if (code != 0) { + errorPrint("%s() LN%d, failed to run command: %s, reason: %s\n", + __func__, __LINE__, command, taos_errstr(result)); + goto _exit_failure; + } + + TAOS_FIELD *fields = taos_fetch_fields(result); - if (arguments->databases) { // input multi dbs - for (int i = 0; arguments->arg_list[i]; i++) { - if (strncasecmp(arguments->arg_list[i], (char *)row[TSDB_SHOW_DB_NAME_INDEX], + while ((row = taos_fetch_row(result)) != NULL) { + // sys database name : 'log', but subsequent version changed to 'log' + if ((strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) - goto _dump_db_point; - } - continue; - } else if (!arguments->all_databases) { // only input one db - if (strncasecmp(arguments->arg_list[0], (char *)row[TSDB_SHOW_DB_NAME_INDEX], - fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) - goto _dump_db_point; - else - continue; - } + && (!g_args.allow_sys)) { + continue; + } + + if (g_args.databases) { // input multi dbs + for (int i = 0; g_args.arg_list[i]; i++) { + if (strncasecmp(g_args.arg_list[i], + (char *)row[TSDB_SHOW_DB_NAME_INDEX], + fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) + goto _dump_db_point; + } + continue; + } else if (!g_args.all_databases) { // only input one db + if (strncasecmp(g_args.arg_list[0], + (char *)row[TSDB_SHOW_DB_NAME_INDEX], + fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) + goto _dump_db_point; + else + continue; + } _dump_db_point: - dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo)); - if (dbInfos[count] == NULL) { - errorPrint("%s() LN%d, failed to allocate %"PRIu64" memory\n", - __func__, __LINE__, (uint64_t)(sizeof(SDbInfo))); - goto _exit_failure; - } + g_dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo)); + if (g_dbInfos[count] == NULL) { + errorPrint("%s() LN%d, failed to allocate %"PRIu64" memory\n", + __func__, __LINE__, (uint64_t)sizeof(SDbInfo)); + goto _exit_failure; + } - strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], - fields[TSDB_SHOW_DB_NAME_INDEX].bytes); - if (arguments->with_property) { - dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); - dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); - dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); - dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); - dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); - - strncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], - fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); - //dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]); - //dbInfos[count]->daysToKeep1; - //dbInfos[count]->daysToKeep2; - dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]); - dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]); - dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]); - dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]); - dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]); - dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]); - dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); - dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); - - strncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], - fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); - //dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]); - dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); - } - count++; - - if (arguments->databases) { - if (count > arguments->arg_list_len) break; - - } else if (!arguments->all_databases) { - if (count >= 1) break; - } - } + strncpy(g_dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], + fields[TSDB_SHOW_DB_NAME_INDEX].bytes); + if (g_args.with_property) { + g_dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); + g_dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); + g_dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); + g_dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); + g_dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); + + strncpy(g_dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], + fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); + //g_dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]); + //g_dbInfos[count]->daysToKeep1; + //g_dbInfos[count]->daysToKeep2; + g_dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]); + g_dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]); + g_dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]); + g_dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]); + g_dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]); + g_dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]); + g_dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); + g_dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); + + strncpy(g_dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], + fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); + //g_dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]); + g_dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); + } + count++; - if (count == 0) { - fprintf(stderr, "No databases valid to dump\n"); - goto _exit_failure; - } + if (g_args.databases) { + if (count > g_args.arg_list_len) break; - if (arguments->databases || arguments->all_databases) { // case: taosdump --databases dbx dby ... OR taosdump --all-databases - for (int i = 0; i < count; i++) { - taosDumpDb(dbInfos[i], arguments, fp, taos); + } else if (!g_args.all_databases) { + if (count >= 1) break; + } } - } else { - if (arguments->arg_list_len == 1) { // case: taosdump - taosDumpDb(dbInfos[0], arguments, fp, taos); - } else { // case: taosdump tablex tabley ... - taosDumpCreateDbClause(dbInfos[0], arguments->with_property, fp); - fprintf(g_fpOfResult, "\n#### database: %s\n", dbInfos[0]->name); - g_resultStatistics.totalDatabasesOfDumpOut++; - - sprintf(command, "use %s", dbInfos[0]->name); - result = taos_query(taos, command); - code = taos_errno(result); - if (code != 0) { - fprintf(stderr, "invalid database %s\n", dbInfos[0]->name); + if (count == 0) { + errorPrint("%d databases valid to dump\n", count); goto _exit_failure; - } - - fprintf(fp, "USE %s;\n\n", dbInfos[0]->name); + } - int32_t totalNumOfThread = 1; // 0: all normal talbe into .tables.tmp.0 - int normalTblFd = -1; - int32_t retCode; - int superTblCnt = 0 ; - for (int i = 1; arguments->arg_list[i]; i++) { - if (taosGetTableRecordInfo(arguments->arg_list[i], &tableRecordInfo, taos) < 0) { - fprintf(stderr, "input the invalide table %s\n", arguments->arg_list[i]); - continue; + if (g_args.databases || g_args.all_databases) { // case: taosdump --databases dbx dby ... OR taosdump --all-databases + for (int i = 0; i < count; i++) { + taosDumpDb(g_dbInfos[i], fp, taos); } - - if (tableRecordInfo.isMetric) { // dump all table of this metric - int ret = taosDumpStable( - tableRecordInfo.tableRecord.metric, - fp, taos, dbInfos[0]->name); - if (0 == ret) { - superTblCnt++; - } - retCode = taosSaveTableOfMetricToTempFile( - taos, tableRecordInfo.tableRecord.metric, - arguments, &totalNumOfThread); - } else { - if (tableRecordInfo.tableRecord.metric[0] != '\0') { // dump this sub table and it's metric - int ret = taosDumpStable(tableRecordInfo.tableRecord.metric, - fp, taos, dbInfos[0]->name); - if (0 == ret) { - superTblCnt++; + } else { + if (g_args.arg_list_len == 1) { // case: taosdump + taosDumpDb(g_dbInfos[0], fp, taos); + } else { // case: taosdump tablex tabley ... + taosDumpCreateDbClause(g_dbInfos[0], g_args.with_property, fp); + fprintf(g_fpOfResult, "\n#### database: %s\n", + g_dbInfos[0]->name); + g_resultStatistics.totalDatabasesOfDumpOut++; + + sprintf(command, "use %s", g_dbInfos[0]->name); + + result = taos_query(taos, command); + code = taos_errno(result); + if (code != 0) { + errorPrint("invalid database %s\n", g_dbInfos[0]->name); + goto _exit_failure; } - } - retCode = taosSaveAllNormalTableToTempFile( - taos, tableRecordInfo.tableRecord.name, - tableRecordInfo.tableRecord.metric, &normalTblFd); - } - if (retCode < 0) { - if (-1 != normalTblFd){ - taosClose(normalTblFd); - } - goto _clean_tmp_file; - } - } + fprintf(fp, "USE %s;\n\n", g_dbInfos[0]->name); + + int32_t totalNumOfThread = 1; // 0: all normal talbe into .tables.tmp.0 + int normalTblFd = -1; + int32_t retCode; + int superTblCnt = 0 ; + for (int i = 1; g_args.arg_list[i]; i++) { + if (taosGetTableRecordInfo(g_args.arg_list[i], + &tableRecordInfo, taos) < 0) { + errorPrint("input the invalide table %s\n", + g_args.arg_list[i]); + continue; + } + + if (tableRecordInfo.isMetric) { // dump all table of this metric + int ret = taosDumpStable( + tableRecordInfo.tableRecord.metric, + fp, taos, g_dbInfos[0]->name); + if (0 == ret) { + superTblCnt++; + } + retCode = taosSaveTableOfMetricToTempFile( + taos, tableRecordInfo.tableRecord.metric, + &totalNumOfThread); + } else { + if (tableRecordInfo.tableRecord.metric[0] != '\0') { // dump this sub table and it's metric + int ret = taosDumpStable( + tableRecordInfo.tableRecord.metric, + fp, taos, g_dbInfos[0]->name); + if (0 == ret) { + superTblCnt++; + } + } + retCode = taosSaveAllNormalTableToTempFile( + taos, tableRecordInfo.tableRecord.name, + tableRecordInfo.tableRecord.metric, &normalTblFd); + } + + if (retCode < 0) { + if (-1 != normalTblFd){ + taosClose(normalTblFd); + } + goto _clean_tmp_file; + } + } - // TODO: save dump super table into result_output.txt - fprintf(g_fpOfResult, "# super table counter: %d\n", superTblCnt); - g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; + // TODO: save dump super table into result_output.txt + fprintf(g_fpOfResult, "# super table counter: %d\n", + superTblCnt); + g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; - if (-1 != normalTblFd){ - taosClose(normalTblFd); - } + if (-1 != normalTblFd){ + taosClose(normalTblFd); + } - // start multi threads to dumpout - taosStartDumpOutWorkThreads(taos, arguments, totalNumOfThread, dbInfos[0]->name); + // start multi threads to dumpout + taosStartDumpOutWorkThreads(totalNumOfThread, + g_dbInfos[0]->name); - char tmpFileName[TSDB_FILENAME_LEN + 1]; - _clean_tmp_file: - for (int loopCnt = 0; loopCnt < totalNumOfThread; loopCnt++) { - sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); - remove(tmpFileName); - } + char tmpFileName[MAX_FILE_NAME_LEN]; +_clean_tmp_file: + for (int loopCnt = 0; loopCnt < totalNumOfThread; loopCnt++) { + sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); + remove(tmpFileName); + } + } } - } - /* Close the handle and return */ - fclose(fp); - taos_close(taos); - taos_free_result(result); - tfree(command); - taosFreeDbInfos(); - fprintf(stderr, "dump out rows: %" PRId64 "\n", totalDumpOutRows); - return 0; + /* Close the handle and return */ + fclose(fp); + taos_close(taos); + taos_free_result(result); + tfree(command); + taosFreeDbInfos(); + fprintf(stderr, "dump out rows: %" PRId64 "\n", g_totalDumpOutRows); + return 0; _exit_failure: - fclose(fp); - taos_close(taos); - taos_free_result(result); - tfree(command); - taosFreeDbInfos(); - fprintf(stderr, "dump out rows: %" PRId64 "\n", totalDumpOutRows); - return -1; + fclose(fp); + taos_close(taos); + taos_free_result(result); + tfree(command); + taosFreeDbInfos(); + errorPrint("dump out rows: %" PRId64 "\n", g_totalDumpOutRows); + return -1; } static int taosGetTableDes( char* dbName, char *table, STableDef *tableDes, TAOS* taosCon, bool isSuperTable) { - TAOS_ROW row = NULL; - TAOS_RES* res = NULL; - int count = 0; + TAOS_ROW row = NULL; + TAOS_RES* res = NULL; + int count = 0; - char sqlstr[COMMAND_SIZE]; - sprintf(sqlstr, "describe %s.%s;", dbName, table); + char sqlstr[COMMAND_SIZE]; + sprintf(sqlstr, "describe %s.%s;", dbName, table); - res = taos_query(taosCon, sqlstr); - int32_t code = taos_errno(res); - if (code != 0) { - errorPrint("%s() LN%d, failed to run command <%s>, reason:%s\n", - __func__, __LINE__, sqlstr, taos_errstr(res)); - taos_free_result(res); - return -1; - } - - TAOS_FIELD *fields = taos_fetch_fields(res); - - tstrncpy(tableDes->name, table, TSDB_TABLE_NAME_LEN); - while ((row = taos_fetch_row(res)) != NULL) { - strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], - fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); - strncpy(tableDes->cols[count].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); - tableDes->cols[count].length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - strncpy(tableDes->cols[count].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], - fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); - - count++; - } + res = taos_query(taosCon, sqlstr); + int32_t code = taos_errno(res); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command <%s>, reason:%s\n", + __func__, __LINE__, sqlstr, taos_errstr(res)); + taos_free_result(res); + return -1; + } - taos_free_result(res); - res = NULL; + TAOS_FIELD *fields = taos_fetch_fields(res); + + tstrncpy(tableDes->name, table, TSDB_TABLE_NAME_LEN); + while ((row = taos_fetch_row(res)) != NULL) { + strncpy(tableDes->cols[count].field, + (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], + fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + strncpy(tableDes->cols[count].type, + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); + tableDes->cols[count].length = + *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + strncpy(tableDes->cols[count].note, + (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + + count++; + } - if (isSuperTable) { - return count; - } + taos_free_result(res); + res = NULL; - // if chidl-table have tag, using select tagName from table to get tagValue - for (int i = 0 ; i < count; i++) { - if (strcmp(tableDes->cols[i].note, "TAG") != 0) continue; + if (isSuperTable) { + return count; + } + // if chidl-table have tag, using select tagName from table to get tagValue + for (int i = 0 ; i < count; i++) { + if (strcmp(tableDes->cols[i].note, "TAG") != 0) continue; - sprintf(sqlstr, "select %s from %s.%s", tableDes->cols[i].field, dbName, table); - res = taos_query(taosCon, sqlstr); - code = taos_errno(res); - if (code != 0) { - errorPrint("%s() LN%d, failed to run command <%s>, reason:%s\n", - __func__, __LINE__, sqlstr, taos_errstr(res)); - taos_free_result(res); - return -1; - } + sprintf(sqlstr, "select %s from %s.%s", + tableDes->cols[i].field, dbName, table); - fields = taos_fetch_fields(res); + res = taos_query(taosCon, sqlstr); + code = taos_errno(res); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command <%s>, reason:%s\n", + __func__, __LINE__, sqlstr, taos_errstr(res)); + taos_free_result(res); + return -1; + } - row = taos_fetch_row(res); - if (NULL == row) { - errorPrint("%s() LN%d, fetch failed to run command <%s>, reason:%s\n", - __func__, __LINE__, sqlstr, taos_errstr(res)); - taos_free_result(res); - return -1; - } + fields = taos_fetch_fields(res); - if (row[0] == NULL) { - sprintf(tableDes->cols[i].note, "%s", "NULL"); - taos_free_result(res); - res = NULL; - continue; - } + row = taos_fetch_row(res); + if (NULL == row) { + errorPrint("%s() LN%d, fetch failed to run command <%s>, reason:%s\n", + __func__, __LINE__, sqlstr, taos_errstr(res)); + taos_free_result(res); + return -1; + } - int32_t* length = taos_fetch_lengths(res); + if (row[0] == NULL) { + sprintf(tableDes->cols[i].note, "%s", "NULL"); + taos_free_result(res); + res = NULL; + continue; + } - //int32_t* length = taos_fetch_lengths(tmpResult); - switch (fields[0].type) { - case TSDB_DATA_TYPE_BOOL: - sprintf(tableDes->cols[i].note, "%d", ((((int32_t)(*((char *)row[0]))) == 1) ? 1 : 0)); - break; - case TSDB_DATA_TYPE_TINYINT: - sprintf(tableDes->cols[i].note, "%d", *((int8_t *)row[0])); - break; - case TSDB_DATA_TYPE_SMALLINT: - sprintf(tableDes->cols[i].note, "%d", *((int16_t *)row[0])); - break; - case TSDB_DATA_TYPE_INT: - sprintf(tableDes->cols[i].note, "%d", *((int32_t *)row[0])); - break; - case TSDB_DATA_TYPE_BIGINT: - sprintf(tableDes->cols[i].note, "%" PRId64 "", *((int64_t *)row[0])); - break; - case TSDB_DATA_TYPE_FLOAT: - sprintf(tableDes->cols[i].note, "%f", GET_FLOAT_VAL(row[0])); - break; - case TSDB_DATA_TYPE_DOUBLE: - sprintf(tableDes->cols[i].note, "%f", GET_DOUBLE_VAL(row[0])); - break; - case TSDB_DATA_TYPE_BINARY: { - memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note)); - tableDes->cols[i].note[0] = '\''; - char tbuf[COL_NOTE_LEN]; - converStringToReadable((char *)row[0], length[0], tbuf, COL_NOTE_LEN); - char* pstr = stpcpy(&(tableDes->cols[i].note[1]), tbuf); - *(pstr++) = '\''; - break; - } - case TSDB_DATA_TYPE_NCHAR: { - memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note)); - char tbuf[COL_NOTE_LEN-2]; // need reserve 2 bytes for ' ' - convertNCharToReadable((char *)row[0], length[0], tbuf, COL_NOTE_LEN); - sprintf(tableDes->cols[i].note, "\'%s\'", tbuf); - break; - } - case TSDB_DATA_TYPE_TIMESTAMP: - sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]); - #if 0 - if (!arguments->mysqlFlag) { - sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]); - } else { - char buf[64] = "\0"; - int64_t ts = *((int64_t *)row[0]); - time_t tt = (time_t)(ts / 1000); - struct tm *ptm = localtime(&tt); - strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm); - sprintf(tableDes->cols[i].note, "\'%s.%03d\'", buf, (int)(ts % 1000)); + int32_t* length = taos_fetch_lengths(res); + + //int32_t* length = taos_fetch_lengths(tmpResult); + switch (fields[0].type) { + case TSDB_DATA_TYPE_BOOL: + sprintf(tableDes->cols[i].note, "%d", + ((((int32_t)(*((char *)row[0]))) == 1) ? 1 : 0)); + break; + case TSDB_DATA_TYPE_TINYINT: + sprintf(tableDes->cols[i].note, "%d", *((int8_t *)row[0])); + break; + case TSDB_DATA_TYPE_SMALLINT: + sprintf(tableDes->cols[i].note, "%d", *((int16_t *)row[0])); + break; + case TSDB_DATA_TYPE_INT: + sprintf(tableDes->cols[i].note, "%d", *((int32_t *)row[0])); + break; + case TSDB_DATA_TYPE_BIGINT: + sprintf(tableDes->cols[i].note, "%" PRId64 "", *((int64_t *)row[0])); + break; + case TSDB_DATA_TYPE_FLOAT: + sprintf(tableDes->cols[i].note, "%f", GET_FLOAT_VAL(row[0])); + break; + case TSDB_DATA_TYPE_DOUBLE: + sprintf(tableDes->cols[i].note, "%f", GET_DOUBLE_VAL(row[0])); + break; + case TSDB_DATA_TYPE_BINARY: + { + memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note)); + tableDes->cols[i].note[0] = '\''; + char tbuf[COL_NOTE_LEN]; + converStringToReadable((char *)row[0], length[0], tbuf, COL_NOTE_LEN); + char* pstr = stpcpy(&(tableDes->cols[i].note[1]), tbuf); + *(pstr++) = '\''; + break; + } + case TSDB_DATA_TYPE_NCHAR: + { + memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note)); + char tbuf[COL_NOTE_LEN-2]; // need reserve 2 bytes for ' ' + convertNCharToReadable((char *)row[0], length[0], tbuf, COL_NOTE_LEN); + sprintf(tableDes->cols[i].note, "\'%s\'", tbuf); + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]); +#if 0 + if (!g_args.mysqlFlag) { + sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]); + } else { + char buf[64] = "\0"; + int64_t ts = *((int64_t *)row[0]); + time_t tt = (time_t)(ts / 1000); + struct tm *ptm = localtime(&tt); + strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm); + sprintf(tableDes->cols[i].note, "\'%s.%03d\'", buf, (int)(ts % 1000)); + } +#endif + break; + default: + break; } - #endif - break; - default: - break; + + taos_free_result(res); + res = NULL; } - taos_free_result(res); - res = NULL; - } + return count; +} - return count; +static int convertSchemaToAvroSchema(STableDef *tableDes, char **avroSchema) +{ + errorPrint("%s() LN%d TODO: covert table schema to avro schema\n", + __func__, __LINE__); + return 0; } static int32_t taosDumpTable( - char *table, char *metric, struct arguments *arguments, + char *table, char *metric, FILE *fp, TAOS* taosCon, char* dbName) { - int count = 0; + int count = 0; - STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) - + sizeof(SColDes) * TSDB_MAX_COLUMNS); + STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + + sizeof(SColDes) * TSDB_MAX_COLUMNS); - if (metric != NULL && metric[0] != '\0') { // dump table schema which is created by using super table - /* - count = taosGetTableDes(metric, tableDes, taosCon); + if (metric != NULL && metric[0] != '\0') { // dump table schema which is created by using super table + /* + count = taosGetTableDes(metric, tableDes, taosCon); - if (count < 0) { - free(tableDes); - return -1; - } + if (count < 0) { + free(tableDes); + return -1; + } - taosDumpCreateTableClause(tableDes, count, fp); + taosDumpCreateTableClause(tableDes, count, fp); - memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); - */ + memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); + */ - count = taosGetTableDes(dbName, table, tableDes, taosCon, false); + count = taosGetTableDes(dbName, table, tableDes, taosCon, false); - if (count < 0) { - free(tableDes); - return -1; - } + if (count < 0) { + free(tableDes); + return -1; + } - // create child-table using super-table - taosDumpCreateMTableClause(tableDes, metric, count, fp, dbName); + // create child-table using super-table + taosDumpCreateMTableClause(tableDes, metric, count, fp, dbName); - } else { // dump table definition - count = taosGetTableDes(dbName, table, tableDes, taosCon, false); + } else { // dump table definition + count = taosGetTableDes(dbName, table, tableDes, taosCon, false); - if (count < 0) { - free(tableDes); - return -1; + if (count < 0) { + free(tableDes); + return -1; + } + + // create normal-table or super-table + taosDumpCreateTableClause(tableDes, count, fp, dbName); } - // create normal-table or super-table - taosDumpCreateTableClause(tableDes, count, fp, dbName); - } + char *jsonAvroSchema = NULL; + if (g_args.avro) { + convertSchemaToAvroSchema(tableDes, &jsonAvroSchema); + } + + free(tableDes); - free(tableDes); + int32_t ret = 0; + if (!g_args.schemaonly) { + ret = taosDumpTableData(fp, table, taosCon, dbName, + jsonAvroSchema); + } - return taosDumpTableData(fp, table, arguments, taosCon, dbName); + return ret; } -static void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { - char sqlstr[TSDB_MAX_SQL_LEN] = {0}; - - char *pstr = sqlstr; - pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s ", dbInfo->name); - if (isDumpProperty) { - pstr += sprintf(pstr, - "REPLICA %d QUORUM %d DAYS %d KEEP %s CACHE %d BLOCKS %d MINROWS %d MAXROWS %d FSYNC %d CACHELAST %d COMP %d PRECISION '%s' UPDATE %d", - dbInfo->replica, dbInfo->quorum, dbInfo->days, dbInfo->keeplist, dbInfo->cache, - dbInfo->blocks, dbInfo->minrows, dbInfo->maxrows, dbInfo->fsync, dbInfo->cachelast, - dbInfo->comp, dbInfo->precision, dbInfo->update); - } +static void taosDumpCreateDbClause( + SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { + char sqlstr[TSDB_MAX_SQL_LEN] = {0}; + + char *pstr = sqlstr; + pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s ", dbInfo->name); + if (isDumpProperty) { + pstr += sprintf(pstr, + "REPLICA %d QUORUM %d DAYS %d KEEP %s CACHE %d BLOCKS %d MINROWS %d MAXROWS %d FSYNC %d CACHELAST %d COMP %d PRECISION '%s' UPDATE %d", + dbInfo->replica, dbInfo->quorum, dbInfo->days, + dbInfo->keeplist, + dbInfo->cache, + dbInfo->blocks, dbInfo->minrows, dbInfo->maxrows, + dbInfo->fsync, + dbInfo->cachelast, + dbInfo->comp, dbInfo->precision, dbInfo->update); + } - pstr += sprintf(pstr, ";"); - fprintf(fp, "%s\n\n", sqlstr); + pstr += sprintf(pstr, ";"); + fprintf(fp, "%s\n\n", sqlstr); } static void* taosDumpOutWorkThreadFp(void *arg) { - SThreadParaObj *pThread = (SThreadParaObj*)arg; - STableRecord tableRecord; - int fd; - - char tmpBuf[TSDB_FILENAME_LEN*4] = {0}; - sprintf(tmpBuf, ".tables.tmp.%d", pThread->threadIndex); - fd = open(tmpBuf, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - fprintf(stderr, "taosDumpTableFp() failed to open temp file: %s\n", tmpBuf); - return NULL; - } - - FILE *fp = NULL; - memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); - - if (g_args.outpath[0] != 0) { - sprintf(tmpBuf, "%s/%s.tables.%d.sql", g_args.outpath, pThread->dbName, pThread->threadIndex); - } else { - sprintf(tmpBuf, "%s.tables.%d.sql", pThread->dbName, pThread->threadIndex); - } + SThreadParaObj *pThread = (SThreadParaObj*)arg; + STableRecord tableRecord; + int fd; + + char tmpBuf[4096] = {0}; + sprintf(tmpBuf, ".tables.tmp.%d", pThread->threadIndex); + fd = open(tmpBuf, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + errorPrint("%s() LN%d, failed to open temp file: %s\n", + __func__, __LINE__, tmpBuf); + return NULL; + } - fp = fopen(tmpBuf, "w"); - if (fp == NULL) { - fprintf(stderr, "failed to open file %s\n", tmpBuf); - close(fd); - return NULL; - } + FILE *fp = NULL; + memset(tmpBuf, 0, 4096); - memset(tmpBuf, 0, TSDB_FILENAME_LEN); - sprintf(tmpBuf, "use %s", pThread->dbName); + if (g_args.outpath[0] != 0) { + sprintf(tmpBuf, "%s/%s.tables.%d.sql", + g_args.outpath, pThread->dbName, pThread->threadIndex); + } else { + sprintf(tmpBuf, "%s.tables.%d.sql", + pThread->dbName, pThread->threadIndex); + } - TAOS_RES* tmpResult = taos_query(pThread->taosCon, tmpBuf); - int32_t code = taos_errno(tmpResult); - if (code != 0) { - errorPrint("%s() LN%d, invalid database %s. reason: %s\n", - __func__, __LINE__, pThread->dbName, taos_errstr(tmpResult)); - taos_free_result(tmpResult); - fclose(fp); - close(fd); - return NULL; - } + fp = fopen(tmpBuf, "w"); + if (fp == NULL) { + errorPrint("%s() LN%d, failed to open file %s\n", + __func__, __LINE__, tmpBuf); + close(fd); + return NULL; + } - int fileNameIndex = 1; - int tablesInOneFile = 0; - int64_t lastRowsPrint = 5000000; - fprintf(fp, "USE %s;\n\n", pThread->dbName); - while (1) { - ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); - if (readLen <= 0) break; - - int ret = taosDumpTable( - tableRecord.name, tableRecord.metric, &g_args, - fp, pThread->taosCon, pThread->dbName); - if (ret >= 0) { - // TODO: sum table count and table rows by self - pThread->tablesOfDumpOut++; - pThread->rowsOfDumpOut += ret; - - if (pThread->rowsOfDumpOut >= lastRowsPrint) { - printf(" %"PRId64 " rows already be dumpout from database %s\n", - pThread->rowsOfDumpOut, pThread->dbName); - lastRowsPrint += 5000000; - } + memset(tmpBuf, 0, 4096); + sprintf(tmpBuf, "use %s", pThread->dbName); - tablesInOneFile++; - if (tablesInOneFile >= g_args.table_batch) { + TAOS_RES* tmpResult = taos_query(pThread->taosCon, tmpBuf); + int32_t code = taos_errno(tmpResult); + if (code != 0) { + errorPrint("%s() LN%d, invalid database %s. reason: %s\n", + __func__, __LINE__, pThread->dbName, taos_errstr(tmpResult)); + taos_free_result(tmpResult); fclose(fp); - tablesInOneFile = 0; + close(fd); + return NULL; + } - memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); - if (g_args.outpath[0] != 0) { - sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", - g_args.outpath, pThread->dbName, - pThread->threadIndex, fileNameIndex); - } else { - sprintf(tmpBuf, "%s.tables.%d-%d.sql", - pThread->dbName, pThread->threadIndex, fileNameIndex); - } - fileNameIndex++; - - fp = fopen(tmpBuf, "w"); - if (fp == NULL) { - fprintf(stderr, "failed to open file %s\n", tmpBuf); - close(fd); - taos_free_result(tmpResult); - return NULL; +#if 0 + int fileNameIndex = 1; + int tablesInOneFile = 0; +#endif + int64_t lastRowsPrint = 5000000; + fprintf(fp, "USE %s;\n\n", pThread->dbName); + while (1) { + ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); + if (readLen <= 0) break; + + int ret = taosDumpTable( + tableRecord.name, tableRecord.metric, + fp, pThread->taosCon, pThread->dbName); + if (ret >= 0) { + // TODO: sum table count and table rows by self + pThread->tablesOfDumpOut++; + pThread->rowsOfDumpOut += ret; + + if (pThread->rowsOfDumpOut >= lastRowsPrint) { + printf(" %"PRId64 " rows already be dumpout from database %s\n", + pThread->rowsOfDumpOut, pThread->dbName); + lastRowsPrint += 5000000; + } + +#if 0 + tablesInOneFile++; + if (tablesInOneFile >= g_args.table_batch) { + fclose(fp); + tablesInOneFile = 0; + + memset(tmpBuf, 0, 4096); + if (g_args.outpath[0] != 0) { + sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", + g_args.outpath, pThread->dbName, + pThread->threadIndex, fileNameIndex); + } else { + sprintf(tmpBuf, "%s.tables.%d-%d.sql", + pThread->dbName, pThread->threadIndex, fileNameIndex); + } + fileNameIndex++; + + fp = fopen(tmpBuf, "w"); + if (fp == NULL) { + errorPrint("%s() LN%d, failed to open file %s\n", + __func__, __LINE__, tmpBuf); + close(fd); + taos_free_result(tmpResult); + return NULL; + } + } +#endif } - } } - } - taos_free_result(tmpResult); - close(fd); - fclose(fp); + taos_free_result(tmpResult); + close(fd); + fclose(fp); - return NULL; + return NULL; } -static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, - int32_t numOfThread, char *dbName) +static void taosStartDumpOutWorkThreads(int32_t numOfThread, char *dbName) { - pthread_attr_t thattr; - SThreadParaObj *threadObj = - (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); - for (int t = 0; t < numOfThread; ++t) { - SThreadParaObj *pThread = threadObj + t; - pThread->rowsOfDumpOut = 0; - pThread->tablesOfDumpOut = 0; - pThread->threadIndex = t; - pThread->totalThreads = numOfThread; - tstrncpy(pThread->dbName, dbName, TSDB_DB_NAME_LEN); - pThread->taosCon = taosCon; - - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&(pThread->threadID), &thattr, taosDumpOutWorkThreadFp, - (void*)pThread) != 0) { - errorPrint("thread:%d failed to start\n", pThread->threadIndex); - exit(-1); + pthread_attr_t thattr; + SThreadParaObj *threadObj = + (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); + + if (threadObj == NULL) { + errorPrint("%s() LN%d, memory allocation failed!\n", + __func__, __LINE__); + return; } - } - for (int32_t t = 0; t < numOfThread; ++t) { - pthread_join(threadObj[t].threadID, NULL); - } + for (int t = 0; t < numOfThread; ++t) { + SThreadParaObj *pThread = threadObj + t; + pThread->rowsOfDumpOut = 0; + pThread->tablesOfDumpOut = 0; + pThread->threadIndex = t; + pThread->totalThreads = numOfThread; + tstrncpy(pThread->dbName, dbName, TSDB_DB_NAME_LEN); + pThread->taosCon = taos_connect(g_args.host, g_args.user, g_args.password, + NULL, g_args.port); + if (pThread->taosCon == NULL) { + errorPrint("Failed to connect to TDengine server %s\n", g_args.host); + return; + } + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&(pThread->threadID), &thattr, + taosDumpOutWorkThreadFp, + (void*)pThread) != 0) { + errorPrint("%s() LN%d, thread:%d failed to start\n", + __func__, __LINE__, pThread->threadIndex); + exit(-1); + } + } - // TODO: sum all thread dump table count and rows of per table, then save into result_output.txt - int64_t totalRowsOfDumpOut = 0; - int64_t totalChildTblsOfDumpOut = 0; - for (int32_t t = 0; t < numOfThread; ++t) { - totalChildTblsOfDumpOut += threadObj[t].tablesOfDumpOut; - totalRowsOfDumpOut += threadObj[t].rowsOfDumpOut; - } + for (int32_t t = 0; t < numOfThread; ++t) { + pthread_join(threadObj[t].threadID, NULL); + } - fprintf(g_fpOfResult, "# child table counter: %"PRId64"\n", totalChildTblsOfDumpOut); - fprintf(g_fpOfResult, "# row counter: %"PRId64"\n", totalRowsOfDumpOut); - g_resultStatistics.totalChildTblsOfDumpOut += totalChildTblsOfDumpOut; - g_resultStatistics.totalRowsOfDumpOut += totalRowsOfDumpOut; - free(threadObj); + // TODO: sum all thread dump table count and rows of per table, then save into result_output.txt + int64_t totalRowsOfDumpOut = 0; + int64_t totalChildTblsOfDumpOut = 0; + for (int32_t t = 0; t < numOfThread; ++t) { + totalChildTblsOfDumpOut += threadObj[t].tablesOfDumpOut; + totalRowsOfDumpOut += threadObj[t].rowsOfDumpOut; + } + + fprintf(g_fpOfResult, "# child table counter: %"PRId64"\n", + totalChildTblsOfDumpOut); + fprintf(g_fpOfResult, "# row counter: %"PRId64"\n", + totalRowsOfDumpOut); + g_resultStatistics.totalChildTblsOfDumpOut += totalChildTblsOfDumpOut; + g_resultStatistics.totalRowsOfDumpOut += totalRowsOfDumpOut; + free(threadObj); } static int32_t taosDumpStable(char *table, FILE *fp, @@ -1496,186 +1598,191 @@ static int32_t taosDumpStable(char *table, FILE *fp, static int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp) { - TAOS_ROW row; - int fd = -1; - STableRecord tableRecord; - char sqlstr[TSDB_MAX_SQL_LEN] = {0}; - - sprintf(sqlstr, "show %s.stables", dbName); + TAOS_ROW row; + int fd = -1; + STableRecord tableRecord; + char sqlstr[TSDB_MAX_SQL_LEN] = {0}; - TAOS_RES* res = taos_query(taosCon, sqlstr); - int32_t code = taos_errno(res); - if (code != 0) { - fprintf(stderr, "failed to run command <%s>, reason: %s\n", sqlstr, taos_errstr(res)); - taos_free_result(res); - exit(-1); - } + sprintf(sqlstr, "show %s.stables", dbName); - TAOS_FIELD *fields = taos_fetch_fields(res); + TAOS_RES* res = taos_query(taosCon, sqlstr); + int32_t code = taos_errno(res); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command <%s>, reason: %s\n", + __func__, __LINE__, sqlstr, taos_errstr(res)); + taos_free_result(res); + exit(-1); + } - char tmpFileName[TSDB_FILENAME_LEN + 1]; - memset(tmpFileName, 0, TSDB_FILENAME_LEN); - sprintf(tmpFileName, ".stables.tmp"); - fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - fprintf(stderr, "failed to open temp file: %s\n", tmpFileName); - taos_free_result(res); - (void)remove(".stables.tmp"); - exit(-1); - } + TAOS_FIELD *fields = taos_fetch_fields(res); + + char tmpFileName[MAX_FILE_NAME_LEN]; + memset(tmpFileName, 0, MAX_FILE_NAME_LEN); + sprintf(tmpFileName, ".stables.tmp"); + fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + errorPrint("%s() LN%d, failed to open temp file: %s\n", + __func__, __LINE__, tmpFileName); + taos_free_result(res); + (void)remove(".stables.tmp"); + exit(-1); + } - while ((row = taos_fetch_row(res)) != NULL) { - memset(&tableRecord, 0, sizeof(STableRecord)); - strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], - fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); - taosWrite(fd, &tableRecord, sizeof(STableRecord)); - } + while ((row = taos_fetch_row(res)) != NULL) { + memset(&tableRecord, 0, sizeof(STableRecord)); + strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], + fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); + taosWrite(fd, &tableRecord, sizeof(STableRecord)); + } - taos_free_result(res); - (void)lseek(fd, 0, SEEK_SET); + taos_free_result(res); + (void)lseek(fd, 0, SEEK_SET); - int superTblCnt = 0; - while (1) { - ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); - if (readLen <= 0) break; + int superTblCnt = 0; + while (1) { + ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); + if (readLen <= 0) break; - int ret = taosDumpStable(tableRecord.name, fp, taosCon, dbName); - if (0 == ret) { - superTblCnt++; + int ret = taosDumpStable(tableRecord.name, fp, taosCon, dbName); + if (0 == ret) { + superTblCnt++; + } } - } - // TODO: save dump super table into result_output.txt - fprintf(g_fpOfResult, "# super table counter: %d\n", superTblCnt); - g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; + // TODO: save dump super table into result_output.txt + fprintf(g_fpOfResult, "# super table counter: %d\n", superTblCnt); + g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; - close(fd); - (void)remove(".stables.tmp"); + close(fd); + (void)remove(".stables.tmp"); - return 0; + return 0; } -static int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon) { - TAOS_ROW row; - int fd = -1; - STableRecord tableRecord; - - taosDumpCreateDbClause(dbInfo, arguments->with_property, fp); +static int taosDumpDb(SDbInfo *dbInfo, FILE *fp, TAOS *taosCon) { + TAOS_ROW row; + int fd = -1; + STableRecord tableRecord; - fprintf(g_fpOfResult, "\n#### database: %s\n", dbInfo->name); - g_resultStatistics.totalDatabasesOfDumpOut++; + taosDumpCreateDbClause(dbInfo, g_args.with_property, fp); - char sqlstr[TSDB_MAX_SQL_LEN] = {0}; + fprintf(g_fpOfResult, "\n#### database: %s\n", + dbInfo->name); + g_resultStatistics.totalDatabasesOfDumpOut++; - fprintf(fp, "USE %s;\n\n", dbInfo->name); + char sqlstr[TSDB_MAX_SQL_LEN] = {0}; - (void)taosDumpCreateSuperTableClause(taosCon, dbInfo->name, fp); + fprintf(fp, "USE %s;\n\n", dbInfo->name); - sprintf(sqlstr, "show %s.tables", dbInfo->name); + (void)taosDumpCreateSuperTableClause(taosCon, dbInfo->name, fp); - TAOS_RES* res = taos_query(taosCon, sqlstr); - int code = taos_errno(res); - if (code != 0) { - fprintf(stderr, "failed to run command <%s>, reason:%s\n", sqlstr, taos_errstr(res)); - taos_free_result(res); - return -1; - } + sprintf(sqlstr, "show %s.tables", dbInfo->name); - char tmpBuf[TSDB_FILENAME_LEN + 1]; - memset(tmpBuf, 0, TSDB_FILENAME_LEN); - sprintf(tmpBuf, ".show-tables.tmp"); - fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (fd == -1) { - fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); - taos_free_result(res); - return -1; - } - - TAOS_FIELD *fields = taos_fetch_fields(res); + TAOS_RES* res = taos_query(taosCon, sqlstr); + int code = taos_errno(res); + if (code != 0) { + errorPrint("%s() LN%d, failed to run command <%s>, reason:%s\n", + __func__, __LINE__, sqlstr, taos_errstr(res)); + taos_free_result(res); + return -1; + } - int32_t numOfTable = 0; - while ((row = taos_fetch_row(res)) != NULL) { - memset(&tableRecord, 0, sizeof(STableRecord)); - tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], - fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); - tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], - min(TSDB_TABLE_NAME_LEN, fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes)); + char tmpBuf[MAX_FILE_NAME_LEN]; + memset(tmpBuf, 0, MAX_FILE_NAME_LEN); + sprintf(tmpBuf, ".show-tables.tmp"); + fd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (fd == -1) { + errorPrint("%s() LN%d, failed to open temp file: %s\n", + __func__, __LINE__, tmpBuf); + taos_free_result(res); + return -1; + } - taosWrite(fd, &tableRecord, sizeof(STableRecord)); + TAOS_FIELD *fields = taos_fetch_fields(res); - numOfTable++; - } - taos_free_result(res); - lseek(fd, 0, SEEK_SET); - - int maxThreads = g_args.thread_num; - int tableOfPerFile ; - if (numOfTable <= g_args.thread_num) { - tableOfPerFile = 1; - maxThreads = numOfTable; - } else { - tableOfPerFile = numOfTable / g_args.thread_num; - if (0 != numOfTable % g_args.thread_num) { - tableOfPerFile += 1; - } - } + int32_t numOfTable = 0; + while ((row = taos_fetch_row(res)) != NULL) { + memset(&tableRecord, 0, sizeof(STableRecord)); + tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], + fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); + tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], + min(TSDB_TABLE_NAME_LEN, fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes)); - char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord)); - if (NULL == tblBuf){ - fprintf(stderr, "failed to calloc %" PRIzu "\n", tableOfPerFile * sizeof(STableRecord)); - close(fd); - return -1; - } + taosWrite(fd, &tableRecord, sizeof(STableRecord)); - int32_t numOfThread = 0; - int subFd = -1; - for (numOfThread = 0; numOfThread < maxThreads; numOfThread++) { - memset(tmpBuf, 0, TSDB_FILENAME_LEN); - sprintf(tmpBuf, ".tables.tmp.%d", numOfThread); - subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); - if (subFd == -1) { - fprintf(stderr, "failed to open temp file: %s\n", tmpBuf); - for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { - sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); - (void)remove(tmpBuf); - } - sprintf(tmpBuf, ".show-tables.tmp"); - (void)remove(tmpBuf); - free(tblBuf); - close(fd); - return -1; + numOfTable++; } + taos_free_result(res); + lseek(fd, 0, SEEK_SET); - // read tableOfPerFile for fd, write to subFd - ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord)); - if (readLen <= 0) { - close(subFd); - break; + int maxThreads = g_args.thread_num; + int tableOfPerFile ; + if (numOfTable <= g_args.thread_num) { + tableOfPerFile = 1; + maxThreads = numOfTable; + } else { + tableOfPerFile = numOfTable / g_args.thread_num; + if (0 != numOfTable % g_args.thread_num) { + tableOfPerFile += 1; + } } - taosWrite(subFd, tblBuf, readLen); - close(subFd); - } - sprintf(tmpBuf, ".show-tables.tmp"); - (void)remove(tmpBuf); + char* tblBuf = (char*)calloc(1, tableOfPerFile * sizeof(STableRecord)); + if (NULL == tblBuf){ + errorPrint("failed to calloc %" PRIzu "\n", + tableOfPerFile * sizeof(STableRecord)); + close(fd); + return -1; + } - if (fd >= 0) { - close(fd); - fd = -1; - } + int32_t numOfThread = 0; + int subFd = -1; + for (numOfThread = 0; numOfThread < maxThreads; numOfThread++) { + memset(tmpBuf, 0, MAX_FILE_NAME_LEN); + sprintf(tmpBuf, ".tables.tmp.%d", numOfThread); + subFd = open(tmpBuf, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); + if (subFd == -1) { + errorPrint("%s() LN%d, failed to open temp file: %s\n", + __func__, __LINE__, tmpBuf); + for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { + sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); + (void)remove(tmpBuf); + } + sprintf(tmpBuf, ".show-tables.tmp"); + (void)remove(tmpBuf); + free(tblBuf); + close(fd); + return -1; + } - taos_free_result(res); + // read tableOfPerFile for fd, write to subFd + ssize_t readLen = read(fd, tblBuf, tableOfPerFile * sizeof(STableRecord)); + if (readLen <= 0) { + close(subFd); + break; + } + taosWrite(subFd, tblBuf, readLen); + close(subFd); + } - // start multi threads to dumpout - taosStartDumpOutWorkThreads(taosCon, arguments, numOfThread, dbInfo->name); - for (int loopCnt = 0; loopCnt < numOfThread; loopCnt++) { - sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); + sprintf(tmpBuf, ".show-tables.tmp"); (void)remove(tmpBuf); - } - free(tblBuf); - return 0; + if (fd >= 0) { + close(fd); + fd = -1; + } + + // start multi threads to dumpout + taosStartDumpOutWorkThreads(numOfThread, dbInfo->name); + for (int loopCnt = 0; loopCnt < numOfThread; loopCnt++) { + sprintf(tmpBuf, ".tables.tmp.%d", loopCnt); + (void)remove(tmpBuf); + } + + free(tblBuf); + return 0; } static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, @@ -1730,259 +1837,285 @@ static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, static void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName) { - int counter = 0; - int count_temp = 0; - - char* tmpBuf = (char *)malloc(COMMAND_SIZE); - if (tmpBuf == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return; - } + int counter = 0; + int count_temp = 0; + + char* tmpBuf = (char *)malloc(COMMAND_SIZE); + if (tmpBuf == NULL) { + errorPrint("%s() LN%d, failed to allocate %d memory\n", + __func__, __LINE__, COMMAND_SIZE); + return; + } - char *pstr = NULL; - pstr = tmpBuf; + char *pstr = NULL; + pstr = tmpBuf; - pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", - dbName, tableDes->name, dbName, metric); + pstr += sprintf(tmpBuf, + "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", + dbName, tableDes->name, dbName, metric); - for (; counter < numOfCols; counter++) { - if (tableDes->cols[counter].note[0] != '\0') break; - } + for (; counter < numOfCols; counter++) { + if (tableDes->cols[counter].note[0] != '\0') break; + } - assert(counter < numOfCols); - count_temp = counter; + assert(counter < numOfCols); + count_temp = counter; + + for (; counter < numOfCols; counter++) { + if (counter != count_temp) { + if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || + strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { + //pstr += sprintf(pstr, ", \'%s\'", tableDes->cols[counter].note); + pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note); + } else { + pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note); + } + } else { + if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || + strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { + //pstr += sprintf(pstr, "\'%s\'", tableDes->cols[counter].note); + pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); + } else { + pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); + } + /* pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); */ + } - for (; counter < numOfCols; counter++) { - if (counter != count_temp) { - if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || - strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { - //pstr += sprintf(pstr, ", \'%s\'", tableDes->cols[counter].note); - pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note); - } else { - pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note); - } - } else { - if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || - strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { - //pstr += sprintf(pstr, "\'%s\'", tableDes->cols[counter].note); - pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); - } else { - pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); - } - /* pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); */ + /* if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || strcasecmp(tableDes->cols[counter].type, "nchar") + * == 0) { */ + /* pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length); */ + /* } */ } - /* if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || strcasecmp(tableDes->cols[counter].type, "nchar") - * == 0) { */ - /* pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length); */ - /* } */ - } - - pstr += sprintf(pstr, ");"); + pstr += sprintf(pstr, ");"); - fprintf(fp, "%s\n", tmpBuf); - free(tmpBuf); + fprintf(fp, "%s\n", tmpBuf); + free(tmpBuf); } -static int taosDumpTableData(FILE *fp, char *tbname, - struct arguments *arguments, TAOS* taosCon, char* dbName) { - int64_t lastRowsPrint = 5000000; - int64_t totalRows = 0; - int count = 0; - char *pstr = NULL; - TAOS_ROW row = NULL; - int numFields = 0; +static int writeSchemaToAvro(char *jsonAvroSchema) +{ + errorPrint("%s() LN%d, TODO: implement write schema to avro", + __func__, __LINE__); + return 0; +} - if (arguments->schemaonly) { +static int64_t writeResultToAvro(TAOS_RES *res) +{ + errorPrint("%s() LN%d, TODO: implementation need\n", __func__, __LINE__); return 0; - } +} - int32_t sql_buf_len = arguments->max_sql_len; - char* tmpBuffer = (char *)calloc(1, sql_buf_len + 128); - if (tmpBuffer == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return -1; - } +static int64_t writeResultToSql(TAOS_RES *res, FILE *fp, char *dbName, char *tbName) +{ + int64_t totalRows = 0; - pstr = tmpBuffer; + int32_t sql_buf_len = g_args.max_sql_len; + char* tmpBuffer = (char *)calloc(1, sql_buf_len + 128); + if (tmpBuffer == NULL) { + errorPrint("failed to allocate %d memory\n", sql_buf_len + 128); + return -1; + } - char sqlstr[1024] = {0}; - sprintf(sqlstr, - "select * from %s.%s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc;", - dbName, tbname, arguments->start_time, arguments->end_time); + char *pstr = tmpBuffer; - TAOS_RES* tmpResult = taos_query(taosCon, sqlstr); - int32_t code = taos_errno(tmpResult); - if (code != 0) { - fprintf(stderr, "failed to run command %s, reason: %s\n", sqlstr, taos_errstr(tmpResult)); - free(tmpBuffer); - taos_free_result(tmpResult); - return -1; - } + TAOS_ROW row = NULL; + int numFields = 0; + int rowFlag = 0; + int64_t lastRowsPrint = 5000000; + int count = 0; - numFields = taos_field_count(tmpResult); - assert(numFields > 0); - TAOS_FIELD *fields = taos_fetch_fields(tmpResult); + numFields = taos_field_count(res); + assert(numFields > 0); + TAOS_FIELD *fields = taos_fetch_fields(res); - int rowFlag = 0; - int32_t curr_sqlstr_len = 0; - int32_t total_sqlstr_len = 0; - count = 0; - while ((row = taos_fetch_row(tmpResult)) != NULL) { - pstr = tmpBuffer; - curr_sqlstr_len = 0; + int32_t curr_sqlstr_len = 0; + int32_t total_sqlstr_len = 0; - int32_t* length = taos_fetch_lengths(tmpResult); // act len + while ((row = taos_fetch_row(res)) != NULL) { + curr_sqlstr_len = 0; - if (count == 0) { - total_sqlstr_len = 0; - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "INSERT INTO %s.%s VALUES (", dbName, tbname); - } else { - if (arguments->mysqlFlag) { - if (0 == rowFlag) { - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "("); - rowFlag++; + int32_t* length = taos_fetch_lengths(res); // act len + + if (count == 0) { + total_sqlstr_len = 0; + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, + "INSERT INTO %s.%s VALUES (", dbName, tbName); } else { - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ", ("); + if (g_args.mysqlFlag) { + if (0 == rowFlag) { + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "("); + rowFlag++; + } else { + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ", ("); + } + } else { + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "("); + } } - } else { - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "("); - } - } - for (int col = 0; col < numFields; col++) { - if (col != 0) curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ", "); + for (int col = 0; col < numFields; col++) { + if (col != 0) curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ", "); - if (row[col] == NULL) { - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "NULL"); - continue; - } + if (row[col] == NULL) { + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "NULL"); + continue; + } - switch (fields[col].type) { - case TSDB_DATA_TYPE_BOOL: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", - ((((int32_t)(*((char *)row[col]))) == 1) ? 1 : 0)); - break; - case TSDB_DATA_TYPE_TINYINT: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int8_t *)row[col])); - break; - case TSDB_DATA_TYPE_SMALLINT: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int16_t *)row[col])); - break; - case TSDB_DATA_TYPE_INT: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int32_t *)row[col])); - break; - case TSDB_DATA_TYPE_BIGINT: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRId64 "", - *((int64_t *)row[col])); - break; - case TSDB_DATA_TYPE_FLOAT: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%f", GET_FLOAT_VAL(row[col])); - break; - case TSDB_DATA_TYPE_DOUBLE: - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%f", GET_DOUBLE_VAL(row[col])); - break; - case TSDB_DATA_TYPE_BINARY: { - char tbuf[COMMAND_SIZE] = {0}; - //*(pstr++) = '\''; - converStringToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); - //pstr = stpcpy(pstr, tbuf); - //*(pstr++) = '\''; - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); - break; + switch (fields[col].type) { + case TSDB_DATA_TYPE_BOOL: + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", + ((((int32_t)(*((char *)row[col]))) == 1) ? 1 : 0)); + break; + case TSDB_DATA_TYPE_TINYINT: + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int8_t *)row[col])); + break; + case TSDB_DATA_TYPE_SMALLINT: + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int16_t *)row[col])); + break; + case TSDB_DATA_TYPE_INT: + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%d", *((int32_t *)row[col])); + break; + case TSDB_DATA_TYPE_BIGINT: + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRId64 "", + *((int64_t *)row[col])); + break; + case TSDB_DATA_TYPE_FLOAT: + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%f", GET_FLOAT_VAL(row[col])); + break; + case TSDB_DATA_TYPE_DOUBLE: + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%f", GET_DOUBLE_VAL(row[col])); + break; + case TSDB_DATA_TYPE_BINARY: + { + char tbuf[COMMAND_SIZE] = {0}; + //*(pstr++) = '\''; + converStringToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); + //pstr = stpcpy(pstr, tbuf); + //*(pstr++) = '\''; + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); + break; + } + case TSDB_DATA_TYPE_NCHAR: + { + char tbuf[COMMAND_SIZE] = {0}; + convertNCharToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); + break; + } + case TSDB_DATA_TYPE_TIMESTAMP: + if (!g_args.mysqlFlag) { + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRId64 "", + *(int64_t *)row[col]); + } else { + char buf[64] = "\0"; + int64_t ts = *((int64_t *)row[col]); + time_t tt = (time_t)(ts / 1000); + struct tm *ptm = localtime(&tt); + strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm); + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s.%03d\'", + buf, (int)(ts % 1000)); + } + break; + default: + break; + } } - case TSDB_DATA_TYPE_NCHAR: { - char tbuf[COMMAND_SIZE] = {0}; - convertNCharToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); - break; + + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ")"); + + totalRows++; + count++; + fprintf(fp, "%s", tmpBuffer); + + if (totalRows >= lastRowsPrint) { + printf(" %"PRId64 " rows already be dumpout from %s.%s\n", + totalRows, dbName, tbName); + lastRowsPrint += 5000000; + } + + total_sqlstr_len += curr_sqlstr_len; + + if ((count >= g_args.data_batch) + || (sql_buf_len - total_sqlstr_len < TSDB_MAX_BYTES_PER_ROW)) { + fprintf(fp, ";\n"); + count = 0; } - case TSDB_DATA_TYPE_TIMESTAMP: - if (!arguments->mysqlFlag) { - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRId64 "", - *(int64_t *)row[col]); - } else { - char buf[64] = "\0"; - int64_t ts = *((int64_t *)row[col]); - time_t tt = (time_t)(ts / 1000); - struct tm *ptm = localtime(&tt); - strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm); - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s.%03d\'", - buf, (int)(ts % 1000)); - } - break; - default: - break; - } } - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, ") "); + debugPrint("total_sqlstr_len: %d\n", total_sqlstr_len); - totalRows++; - count++; - fprintf(fp, "%s", tmpBuffer); + fprintf(fp, "\n"); + atomic_add_fetch_64(&g_totalDumpOutRows, totalRows); + free(tmpBuffer); - if (totalRows >= lastRowsPrint) { - printf(" %"PRId64 " rows already be dumpout from %s.%s\n", - totalRows, dbName, tbname); - lastRowsPrint += 5000000; - } + return 0; +} - total_sqlstr_len += curr_sqlstr_len; +static int taosDumpTableData(FILE *fp, char *tbName, + TAOS* taosCon, char* dbName, + char *jsonAvroSchema) { + int64_t totalRows = 0; - if ((count >= arguments->data_batch) - || (sql_buf_len - total_sqlstr_len < TSDB_MAX_BYTES_PER_ROW)) { - fprintf(fp, ";\n"); - count = 0; - } //else { - //fprintf(fp, "\\\n"); - //} - } + char sqlstr[1024] = {0}; + sprintf(sqlstr, + "select * from %s.%s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc;", + dbName, tbName, g_args.start_time, g_args.end_time); - printf("total_sqlstr_len: %d\n", total_sqlstr_len); + TAOS_RES* res = taos_query(taosCon, sqlstr); + int32_t code = taos_errno(res); + if (code != 0) { + errorPrint("failed to run command %s, reason: %s\n", + sqlstr, taos_errstr(res)); + taos_free_result(res); + return -1; + } - fprintf(fp, "\n"); - atomic_add_fetch_64(&totalDumpOutRows, totalRows); + if (g_args.avro) { + writeSchemaToAvro(jsonAvroSchema); + totalRows = writeResultToAvro(res); + } else { + totalRows = writeResultToSql(res, fp, dbName, tbName); + } - taos_free_result(tmpResult); - free(tmpBuffer); - return totalRows; + taos_free_result(res); + return totalRows; } static int taosCheckParam(struct arguments *arguments) { - if (arguments->all_databases && arguments->databases) { - fprintf(stderr, "conflict option --all-databases and --databases\n"); - return -1; - } + if (g_args.all_databases && g_args.databases) { + fprintf(stderr, "conflict option --all-databases and --databases\n"); + return -1; + } - if (arguments->start_time > arguments->end_time) { - fprintf(stderr, "start time is larger than end time\n"); - return -1; - } + if (g_args.start_time > g_args.end_time) { + fprintf(stderr, "start time is larger than end time\n"); + return -1; + } - if (arguments->arg_list_len == 0) { - if ((!arguments->all_databases) && (!arguments->isDumpIn)) { - fprintf(stderr, "taosdump requires parameters\n"); - return -1; + if (g_args.arg_list_len == 0) { + if ((!g_args.all_databases) && (!g_args.isDumpIn)) { + fprintf(stderr, "taosdump requires parameters\n"); + return -1; + } + } + /* + if (g_args.isDumpIn && (strcmp(g_args.outpath, DEFAULT_DUMP_FILE) != 0)) { + fprintf(stderr, "duplicate parameter input and output file path\n"); + return -1; + } + */ + if (!g_args.isDumpIn && g_args.encode != NULL) { + fprintf(stderr, "invalid option in dump out\n"); + return -1; } - } -/* - if (arguments->isDumpIn && (strcmp(arguments->outpath, DEFAULT_DUMP_FILE) != 0)) { - fprintf(stderr, "duplicate parameter input and output file path\n"); - return -1; - } -*/ - if (!arguments->isDumpIn && arguments->encode != NULL) { - fprintf(stderr, "invalid option in dump out\n"); - return -1; - } - if (arguments->table_batch <= 0) { - fprintf(stderr, "invalid option in dump out\n"); - return -1; - } + if (g_args.table_batch <= 0) { + fprintf(stderr, "invalid option in dump out\n"); + return -1; + } - return 0; + return 0; } /* @@ -2065,374 +2198,419 @@ char *ascii_literal_list[] = { "\\xf7", "\\xf8", "\\xf9", "\\xfa", "\\xfb", "\\xfc", "\\xfd", "\\xfe", "\\xff"}; static int converStringToReadable(char *str, int size, char *buf, int bufsize) { - char *pstr = str; - char *pbuf = buf; - while (size > 0) { - if (*pstr == '\0') break; - pbuf = stpcpy(pbuf, ascii_literal_list[((uint8_t)(*pstr))]); - pstr++; - size--; - } - *pbuf = '\0'; - return 0; + char *pstr = str; + char *pbuf = buf; + while (size > 0) { + if (*pstr == '\0') break; + pbuf = stpcpy(pbuf, ascii_literal_list[((uint8_t)(*pstr))]); + pstr++; + size--; + } + *pbuf = '\0'; + return 0; } static int convertNCharToReadable(char *str, int size, char *buf, int bufsize) { - char *pstr = str; - char *pbuf = buf; - // TODO - wchar_t wc; - while (size > 0) { - if (*pstr == '\0') break; - int byte_width = mbtowc(&wc, pstr, MB_CUR_MAX); - if (byte_width < 0) { - errorPrint("%s() LN%d, mbtowc() return fail.\n", __func__, __LINE__); - exit(-1); - } - - if ((int)wc < 256) { - pbuf = stpcpy(pbuf, ascii_literal_list[(int)wc]); - } else { - memcpy(pbuf, pstr, byte_width); - pbuf += byte_width; + char *pstr = str; + char *pbuf = buf; + // TODO + wchar_t wc; + while (size > 0) { + if (*pstr == '\0') break; + int byte_width = mbtowc(&wc, pstr, MB_CUR_MAX); + if (byte_width < 0) { + errorPrint("%s() LN%d, mbtowc() return fail.\n", __func__, __LINE__); + exit(-1); + } + + if ((int)wc < 256) { + pbuf = stpcpy(pbuf, ascii_literal_list[(int)wc]); + } else { + memcpy(pbuf, pstr, byte_width); + pbuf += byte_width; + } + pstr += byte_width; } - pstr += byte_width; - } - *pbuf = '\0'; + *pbuf = '\0'; - return 0; + return 0; } static void taosDumpCharset(FILE *fp) { - char charsetline[256]; + char charsetline[256]; - (void)fseek(fp, 0, SEEK_SET); - sprintf(charsetline, "#!%s\n", tsCharset); - (void)fwrite(charsetline, strlen(charsetline), 1, fp); + (void)fseek(fp, 0, SEEK_SET); + sprintf(charsetline, "#!%s\n", tsCharset); + (void)fwrite(charsetline, strlen(charsetline), 1, fp); } static void taosLoadFileCharset(FILE *fp, char *fcharset) { - char * line = NULL; - size_t line_size = 0; + char * line = NULL; + size_t line_size = 0; - (void)fseek(fp, 0, SEEK_SET); - ssize_t size = getline(&line, &line_size, fp); - if (size <= 2) { - goto _exit_no_charset; - } + (void)fseek(fp, 0, SEEK_SET); + ssize_t size = getline(&line, &line_size, fp); + if (size <= 2) { + goto _exit_no_charset; + } - if (strncmp(line, "#!", 2) != 0) { - goto _exit_no_charset; - } - if (line[size - 1] == '\n') { - line[size - 1] = '\0'; - size--; - } - strcpy(fcharset, line + 2); + if (strncmp(line, "#!", 2) != 0) { + goto _exit_no_charset; + } + if (line[size - 1] == '\n') { + line[size - 1] = '\0'; + size--; + } + strcpy(fcharset, line + 2); - tfree(line); - return; + tfree(line); + return; _exit_no_charset: - (void)fseek(fp, 0, SEEK_SET); - *fcharset = '\0'; - tfree(line); - return; + (void)fseek(fp, 0, SEEK_SET); + *fcharset = '\0'; + tfree(line); + return; } // ======== dumpIn support multi threads functions ================================// -static char **tsDumpInSqlFiles = NULL; -static int32_t tsSqlFileNum = 0; -static char tsDbSqlFile[TSDB_FILENAME_LEN] = {0}; -static char tsfCharset[64] = {0}; -static int taosGetFilesNum(const char *directoryName, const char *prefix) +static char **g_tsDumpInSqlFiles = NULL; +static int32_t g_tsSqlFileNum = 0; +static char g_tsDbSqlFile[MAX_FILE_NAME_LEN] = {0}; +static char g_tsCharset[64] = {0}; + +static int taosGetFilesNum(const char *directoryName, + const char *prefix, const char *prefix2) { - char cmd[1024] = { 0 }; - sprintf(cmd, "ls %s/*.%s | wc -l ", directoryName, prefix); + char cmd[1024] = { 0 }; - FILE *fp = popen(cmd, "r"); - if (fp == NULL) { - errorPrint("failed to execute:%s, error:%s\n", cmd, strerror(errno)); - exit(-1); - } + if (prefix2) + sprintf(cmd, "ls %s/*.%s %s/*.%s | wc -l ", + directoryName, prefix, directoryName, prefix2); + else + sprintf(cmd, "ls %s/*.%s | wc -l ", directoryName, prefix); - int fileNum = 0; - if (fscanf(fp, "%d", &fileNum) != 1) { - errorPrint("failed to execute:%s, parse result error\n", cmd); - exit(-1); - } + FILE *fp = popen(cmd, "r"); + if (fp == NULL) { + errorPrint("failed to execute:%s, error:%s\n", cmd, strerror(errno)); + exit(-1); + } - if (fileNum <= 0) { - errorPrint("directory:%s is empry\n", directoryName); - exit(-1); - } + int fileNum = 0; + if (fscanf(fp, "%d", &fileNum) != 1) { + errorPrint("failed to execute:%s, parse result error\n", cmd); + exit(-1); + } + + if (fileNum <= 0) { + errorPrint("directory:%s is empry\n", directoryName); + exit(-1); + } - pclose(fp); - return fileNum; + pclose(fp); + return fileNum; } -static void taosParseDirectory(const char *directoryName, const char *prefix, char **fileArray, int totalFiles) +static void taosParseDirectory(const char *directoryName, + const char *prefix, const char *prefix2, + char **fileArray, int totalFiles) { - char cmd[1024] = { 0 }; - sprintf(cmd, "ls %s/*.%s | sort", directoryName, prefix); + char cmd[1024] = { 0 }; - FILE *fp = popen(cmd, "r"); - if (fp == NULL) { - errorPrint("failed to execute:%s, error:%s\n", cmd, strerror(errno)); - exit(-1); - } + if (prefix2) { + sprintf(cmd, "ls %s/*.%s %s/*.%s | sort", + directoryName, prefix, directoryName, prefix2); + } else { + sprintf(cmd, "ls %s/*.%s | sort", directoryName, prefix); + } - int fileNum = 0; - while (fscanf(fp, "%128s", fileArray[fileNum++])) { - if (strcmp(fileArray[fileNum-1], tsDbSqlFile) == 0) { - fileNum--; + FILE *fp = popen(cmd, "r"); + if (fp == NULL) { + errorPrint("failed to execute:%s, error:%s\n", cmd, strerror(errno)); + exit(-1); + } + + int fileNum = 0; + while (fscanf(fp, "%128s", fileArray[fileNum++])) { + if (strcmp(fileArray[fileNum-1], g_tsDbSqlFile) == 0) { + fileNum--; + } + if (fileNum >= totalFiles) { + break; + } } - if (fileNum >= totalFiles) { - break; + + if (fileNum != totalFiles) { + errorPrint("directory:%s changed while read\n", directoryName); + pclose(fp); + exit(-1); } - } - if (fileNum != totalFiles) { - errorPrint("directory:%s changed while read\n", directoryName); pclose(fp); - exit(-1); - } - - pclose(fp); } -static void taosCheckTablesSQLFile(const char *directoryName) +static void taosCheckDatabasesSQLFile(const char *directoryName) { - char cmd[1024] = { 0 }; - sprintf(cmd, "ls %s/dbs.sql", directoryName); + char cmd[1024] = { 0 }; + sprintf(cmd, "ls %s/dbs.sql", directoryName); - FILE *fp = popen(cmd, "r"); - if (fp == NULL) { - errorPrint("failed to execute:%s, error:%s\n", cmd, strerror(errno)); - exit(-1); - } + FILE *fp = popen(cmd, "r"); + if (fp == NULL) { + errorPrint("failed to execute:%s, error:%s\n", cmd, strerror(errno)); + exit(-1); + } - while (fscanf(fp, "%128s", tsDbSqlFile)) { - break; - } + while (fscanf(fp, "%128s", g_tsDbSqlFile)) { + break; + } - pclose(fp); + pclose(fp); } -static void taosMallocSQLFiles() +static void taosMallocDumpFiles() { - tsDumpInSqlFiles = (char**)calloc(tsSqlFileNum, sizeof(char*)); - for (int i = 0; i < tsSqlFileNum; i++) { - tsDumpInSqlFiles[i] = calloc(1, TSDB_FILENAME_LEN); - } + g_tsDumpInSqlFiles = (char**)calloc(g_tsSqlFileNum, sizeof(char*)); + for (int i = 0; i < g_tsSqlFileNum; i++) { + g_tsDumpInSqlFiles[i] = calloc(1, MAX_FILE_NAME_LEN); + } } -static void taosFreeSQLFiles() +static void taosFreeDumpFiles() { - for (int i = 0; i < tsSqlFileNum; i++) { - tfree(tsDumpInSqlFiles[i]); - } - tfree(tsDumpInSqlFiles); + for (int i = 0; i < g_tsSqlFileNum; i++) { + tfree(g_tsDumpInSqlFiles[i]); + } + tfree(g_tsDumpInSqlFiles); } static void taosGetDirectoryFileList(char *inputDir) { - struct stat fileStat; - if (stat(inputDir, &fileStat) < 0) { - errorPrint("%s not exist\n", inputDir); - exit(-1); - } - - if (fileStat.st_mode & S_IFDIR) { - taosCheckTablesSQLFile(inputDir); - tsSqlFileNum = taosGetFilesNum(inputDir, "sql"); - int tsSqlFileNumOfTbls = tsSqlFileNum; - if (tsDbSqlFile[0] != 0) { - tsSqlFileNumOfTbls--; + struct stat fileStat; + if (stat(inputDir, &fileStat) < 0) { + errorPrint("%s not exist\n", inputDir); + exit(-1); } - taosMallocSQLFiles(); - if (0 != tsSqlFileNumOfTbls) { - taosParseDirectory(inputDir, "sql", tsDumpInSqlFiles, tsSqlFileNumOfTbls); + + if (fileStat.st_mode & S_IFDIR) { + taosCheckDatabasesSQLFile(inputDir); + if (g_args.avro) + g_tsSqlFileNum = taosGetFilesNum(inputDir, "sql", "avro"); + else + g_tsSqlFileNum += taosGetFilesNum(inputDir, "sql", NULL); + + int tsSqlFileNumOfTbls = g_tsSqlFileNum; + if (g_tsDbSqlFile[0] != 0) { + tsSqlFileNumOfTbls--; + } + taosMallocDumpFiles(); + if (0 != tsSqlFileNumOfTbls) { + if (g_args.avro) { + taosParseDirectory(inputDir, "sql", "avro", + g_tsDumpInSqlFiles, tsSqlFileNumOfTbls); + } else { + taosParseDirectory(inputDir, "sql", NULL, + g_tsDumpInSqlFiles, tsSqlFileNumOfTbls); + } + } + fprintf(stdout, "\nstart to dispose %d files in %s\n", + g_tsSqlFileNum, inputDir); + } else { + errorPrint("%s is not a directory\n", inputDir); + exit(-1); } - fprintf(stdout, "\nstart to dispose %d files in %s\n", tsSqlFileNum, inputDir); - } - else { - errorPrint("%s is not a directory\n", inputDir); - exit(-1); - } } static FILE* taosOpenDumpInFile(char *fptr) { - wordexp_t full_path; - - if (wordexp(fptr, &full_path, 0) != 0) { - fprintf(stderr, "ERROR: illegal file name: %s\n", fptr); - return NULL; - } + wordexp_t full_path; - char *fname = full_path.we_wordv[0]; + if (wordexp(fptr, &full_path, 0) != 0) { + errorPrint("illegal file name: %s\n", fptr); + return NULL; + } - if ((fname) && (strlen(fname) > 0)) { - FILE *f = fopen(fname, "r"); - if (f == NULL) { - errorPrint("%s() LN%d, failed to open file %s\n", - __func__, __LINE__, fname); - } + char *fname = full_path.we_wordv[0]; - wordfree(&full_path); - return f; - } + FILE *f = NULL; + if ((fname) && (strlen(fname) > 0)) { + f = fopen(fname, "r"); + if (f == NULL) { + errorPrint("%s() LN%d, failed to open file %s\n", + __func__, __LINE__, fname); + } + } - return NULL; + wordfree(&full_path); + return f; } static int taosDumpInOneFile(TAOS* taos, FILE* fp, char* fcharset, char* encode, char* fileName) { - int read_len = 0; - char * cmd = NULL; - size_t cmd_len = 0; - char * line = NULL; - size_t line_len = 0; - - cmd = (char *)malloc(TSDB_MAX_ALLOWED_SQL_LEN); - if (cmd == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return -1; - } + int read_len = 0; + char * cmd = NULL; + size_t cmd_len = 0; + char * line = NULL; + size_t line_len = 0; + + cmd = (char *)malloc(TSDB_MAX_ALLOWED_SQL_LEN); + if (cmd == NULL) { + errorPrint("%s() LN%d, failed to allocate memory\n", + __func__, __LINE__); + return -1; + } - int lastRowsPrint = 5000000; - int lineNo = 0; - while ((read_len = getline(&line, &line_len, fp)) != -1) { - ++lineNo; - if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; - line[--read_len] = '\0'; + int lastRowsPrint = 5000000; + int lineNo = 0; + while ((read_len = getline(&line, &line_len, fp)) != -1) { + ++lineNo; + if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; + line[--read_len] = '\0'; - //if (read_len == 0 || isCommentLine(line)) { // line starts with # - if (read_len == 0 ) { - continue; - } + //if (read_len == 0 || isCommentLine(line)) { // line starts with # + if (read_len == 0 ) { + continue; + } - if (line[read_len - 1] == '\\') { - line[read_len - 1] = ' '; - memcpy(cmd + cmd_len, line, read_len); - cmd_len += read_len; - continue; - } + if (line[read_len - 1] == '\\') { + line[read_len - 1] = ' '; + memcpy(cmd + cmd_len, line, read_len); + cmd_len += read_len; + continue; + } - memcpy(cmd + cmd_len, line, read_len); - cmd[read_len + cmd_len]= '\0'; - if (queryDbImpl(taos, cmd)) { - fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName); - fprintf(g_fpOfResult, "error sql: linenu:%d, file:%s\n", lineNo, fileName); - } + memcpy(cmd + cmd_len, line, read_len); + cmd[read_len + cmd_len]= '\0'; + if (queryDbImpl(taos, cmd)) { + errorPrint("%s() LN%d, error sql: linenu:%d, file:%s\n", + __func__, __LINE__, lineNo, fileName); + fprintf(g_fpOfResult, "error sql: linenu:%d, file:%s\n", lineNo, fileName); + } - memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); - cmd_len = 0; + memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); + cmd_len = 0; - if (lineNo >= lastRowsPrint) { - printf(" %d lines already be executed from file %s\n", lineNo, fileName); - lastRowsPrint += 5000000; + if (lineNo >= lastRowsPrint) { + printf(" %d lines already be executed from file %s\n", lineNo, fileName); + lastRowsPrint += 5000000; + } } - } - tfree(cmd); - tfree(line); - fclose(fp); - return 0; + tfree(cmd); + tfree(line); + fclose(fp); + return 0; } static void* taosDumpInWorkThreadFp(void *arg) { - SThreadParaObj *pThread = (SThreadParaObj*)arg; - for (int32_t f = 0; f < tsSqlFileNum; ++f) { - if (f % pThread->totalThreads == pThread->threadIndex) { - char *SQLFileName = tsDumpInSqlFiles[f]; - FILE* fp = taosOpenDumpInFile(SQLFileName); - if (NULL == fp) { - continue; - } - fprintf(stderr, "Success Open input file: %s\n", SQLFileName); - taosDumpInOneFile(pThread->taosCon, fp, tsfCharset, g_args.encode, SQLFileName); + SThreadParaObj *pThread = (SThreadParaObj*)arg; + for (int32_t f = 0; f < g_tsSqlFileNum; ++f) { + if (f % pThread->totalThreads == pThread->threadIndex) { + char *SQLFileName = g_tsDumpInSqlFiles[f]; + FILE* fp = taosOpenDumpInFile(SQLFileName); + if (NULL == fp) { + continue; + } + fprintf(stderr, ", Success Open input file: %s\n", + SQLFileName); + taosDumpInOneFile(pThread->taosCon, fp, g_tsCharset, g_args.encode, SQLFileName); + } } - } - return NULL; + return NULL; } -static void taosStartDumpInWorkThreads(void* taosCon, struct arguments *args) +static void taosStartDumpInWorkThreads() { - pthread_attr_t thattr; - SThreadParaObj *pThread; - int32_t totalThreads = args->thread_num; + pthread_attr_t thattr; + SThreadParaObj *pThread; + int32_t totalThreads = g_args.thread_num; - if (totalThreads > tsSqlFileNum) { - totalThreads = tsSqlFileNum; - } + if (totalThreads > g_tsSqlFileNum) { + totalThreads = g_tsSqlFileNum; + } - SThreadParaObj *threadObj = (SThreadParaObj *)calloc(totalThreads, sizeof(SThreadParaObj)); - for (int32_t t = 0; t < totalThreads; ++t) { - pThread = threadObj + t; - pThread->threadIndex = t; - pThread->totalThreads = totalThreads; - pThread->taosCon = taosCon; + SThreadParaObj *threadObj = (SThreadParaObj *)calloc( + totalThreads, sizeof(SThreadParaObj)); - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + if (NULL == threadObj) { + errorPrint("%s() LN%d, memory allocation failed\n", __func__, __LINE__); + } - if (pthread_create(&(pThread->threadID), &thattr, taosDumpInWorkThreadFp, (void*)pThread) != 0) { - fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); - exit(0); + for (int32_t t = 0; t < totalThreads; ++t) { + pThread = threadObj + t; + pThread->threadIndex = t; + pThread->totalThreads = totalThreads; + pThread->taosCon = taos_connect(g_args.host, g_args.user, g_args.password, + NULL, g_args.port); + if (pThread->taosCon == NULL) { + errorPrint("Failed to connect to TDengine server %s\n", g_args.host); + return; + } + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&(pThread->threadID), &thattr, + taosDumpInWorkThreadFp, (void*)pThread) != 0) { + errorPrint("%s() LN%d, thread:%d failed to start\n", + __func__, __LINE__, pThread->threadIndex); + exit(0); + } } - } - for (int t = 0; t < totalThreads; ++t) { - pthread_join(threadObj[t].threadID, NULL); - } + for (int t = 0; t < totalThreads; ++t) { + pthread_join(threadObj[t].threadID, NULL); + } - for (int t = 0; t < totalThreads; ++t) { - taos_close(threadObj[t].taosCon); - } - free(threadObj); + for (int t = 0; t < totalThreads; ++t) { + taos_close(threadObj[t].taosCon); + } + free(threadObj); } -static int taosDumpIn(struct arguments *arguments) { - assert(arguments->isDumpIn); +static int taosDumpIn() { + assert(g_args.isDumpIn); - TAOS *taos = NULL; - FILE *fp = NULL; + TAOS *taos = NULL; + FILE *fp = NULL; - taos = taos_connect(arguments->host, arguments->user, arguments->password, NULL, arguments->port); - if (taos == NULL) { - fprintf(stderr, "failed to connect to TDengine server\n"); - return -1; - } + taos = taos_connect( + g_args.host, g_args.user, g_args.password, + NULL, g_args.port); + if (taos == NULL) { + errorPrint("%s() LN%d, failed to connect to TDengine server\n", + __func__, __LINE__); + return -1; + } - taosGetDirectoryFileList(arguments->inpath); + taosGetDirectoryFileList(g_args.inpath); - int32_t tsSqlFileNumOfTbls = tsSqlFileNum; - if (tsDbSqlFile[0] != 0) { - tsSqlFileNumOfTbls--; + int32_t tsSqlFileNumOfTbls = g_tsSqlFileNum; + if (g_tsDbSqlFile[0] != 0) { + tsSqlFileNumOfTbls--; - fp = taosOpenDumpInFile(tsDbSqlFile); - if (NULL == fp) { - fprintf(stderr, "failed to open input file %s\n", tsDbSqlFile); - return -1; - } - fprintf(stderr, "Success Open input file: %s\n", tsDbSqlFile); + fp = taosOpenDumpInFile(g_tsDbSqlFile); + if (NULL == fp) { + errorPrint("%s() LN%d, failed to open input file %s\n", + __func__, __LINE__, g_tsDbSqlFile); + return -1; + } + fprintf(stderr, "Success Open input file: %s\n", g_tsDbSqlFile); - taosLoadFileCharset(fp, tsfCharset); + taosLoadFileCharset(fp, g_tsCharset); - taosDumpInOneFile(taos, fp, tsfCharset, arguments->encode, tsDbSqlFile); - } + taosDumpInOneFile(taos, fp, g_tsCharset, g_args.encode, + g_tsDbSqlFile); + } - if (0 != tsSqlFileNumOfTbls) { - taosStartDumpInWorkThreads(taos, arguments); - } + taos_close(taos); - taos_close(taos); - taosFreeSQLFiles(); - return 0; + if (0 != tsSqlFileNumOfTbls) { + taosStartDumpInWorkThreads(); + } + + taosFreeDumpFiles(); + return 0; } diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index 417513314c7013a3e707999bfbd7f9dbd1a4baa8..b7b268b19e6b6f92babb74cfd3f23793be037cd0 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -25,6 +25,13 @@ void osInit() { strcpy(tsDataDir, "/var/lib/power"); strcpy(tsLogDir, "/var/log/power"); strcpy(tsScriptDir, "/etc/power"); +#elif (_TD_TQ_ == true) + if (configDir[0] == 0) { + strcpy(configDir, "/etc/tq"); + } + strcpy(tsDataDir, "/var/lib/tq"); + strcpy(tsLogDir, "/var/log/tq"); + strcpy(tsScriptDir, "/etc/tq"); #else if (configDir[0] == 0) { strcpy(configDir, "/etc/taos"); diff --git a/src/os/src/windows/wEnv.c b/src/os/src/windows/wEnv.c index 21c6aa66426c6184ef64dff8535745425c8f7973..b35cb8f040aec5ff4b4fb12665d0842e72958ba1 100644 --- a/src/os/src/windows/wEnv.c +++ b/src/os/src/windows/wEnv.c @@ -31,7 +31,14 @@ void osInit() { strcpy(tsDataDir, "C:/PowerDB/data"); strcpy(tsLogDir, "C:/PowerDB/log"); strcpy(tsScriptDir, "C:/PowerDB/script"); - +#elif (_TD_TQ_ == true) + if (configDir[0] == 0) { + strcpy(configDir, "C:/TQ/cfg"); + } + strcpy(tsVnodeDir, "C:/TQ/data"); + strcpy(tsDataDir, "C:/TQ/data"); + strcpy(tsLogDir, "C:/TQ/log"); + strcpy(tsScriptDir, "C:/TQ/script"); #else if (configDir[0] == 0) { strcpy(configDir, "C:/TDengine/cfg"); diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index 54372ae8c28d91a72243256a74a8fb53c317eab2..e53d2826c76acb057020b05bfeba4e22cf128c51 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -771,7 +771,7 @@ int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta) { int64_t maxBufSize = 0; SMFInfo minfo; - taosHashEmpty(pfs->metaCache); + taosHashClear(pfs->metaCache); // No meta file, just return if (pfs->cstatus->pmf == NULL) return 0; diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index c37426069cf64bb35ac5e1100d49e8103c851625..616b844c1388575130a2b1c02033cfedb7ef9e57 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -140,7 +140,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param); -void taosHashEmpty(SHashObj *pHashObj); +void taosHashClear(SHashObj *pHashObj); /** * clean up hash table diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 5b3218382fff38382a536c8ece077fd2f350adf2..d7bee9b67cad8fe91a182d76a443c04fd82be44c 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -144,6 +144,14 @@ static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* */ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); +/** + * Check whether the hash table is empty or not. + * + * @param pHashObj the hash table object + * @return if the hash table is empty or not + */ +static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj); + /** * Get the next element in hash table for iterator * @param pIter @@ -195,7 +203,16 @@ void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp) { } } -int32_t taosHashGetSize(const SHashObj *pHashObj) { return (int32_t)((pHashObj == NULL) ? 0 : pHashObj->size); } +int32_t taosHashGetSize(const SHashObj *pHashObj) { + if (!pHashObj) { + return 0; + } + return (int32_t)atomic_load_64(&pHashObj->size); +} + +static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { + return taosHashGetSize(pHashObj) == 0; +} int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); @@ -281,7 +298,7 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { } void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void* d, size_t dsize) { - if (pHashObj->size <= 0 || keyLen == 0 || key == NULL) { + if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { return NULL; } @@ -338,7 +355,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { } int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t dsize) { - if (pHashObj == NULL || pHashObj->size <= 0) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { return -1; } @@ -405,7 +422,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe } int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) { - if (pHashObj == NULL || pHashObj->size == 0) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { return 0; } @@ -478,7 +495,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi return 0; } -void taosHashEmpty(SHashObj *pHashObj) { +void taosHashClear(SHashObj *pHashObj) { if (pHashObj == NULL) { return; } @@ -517,7 +534,7 @@ void taosHashCleanup(SHashObj *pHashObj) { return; } - taosHashEmpty(pHashObj); + taosHashClear(pHashObj); tfree(pHashObj->hashList); // destroy mem block @@ -535,7 +552,7 @@ void taosHashCleanup(SHashObj *pHashObj) { // for profile only int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { - if (pHashObj == NULL || pHashObj->size == 0) { + if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { return 0; } diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 30d53af4b978c1c697a3ac892bad9b279af1252c..80071986d6d2396ef2aec8f7841b0897cb3d7b26 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -312,6 +312,9 @@ void taosReadGlobalLogCfg() { #ifdef _TD_POWER_ printf("configDir:%s not there, use default value: /etc/power", configDir); strcpy(configDir, "/etc/power"); + #elif (_TD_TQ_ == true) + printf("configDir:%s not there, use default value: /etc/tq", configDir); + strcpy(configDir, "/etc/tq"); #else printf("configDir:%s not there, use default value: /etc/taos", configDir); strcpy(configDir, "/etc/taos"); diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 7f127fc396a13f0a7796dcb4ce1dd63ce96cb951..385dcac0756392457e4edabcd918fa70da302984 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -83,6 +83,8 @@ int64_t dbgWSize = 0; #ifdef _TD_POWER_ char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power"; +#elif (_TD_TQ_ == true) +char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/tq"; #else char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/taos"; #endif diff --git a/tests/pytest/dbmgmt/nanoSecondCheck.py b/tests/pytest/dbmgmt/nanoSecondCheck.py index 27050a2213f7e6bddeb5cc6135c7fe4760018f61..a5e9adacee53a9172a2d8990ccc4d83feb983bdd 100644 --- a/tests/pytest/dbmgmt/nanoSecondCheck.py +++ b/tests/pytest/dbmgmt/nanoSecondCheck.py @@ -99,6 +99,15 @@ class TDTestCase: tdSql.query('select avg(speed) from tb interval(100000000b)') tdSql.checkRows(4) + tdSql.error('select avg(speed) from tb interval(1b);') + tdSql.error('select avg(speed) from tb interval(999b);') + + tdSql.query('select avg(speed) from tb interval(1000b);') + tdSql.checkRows(5) + + tdSql.query('select avg(speed) from tb interval(1u);') + tdSql.checkRows(5) + tdSql.query('select avg(speed) from tb interval(100000000b) sliding (100000000b);') tdSql.checkRows(4) diff --git a/tests/pytest/tools/taosdemoAllTest/sub_no_result.json b/tests/pytest/tools/taosdemoAllTest/sub_no_result.json new file mode 100644 index 0000000000000000000000000000000000000000..cdf7c2314ede28e9c3ccaa9d53864737ff3fac96 --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/sub_no_result.json @@ -0,0 +1,25 @@ +{ + "filetype": "subscribe", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "databases": "db", + "confirm_parameter_prompt": "no", + "specified_table_query": + { + "concurrent":1, + "mode":"sync", + "interval": 0, + "restart":"yes", + "keepProgress":"no", + "endAfterConsume": 1100000, + "sqls": [ + { + "sql": "select * from st;", + "result": "" + }] + } + } + \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoAllTest/subscribeNoResult.py b/tests/pytest/tools/taosdemoAllTest/subscribeNoResult.py new file mode 100644 index 0000000000000000000000000000000000000000..270eea17cb6c913719fb67c4b8f33065b0a0445d --- /dev/null +++ b/tests/pytest/tools/taosdemoAllTest/subscribeNoResult.py @@ -0,0 +1,82 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import _thread + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1601481600000 + self.numberOfRecords = 1100000 + + def execCmdAndGetOutput(self, cmd): + r = os.popen(cmd) + text = r.read() + r.close() + return text + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + tdSql.prepare() + tdSql.execute("create table st(ts timestamp, c1 timestamp, c2 int, c3 bigint, c4 float, c5 double, c6 binary(8), c7 smallint, c8 tinyint, c9 bool, c10 nchar(8)) tags(t1 int)") + tdSql.execute("create table t1 using st tags(0)") + currts = self.ts + finish = 0 + while(finish < self.numberOfRecords): + sql = "insert into t1 values" + for i in range(finish, self.numberOfRecords): + sql += "(%d, 1019774612, 29931, 1442173978, 165092.468750, 1128.643179, 'MOCq1pTu', 18405, 82, 0, 'g0A6S0Fu')" % (currts + i) + finish = i + 1 + if (1048576 - len(sql)) < 16384: + break + tdSql.execute(sql) + + binPath = buildPath+ "/build/bin/" + + os.system("%staosdemo -f tools/taosdemoAllTest/sub_no_result.json -g 2>&1 | tee sub_no_result.log" % binPath) + test_line = int(self.execCmdAndGetOutput("cat sub_no_result.log | wc -l")) + if(test_line < 1100024): + tdLog.exit("failed test subscribeNoResult: %d != expected(1100024)" % test_line) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file