Feature/sangshuduo/td 4959 taosdump support binarydump (#8522)

* [TD4959]<feature>: taosdump support dumping binary data.

* [TD-4959]<feature>: taosdump support avro format.

add avro as submodule.

* [TD-4959]<feature>: taosdump support avro format.

add avro as submodule.

* taosdump code refactoring.

* add avro argument interface.

* fix arm32 build issue.

* hugh refactoring.

* fix tag value rather than reuse note.

* redefine note/varible

* taosdump text-format dump reworked.

* use avro standard schema format.

* build avro-c on linux only.

* avro schema write works.

* support bool,tinyint,smallint,int,bigint,float,double,timestamp write

* support nchar dump

* support all codecs

* adjust options order.

* refactor code

* multithread dump in sql refactored.

* fix nchar quote mark.

* fix bash script error

* fix arg parse issue.

* add avro dump in stmt interfaces.

* basic dump in works.

* fix src/kit/taosdump/CMakeLists.txt for CentOS

* fix memory leak.

* no avro utils installed

* make cmake and install script supports avro.
Co-authored-by: NShuduo Sang <sdsang@taosdata.com>

.gitmodules

@@ -19,3 +19,6 @@
 [submodule "src/plugins/blm3"]
 	path = src/plugins/blm3
 	url = https://github.com/taosdata/blm3
+[submodule "deps/avro"]
+	path = deps/avro
+	url = https://github.com/apache/avro

CMakeLists.txt

@@ -15,6 +15,26 @@ ELSE ()
   CMAKE_MINIMUM_REQUIRED(VERSION 3.0)
 ENDIF ()
 
+if(NOT WIN32)
+  string(ASCII 27 Esc)
+  set(ColourReset "${Esc}[m")
+  set(ColourBold  "${Esc}[1m")
+  set(Red         "${Esc}[31m")
+  set(Green       "${Esc}[32m")
+  set(Yellow      "${Esc}[33m")
+  set(Blue        "${Esc}[34m")
+  set(Magenta     "${Esc}[35m")
+  set(Cyan        "${Esc}[36m")
+  set(White       "${Esc}[37m")
+  set(BoldRed     "${Esc}[1;31m")
+  set(BoldGreen   "${Esc}[1;32m")
+  set(BoldYellow  "${Esc}[1;33m")
+  set(BoldBlue    "${Esc}[1;34m")
+  set(BoldMagenta "${Esc}[1;35m")
+  set(BoldCyan    "${Esc}[1;36m")
+  set(BoldWhite   "${Esc}[1;37m")
+endif()
+
 SET(TD_ACCOUNT FALSE)
 SET(TD_ADMIN FALSE)
 SET(TD_GRANT FALSE)

cmake/define.inc

@@ -128,7 +128,6 @@ IF (TD_APLHINE)
   MESSAGE(STATUS "aplhine is defined")
 ENDIF ()
 
-MESSAGE("before BUILD_HTTP: " ${BUILD_HTTP})
 IF ("${BUILD_HTTP}" STREQUAL "")
   IF (TD_LINUX)
     IF (TD_ARM_32)
@@ -140,7 +139,6 @@ IF ("${BUILD_HTTP}" STREQUAL "")
     SET(BUILD_HTTP "true")
   ENDIF ()
 ENDIF ()
-MESSAGE("after BUILD_HTTP: " ${BUILD_HTTP})
 
 IF (${BUILD_HTTP} MATCHES "true")
   SET(TD_BUILD_HTTP TRUE)
@@ -150,6 +148,14 @@ IF (TD_BUILD_HTTP)
   ADD_DEFINITIONS(-DHTTP_EMBEDDED)
 ENDIF ()
 
+IF ("${AVRO_SUPPORT}" MATCHES "true")
+  SET(TD_AVRO_SUPPORT TRUE)
+ENDIF ()
+
+IF (TD_AVRO_SUPPORT)
+  ADD_DEFINITIONS(-DAVRO_SUPPORT)
+ENDIF ()
+
 IF (TD_LINUX)
   ADD_DEFINITIONS(-DLINUX)
   ADD_DEFINITIONS(-D_LINUX)

cmake/input.inc

@@ -92,6 +92,8 @@ ENDIF ()
 
 SET(TD_BUILD_HTTP FALSE)
 
+SET(TD_AVRO_SUPPORT FALSE)
+
 SET(TD_MEMORY_SANITIZER FALSE)
 IF (${MEMORY_SANITIZER} MATCHES "true")
   SET(TD_MEMORY_SANITIZER TRUE)

deps/CMakeLists.txt

@@ -25,10 +25,36 @@ IF (TD_DARWIN AND TD_MQTT)
   ADD_SUBDIRECTORY(MQTT-C)
 ENDIF ()
 
+IF (AVRO_SUPPORT)
+  MESSAGE("")
+  MESSAGE("${Green} ENABLE avro format support ${ColourReset}")
+  MESSAGE("")
+  include(ExternalProject)
+  ExternalProject_Add(
+    apache-avro
+    PREFIX          "avro"
+    SOURCE_DIR      ${CMAKE_CURRENT_SOURCE_DIR}/avro/lang/c
+    BUILD_IN_SOURCE 1
+    PATCH_COMMAND
+        COMMAND   git clean -f -d
+        COMMAND   sed -i.bak -e "/TARGETS avroappend/d" ${CMAKE_CURRENT_SOURCE_DIR}/avro/lang/c/src/CMakeLists.txt
+        COMMAND   sed -i.bak -e "/TARGETS avrocat/d" ${CMAKE_CURRENT_SOURCE_DIR}/avro/lang/c/src/CMakeLists.txt
+        COMMAND   sed -i.bak -e "/TARGETS avromod/d" ${CMAKE_CURRENT_SOURCE_DIR}/avro/lang/c/src/CMakeLists.txt
+        COMMAND   sed -i.bak -e "/TARGETS avropipe/d" ${CMAKE_CURRENT_SOURCE_DIR}/avro/lang/c/src/CMakeLists.txt
+    CONFIGURE_COMMAND   cmake -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_BINARY_DIR}/build
+    )
+ELSE ()
+  MESSAGE("")
+  MESSAGE("${Yellow} NO avro format support ${ColourReset}")
+  MESSAGE("")
+ENDIF ()
+
 IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
+  MESSAGE("")
+  MESSAGE("${Green} ENABLE jemalloc ${ColourReset}")
+  MESSAGE("")
   MESSAGE("setup deps/jemalloc, current source dir:" ${CMAKE_CURRENT_SOURCE_DIR})
   MESSAGE("binary dir:" ${CMAKE_BINARY_DIR})
-  include(ExternalProject)
   ExternalProject_Add(jemalloc
     PREFIX        "jemalloc"
     SOURCE_DIR    ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc
@@ -39,5 +65,5 @@ IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
 ENDIF ()
 
 IF (${TSZ_ENABLED} MATCHES "true")
-  ADD_SUBDIRECTORY(TSZ)  
-ENDIF()
\ No newline at end of file
+  ADD_SUBDIRECTORY(TSZ)
+ENDIF()

TSZ @ 0ca5b15a

-Subproject commit 0ca5b15a8eac40327dd737be52c926fa5675712c

avro @ a1fce29d

+Subproject commit a1fce29d9675b4dd95dfee9db32cc505d0b2227c

packaging/check_package.sh

@@ -128,12 +128,12 @@ function check_link() {
 function check_main_path() {
     #check install main dir and all sub dir
     main_dir=("" "cfg" "bin" "connector" "driver" "examples" "include" "init.d")
-    for i in ${main_dir[@]};do
+    for i in "${main_dir[@]}";do
         check_file ${install_main_dir} $i
     done
     if [ "$verMode" == "cluster" ]; then
         nginx_main_dir=("admin" "conf" "html" "sbin" "logs")
-        for i in ${nginx_main_dir[@]};do
+        for i in "${nginx_main_dir[@]}";do
             check_file ${nginx_dir}  $i
         done
     fi
@@ -143,11 +143,11 @@ function check_main_path() {
 function check_bin_path() {
     # check install bin dir and all sub dir
     bin_dir=("taos" "taosd" "blm3" "taosdemo" "taosdump" "remove.sh" "tarbitrator" "set_core.sh")
-    for i in ${bin_dir[@]};do
+    for i in "${bin_dir[@]}";do
         check_file ${sbin_dir} $i
     done
     lbin_dir=("taos" "taosd" "blm3" "taosdemo" "taosdump" "rmtaos" "tarbitrator" "set_core")
-    for i in ${lbin_dir[@]};do
+    for i in "${lbin_dir[@]}";do
         check_link ${bin_link_dir}/$i
     done
     if [ "$verMode" == "cluster" ]; then
@@ -171,7 +171,7 @@ function check_lib_path() {
 function check_header_path() {
 	# check all header
 	header_dir=("taos.h" "taoserror.h")
-    for i in ${header_dir[@]};do
+    for i in "${header_dir[@]}";do
         check_link ${inc_link_dir}/$i
     done
     echo -e "Check bin  path:\033[32mOK\033[0m!"

packaging/release.sh

@@ -151,7 +151,7 @@ function vercomp () {
 }
 
 # 1. check version information
-if (( ! is_valid_version $verNumber ) || ( ! is_valid_version $verNumberComp ) || [[ "$(vercomp $verNumber $verNumberComp)" == '2' ]]); then
+if ( ( ! is_valid_version $verNumber ) || ( ! is_valid_version $verNumberComp ) || [[ "$(vercomp $verNumber $verNumberComp)" == '2' ]] ); then
   echo "please enter correct version"
   exit 0
 fi

packaging/rpm/makerpm.sh

@@ -36,7 +36,7 @@ local cur_dir
 cd $1
 cur_dir=$(pwd)
 
-for dirlist in $(ls ${cur_dir}); do
+for dirlist in "${cur_dir}"/*; do
   if test -d ${dirlist}; then
     cd ${dirlist}
     cp_rpm_package ${cur_dir}/${dirlist}

packaging/tools/check_os.sh

-# /bin/bash
+#!/bin/bash
 #
 CSI=$(echo -e "\033[")
 CRED="${CSI}1;31m"

packaging/tools/install.sh

@@ -303,7 +303,7 @@ function add_newHostname_to_hosts() {
   iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}')
   arr=($iphost)
   IFS="$OLD_IFS"
-  for s in ${arr[@]}
+  for s in "${arr[@]}"
   do
     if [[ "$s" == "$localIp" ]]; then
       return
@@ -358,7 +358,7 @@ function is_correct_ipaddr() {
   IFS=" "
   arr=($iplist)
   IFS="$OLD_IFS"
-  for s in ${arr[@]}
+  for s in "${arr[@]}"
   do
    if [[ "$s" == "$newIp" ]]; then
      return 0

packaging/tools/install_power.sh

@@ -287,7 +287,7 @@ function add_newHostname_to_hosts() {
   iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}')
   arr=($iphost)
   IFS="$OLD_IFS"
-  for s in ${arr[@]}
+  for s in "${arr[@]}"
   do
     if [[ "$s" == "$localIp" ]]; then
       return
@@ -342,7 +342,7 @@ function is_correct_ipaddr() {
   IFS=" "
   arr=($iplist)
   IFS="$OLD_IFS"
-  for s in ${arr[@]}
+  for s in "${arr[@]}"
   do
    if [[ "$s" == "$newIp" ]]; then
      return 0

packaging/tools/install_pro.sh

@@ -278,7 +278,7 @@ function add_newHostname_to_hosts() {
   iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}')
   arr=($iphost)
   IFS="$OLD_IFS"
-  for s in ${arr[@]}
+  for s in "${arr[@]}"
   do
     if [[ "$s" == "$localIp" ]]; then
       return
@@ -305,7 +305,7 @@ function set_hostname() {
    echo "set hostname fail!"
    return
   fi
-  
+
   #ubuntu/centos /etc/hostname
   if [[ -e /etc/hostname ]]; then
     ${csudo} echo $newHostname > /etc/hostname   ||:
@@ -330,7 +330,7 @@ function is_correct_ipaddr() {
   IFS=" "
   arr=($iplist)
   IFS="$OLD_IFS"
-  for s in ${arr[@]}
+  for s in "${arr[@]}"
   do
    if [[ "$s" == "$newIp" ]]; then
      return 0

packaging/tools/install_tq.sh

@@ -287,7 +287,7 @@ function add_newHostname_to_hosts() {
   iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}')
   arr=($iphost)
   IFS="$OLD_IFS"
-  for s in ${arr[@]}
+  for s in "${arr[@]}"
   do
     if [[ "$s" == "$localIp" ]]; then
       return
@@ -342,7 +342,7 @@ function is_correct_ipaddr() {
   IFS=" "
   arr=($iplist)
   IFS="$OLD_IFS"
-  for s in ${arr[@]}
+  for s in "${arr[@]}"
   do
    if [[ "$s" == "$newIp" ]]; then
      return 0

packaging/tools/make_install.sh

@@ -212,7 +212,8 @@ function install_jemalloc() {
         fi
         if [ -f "${binary_dir}/build/include/jemalloc/jemalloc.h" ]; then
             /usr/bin/install -c -d /usr/local/include/jemalloc
-            /usr/bin/install -c -m 644 ${binary_dir}/build/include/jemalloc/jemalloc.h /usr/local/include/jemalloc
+            /usr/bin/install -c -m 644 ${binary_dir}/build/include/jemalloc/jemalloc.h\
+                /usr/local/include/jemalloc
         fi
         if [ -f "${binary_dir}/build/lib/libjemalloc.so.2" ]; then
             /usr/bin/install -c -d /usr/local/lib
@@ -225,23 +226,47 @@ function install_jemalloc() {
                 /usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc_pic.a /usr/local/lib
             if [ -f "${binary_dir}/build/lib/pkgconfig/jemalloc.pc" ]; then
                 /usr/bin/install -c -d /usr/local/lib/pkgconfig
-                /usr/bin/install -c -m 644 ${binary_dir}/build/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig
+                /usr/bin/install -c -m 644 ${binary_dir}/build/lib/pkgconfig/jemalloc.pc\
+                    /usr/local/lib/pkgconfig
+            fi
+            if [ -d /etc/ld.so.conf.d ]; then
+                echo "/usr/local/lib" | ${csudo} tee /etc/ld.so.conf.d/jemalloc.conf
+                ${csudo} ldconfig
+            else
+                echo "/etc/ld.so.conf.d not found!"
             fi
         fi
         if [ -f "${binary_dir}/build/share/doc/jemalloc/jemalloc.html" ]; then
             /usr/bin/install -c -d /usr/local/share/doc/jemalloc
-            /usr/bin/install -c -m 644 ${binary_dir}/build/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc
+            /usr/bin/install -c -m 644 ${binary_dir}/build/share/doc/jemalloc/jemalloc.html\
+                /usr/local/share/doc/jemalloc
         fi
         if [ -f "${binary_dir}/build/share/man/man3/jemalloc.3" ]; then
             /usr/bin/install -c -d /usr/local/share/man/man3
-            /usr/bin/install -c -m 644 ${binary_dir}/build/share/man/man3/jemalloc.3 /usr/local/share/man/man3
+            /usr/bin/install -c -m 644 ${binary_dir}/build/share/man/man3/jemalloc.3\
+                /usr/local/share/man/man3
         fi
 
-        if [ -d /etc/ld.so.conf.d ]; then
-            echo "/usr/local/lib" | ${csudo} tee /etc/ld.so.conf.d/jemalloc.conf
-            ${csudo} ldconfig
-        else
-            echo "/etc/ld.so.conf.d not found!"
+    fi
+}
+
+function install_avro() {
+    if [ "$osType" != "Darwin" ]; then
+        if [ -f "${binary_dir}/build/$1/libavro.so.23.0.0" ]; then
+            /usr/bin/install -c -d /usr/local/$1
+            /usr/bin/install -c -m 755 ${binary_dir}/build/$1/libavro.so.23.0.0 /usr/local/$1
+            ln -sf libavro.so.23.0.0 /usr/local/$1/libavro.so.23
+            ln -sf libavro.so.23 /usr/local/$1/libavro.so
+            /usr/bin/install -c -d /usr/local/$1
+            [ -f ${binary_dir}/build/$1/libavro.a ] &&
+                /usr/bin/install -c -m 755 ${binary_dir}/build/$1/libavro.a /usr/local/$1
+
+            if [ -d /etc/ld.so.conf.d ]; then
+                echo "/usr/local/$1" | ${csudo} tee /etc/ld.so.conf.d/libavro.conf
+                ${csudo} ldconfig
+            else
+                echo "/etc/ld.so.conf.d not found!"
+            fi
         fi
     fi
 }
@@ -292,6 +317,8 @@ function install_lib() {
     fi
 
     install_jemalloc
+    install_avro lib
+    install_avro lib64
 
     if [ "$osType" != "Darwin" ]; then
         ${csudo} ldconfig

packaging/tools/post.sh

@@ -127,7 +127,7 @@ function add_newHostname_to_hosts() {
   iphost=$(cat /etc/hosts | grep $1 | awk '{print $1}')
   arr=($iphost)
   IFS="$OLD_IFS"
-  for s in ${arr[@]}
+  for s in "${arr[@]}"
   do
     if [[ "$s" == "$localIp" ]]; then
       return
@@ -182,7 +182,7 @@ function is_correct_ipaddr() {
   IFS=" "
   arr=($iplist)
   IFS="$OLD_IFS"
-  for s in ${arr[@]}
+  for s in "${arr[@]}"
   do
    if [[ "$s" == "$newIp" ]]; then
      return 0

src/kit/taosdump/CMakeLists.txt

@@ -3,6 +3,7 @@ PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
+INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/build/include)
 INCLUDE_DIRECTORIES(inc)
 AUX_SOURCE_DIRECTORY(. SRC)
 
@@ -61,12 +62,22 @@ ENDIF ()
 MESSAGE("TD_VERSION_NUMBER is:" ${TD_VERSION_NUMBER})
 ADD_DEFINITIONS(-DTD_VERNUMBER="${TD_VERSION_NUMBER}")
 
+LINK_DIRECTORIES(${CMAKE_BINARY_DIR}/build/lib ${CMAKE_BINARY_DIR}/build/lib64)
+
 IF (TD_LINUX)
   ADD_EXECUTABLE(taosdump ${SRC})
   IF (TD_SOMODE_STATIC)
-    TARGET_LINK_LIBRARIES(taosdump taos_static cJson)
+    IF (AVRO_SUPPORT)
+      TARGET_LINK_LIBRARIES(taosdump taos_static avro jansson)
+    ELSE ()
+      TARGET_LINK_LIBRARIES(taosdump taos_static)
+    ENDIF()
   ELSE ()
-    TARGET_LINK_LIBRARIES(taosdump taos cJson)
+    IF (AVRO_SUPPORT)
+      TARGET_LINK_LIBRARIES(taosdump taos avro jansson)
+    ELSE ()
+      TARGET_LINK_LIBRARIES(taosdump taos)
+    ENDIF ()
   ENDIF ()
 ENDIF ()
 
@@ -74,8 +85,8 @@ IF (TD_DARWIN)
   # missing <argp.h> for macosx
   # ADD_EXECUTABLE(taosdump ${SRC})
   # IF (TD_SOMODE_STATIC)
-  #   TARGET_LINK_LIBRARIES(taosdump taos_static cJson)
+  #   TARGET_LINK_LIBRARIES(taosdump taos_static jansson)
   # ELSE ()
-  #   TARGET_LINK_LIBRARIES(taosdump taos cJson)
+  #   TARGET_LINK_LIBRARIES(taosdump taos jansson)
   # ENDIF ()
 ENDIF ()

src/plugins/CMakeLists.txt

 CMAKE_MINIMUM_REQUIRED(VERSION 3.0...3.20)
 PROJECT(TDengine)
 
-if(NOT WIN32)
-  string(ASCII 27 Esc)
-  set(ColourReset "${Esc}[m")
-  set(ColourBold  "${Esc}[1m")
-  set(Red         "${Esc}[31m")
-  set(Green       "${Esc}[32m")
-  set(Yellow      "${Esc}[33m")
-  set(Blue        "${Esc}[34m")
-  set(Magenta     "${Esc}[35m")
-  set(Cyan        "${Esc}[36m")
-  set(White       "${Esc}[37m")
-  set(BoldRed     "${Esc}[1;31m")
-  set(BoldGreen   "${Esc}[1;32m")
-  set(BoldYellow  "${Esc}[1;33m")
-  set(BoldBlue    "${Esc}[1;34m")
-  set(BoldMagenta "${Esc}[1;35m")
-  set(BoldCyan    "${Esc}[1;36m")
-  set(BoldWhite   "${Esc}[1;37m")
-endif()
-
 ADD_SUBDIRECTORY(monitor)
 
 IF (TD_BUILD_HTTP)