Hotfix/sangshuduo/td 3973 use jemalloc for master (#6794)

* [TD-3973]<feature>: add jemalloc as submodule.

* [TD-3973]<feature>: add jemalloc as submodule.

* [TD-3973]<feature>: use jemalloc.

build works as following instructions:

cmake .. -DJEMALLOC_ENABLED=true

make

* fix jemalloc at tag 5.2.1

* fix conflicts

* make install works.

* fix conflicts.

* release script works.

* fix typo

* [TD-3937]<feature>: support jemalloc

add install funtion to all scripts.

* adjust install_jemalloc() position for check compatibility.

* fix position bug.

* add ldconfig for jemalloc library cache refresh.

* cherry pick from develop branch.

* cherry pick from develop branch.
Co-authored-by: NShuduo Sang <sdsang@taosdata.com>

packaging/release.sh

@@ -246,11 +246,11 @@ if [ "$osType" != "Darwin" ]; then
   cd ${script_dir}/tools
 
   if [[ "$dbName" == "taos" ]]; then
-    ${csudo} ./makepkg.sh    ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode}
+    ${csudo} ./makepkg.sh    ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${verNumberComp}
     ${csudo} ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode}
     ${csudo} ./makearbi.sh   ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode}
   else
-    ${csudo} ./makepkg_power.sh    ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName}
+    ${csudo} ./makepkg_power.sh    ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ${verNumberComp}
     ${csudo} ./makeclient_power.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName}
     ${csudo} ./makearbi_power.sh   ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode}
   fi

packaging/tools/install.sh

@@ -270,7 +270,13 @@ function install_jemalloc() {
             ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3
             ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3
         fi
-        ${csudo} ldconfig
+
+        if [ -d /etc/ld.so.conf.d ]; then
+            ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf
+            ${csudo} ldconfig
+        else
+            echo "/etc/ld.so.conf.d not found!"
+        fi
     fi
 }
 
@@ -766,9 +772,13 @@ vercomp () {
 
 function is_version_compatible() {
 
-    curr_version=$(${bin_dir}/taosd -V | head -1 | cut -d ' ' -f 3)
+    curr_version=`ls ${script_dir}/driver/libtaos.so* |cut -d '.' -f 3-6`
 
-    min_compatible_version=$(${script_dir}/bin/taosd -V | head -1 | cut -d ' ' -f 5)
+    if [ -f ${script_dir}/driver/vercomp.txt ]; then
+        min_compatible_version=`cat ${script_dir}/driver/vercomp.txt`
+    else
+        min_compatible_version=$(${script_dir}/bin/tqd -V | head -1 | cut -d ' ' -f 5)
+    fi
 
     vercomp $curr_version $min_compatible_version
     case $? in
@@ -784,7 +794,6 @@ function update_TDengine() {
         echo "File taos.tar.gz does not exist"
         exit 1
     fi
-    install_jemalloc
     tar -zxf taos.tar.gz
     install_jemalloc
 

packaging/tools/install_arbi.sh

@@ -165,7 +165,13 @@ function install_jemalloc() {
             ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3
             ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3
         fi
-        ${csudo} ldconfig
+
+        if [ -d /etc/ld.so.conf.d ]; then
+            ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf
+            ${csudo} ldconfig
+        else
+            echo "/etc/ld.so.conf.d not found!"
+        fi
     fi
 }
 

packaging/tools/install_arbi_power.sh

@@ -158,7 +158,13 @@ function install_jemalloc() {
             ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3
             ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3
         fi
-        ${csudo} ldconfig
+
+        if [ -d /etc/ld.so.conf.d ]; then
+            ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf
+            ${csudo} ldconfig
+        else
+            echo "/etc/ld.so.conf.d not found!"
+        fi
     fi
 }
 

packaging/tools/install_client_power.sh

@@ -182,7 +182,13 @@ function install_jemalloc() {
             ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3
             ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3
         fi
-        ${csudo} ldconfig
+
+        if [ -d /etc/ld.so.conf.d ]; then
+            ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf
+            ${csudo} ldconfig
+        else
+            echo "/etc/ld.so.conf.d not found!"
+        fi
     fi
 }
 
@@ -228,6 +234,7 @@ function update_PowerDB() {
         exit 1
     fi
     tar -zxf power.tar.gz
+    install_jemalloc
 
     echo -e "${GREEN}Start to update PowerDB client...${NC}"
     # Stop the client shell if running
@@ -241,7 +248,6 @@ function update_PowerDB() {
     install_log
     install_header
     install_lib
-    install_jemalloc
     if [ "$pagMode" != "lite" ]; then
       install_connector
     fi

packaging/tools/install_power.sh

@@ -263,7 +263,13 @@ function install_jemalloc() {
             ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3
             ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3
         fi
-        ${csudo} ldconfig
+
+        if [ -d /etc/ld.so.conf.d ]; then
+            ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf
+            ${csudo} ldconfig
+        else
+            echo "/etc/ld.so.conf.d not found!"
+        fi
     fi
 }
 
@@ -735,9 +741,13 @@ vercomp () {
 
 function is_version_compatible() {
 
-    curr_version=$(${bin_dir}/powerd -V | head -1 | cut -d ' ' -f 3)
+    curr_version=`ls ${script_dir}/driver/libtaos.so* |cut -d '.' -f 3-6`
 
-    min_compatible_version=$(${script_dir}/bin/powerd -V | head -1 | cut -d ' ' -f 5)
+    if [ -f ${script_dir}/driver/vercomp.txt ]; then
+        min_compatible_version=`cat ${script_dir}/driver/vercomp.txt`
+    else
+        min_compatible_version=$(${script_dir}/bin/tqd -V | head -1 | cut -d ' ' -f 5)
+    fi
 
     vercomp $curr_version $min_compatible_version
     case $? in
@@ -753,7 +763,6 @@ function update_PowerDB() {
         echo "File power.tar.gz does not exist"
         exit 1
     fi
-    install_jemalloc
     tar -zxf power.tar.gz
     install_jemalloc
 

packaging/tools/make_install.sh

@@ -217,7 +217,13 @@ function install_jemalloc() {
             /usr/bin/install -c -d /usr/local/share/man/man3
             /usr/bin/install -c -m 644 ${binary_dir}/build/share/man/man3/jemalloc.3 /usr/local/share/man/man3
         fi
-        ${csudo} ldconfig
+
+        if [ -d /etc/ld.so.conf.d ]; then
+            ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf
+            ${csudo} ldconfig
+        else
+            echo "/etc/ld.so.conf.d not found!"
+        fi
     fi
 }
 

packaging/tools/makepkg.sh

@@ -14,6 +14,7 @@ osType=$5
 verMode=$6
 verType=$7
 pagMode=$8
+versionComp=$9
 
 script_dir="$(dirname $(readlink -f $0))"
 top_dir="$(readlink -f ${script_dir}/../..)"
@@ -175,8 +176,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
   cp -r ${examples_dir}/C#     ${install_dir}/examples
 fi
 # Copy driver
-mkdir -p ${install_dir}/driver
-cp ${lib_files} ${install_dir}/driver
+mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" > ${install_dir}/driver/vercomp.txt
 
 # Copy connector
 connector_dir="${code_dir}/connector"

packaging/tools/makepkg_power.sh

@@ -14,6 +14,7 @@ osType=$5
 verMode=$6
 verType=$7
 pagMode=$8
+versionComp=$9
 
 script_dir="$(dirname $(readlink -f $0))"
 top_dir="$(readlink -f ${script_dir}/../..)"
@@ -32,10 +33,10 @@ fi
 
 # Directories and files.
 #if [ "$pagMode" == "lite" ]; then
-#  strip ${build_dir}/bin/taosd 
+#  strip ${build_dir}/bin/taosd
 #  strip ${build_dir}/bin/taos
 #  bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${script_dir}/remove_power.sh"
-#else 
+#else
 #  bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${build_dir}/bin/powerdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_power.sh\
 #              ${script_dir}/set_core.sh ${script_dir}/startPre.sh  ${script_dir}/taosd-dump-cfg.gdb"
 #fi
@@ -70,19 +71,19 @@ mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cf
 #mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
 mkdir -p ${install_dir}/bin
 if [ "$pagMode" == "lite" ]; then
-  strip ${build_dir}/bin/taosd 
+  strip ${build_dir}/bin/taosd
   strip ${build_dir}/bin/taos
 #  bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${script_dir}/remove_power.sh"
   cp ${build_dir}/bin/taos          ${install_dir}/bin/power
   cp ${build_dir}/bin/taosd         ${install_dir}/bin/powerd
   cp ${script_dir}/remove_power.sh  ${install_dir}/bin
-else 
+else
 #  bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${build_dir}/bin/powerdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_power.sh ${script_dir}/set_core.sh"
   cp ${build_dir}/bin/taos          ${install_dir}/bin/power
   cp ${build_dir}/bin/taosd         ${install_dir}/bin/powerd
   cp ${script_dir}/remove_power.sh  ${install_dir}/bin
-  cp ${build_dir}/bin/taosdemo      ${install_dir}/bin/powerdemo   
-  cp ${build_dir}/bin/taosdump      ${install_dir}/bin/powerdump  
+  cp ${build_dir}/bin/taosdemo      ${install_dir}/bin/powerdemo
+  cp ${build_dir}/bin/taosdump      ${install_dir}/bin/powerdump
   cp ${build_dir}/bin/tarbitrator   ${install_dir}/bin
   cp ${script_dir}/set_core.sh      ${install_dir}/bin
   cp ${script_dir}/get_client.sh    ${install_dir}/bin
@@ -99,14 +100,14 @@ mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}
 if [ "$verMode" == "cluster" ]; then
     sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove_power.sh >> remove_power_temp.sh
     mv remove_power_temp.sh ${install_dir}/bin/remove_power.sh
-    
+
     mkdir -p ${install_dir}/nginxd && cp -r ${nginx_dir}/* ${install_dir}/nginxd
     cp ${nginx_dir}/png/taos.png ${install_dir}/nginxd/admin/images/taos.png
     rm -rf ${install_dir}/nginxd/png
 
-    sed -i "s/TDengine/PowerDB/g"   ${install_dir}/nginxd/admin/*.html 
+    sed -i "s/TDengine/PowerDB/g"   ${install_dir}/nginxd/admin/*.html
     sed -i "s/TDengine/PowerDB/g"   ${install_dir}/nginxd/admin/js/*.js
-    
+
     sed -i '/dataDir/ {s/taos/power/g}'  ${install_dir}/cfg/taos.cfg
     sed -i '/logDir/  {s/taos/power/g}'  ${install_dir}/cfg/taos.cfg
     sed -i "s/TDengine/PowerDB/g"        ${install_dir}/cfg/taos.cfg
@@ -149,17 +150,16 @@ sed -i '/root/   {s/taosdata/powerdb/g}'  ${install_dir}/examples/c/*.c
 if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
   cp -r ${examples_dir}/JDBC   ${install_dir}/examples
   cp -r ${examples_dir}/matlab ${install_dir}/examples
-  sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/matlab/TDengineDemo.m  
+  sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/matlab/TDengineDemo.m
   cp -r ${examples_dir}/python ${install_dir}/examples
-  sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/python/read_example.py  
+  sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/python/read_example.py
   cp -r ${examples_dir}/R      ${install_dir}/examples
-  sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/R/command.txt  
-  cp -r ${examples_dir}/go     ${install_dir}/examples  
+  sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/R/command.txt
+  cp -r ${examples_dir}/go     ${install_dir}/examples
   sed -i '/root/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/go/taosdemo.go
 fi
 # Copy driver
-mkdir -p ${install_dir}/driver 
-cp ${lib_files} ${install_dir}/driver
+mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" > ${install_dir}/driver/vercomp.txt
 
 # Copy connector
 connector_dir="${code_dir}/connector"
@@ -178,11 +178,11 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
     echo "WARNING: go connector not found, please check if want to use it!"
   fi
   cp -r ${connector_dir}/python  ${install_dir}/connector/
-  
+
   sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/connector/python/taos/cinterface.py
-  
+
   sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/connector/python/taos/subscription.py
-  
+
   sed -i '/self._password/ {s/taosdata/powerdb/g}'  ${install_dir}/connector/python/taos/connection.py
 fi
 # Copy release note
@@ -190,7 +190,7 @@ fi
 
 # exit 1
 
-cd ${release_dir} 
+cd ${release_dir}
 
 if [ "$verMode" == "cluster" ]; then
   pkg_name=${install_dir}-${osType}-${cpuType}
@@ -207,8 +207,8 @@ fi
 
 if [ "$verType" == "beta" ]; then
   pkg_name=${pkg_name}-${verType}
-elif [ "$verType" == "stable" ]; then 
-  pkg_name=${pkg_name} 
+elif [ "$verType" == "stable" ]; then
+  pkg_name=${pkg_name}
 else
   echo "unknow verType, nor stabel or beta"
   exit 1