Hotfix/sangshuduo/td 5702 taosdemo remove memop for master (#7682)

* [TD-5702]<fix>: taosdemo remove memory operation.

* [TD-5702]<fix>: taosdemo remove memory operation.

* add remainderBufLen to check row data generation.

* row data generation with remainder buffer length checking.

* git checkout --patch hotfix/sangshuduo/TD-5702-taosdemo-remove-memop taosdemo.c

* optimized memop for rand too.

* fix stmt rand file generation bug.

* merge make_install.sh for homebrew.
Co-authored-by: NShuduo Sang <sdsang@taosdata.com>

packaging/tools/make_install.sh

@@ -19,6 +19,7 @@ else
 fi
 
 # Dynamic directory
+
 data_dir="/var/lib/taos"
 
 if [ "$osType" != "Darwin" ]; then
@@ -29,25 +30,32 @@ fi
 
 data_link_dir="/usr/local/taos/data"
 log_link_dir="/usr/local/taos/log"
-
-cfg_install_dir="/etc/taos"
+if [ "$osType" != "Darwin" ]; then
+    cfg_install_dir="/etc/taos"
+else
+    cfg_install_dir="/usr/local/Cellar/tdengine/${verNumber}/taos"
+fi
 
 if [ "$osType" != "Darwin" ]; then
     bin_link_dir="/usr/bin"
     lib_link_dir="/usr/lib"
     lib64_link_dir="/usr/lib64"
     inc_link_dir="/usr/include"
-else
-    bin_link_dir="/usr/local/bin"
-    lib_link_dir="/usr/local/lib"
-    inc_link_dir="/usr/local/include"
 fi
 
 #install main path
-install_main_dir="/usr/local/taos"
+if [ "$osType" != "Darwin" ]; then
+    install_main_dir="/usr/local/taos"
+else
+    install_main_dir="/usr/local/Cellar/tdengine/${verNumber}"
+fi
 
 # old bin dir
-bin_dir="/usr/local/taos/bin"
+if [ "$osType" != "Darwin" ]; then
+     bin_dir="/usr/local/taos/bin"
+else
+     bin_dir="/usr/local/Cellar/tdengine/${verNumber}/bin"
+fi
 
 service_config_dir="/etc/systemd/system"
 
@@ -59,12 +67,11 @@ GREEN_UNDERLINE='\033[4;32m'
 NC='\033[0m'
 
 csudo=""
-if command -v sudo > /dev/null; then
-    csudo="sudo"
-fi
 
 if [ "$osType" != "Darwin" ]; then
-
+    if command -v sudo > /dev/null; then
+    csudo="sudo"
+    fi
     initd_mod=0
     service_mod=2
     if pidof systemd &> /dev/null; then
@@ -137,18 +144,16 @@ function install_main_path() {
 
 function install_bin() {
     # Remove links
-    ${csudo} rm -f ${bin_link_dir}/taos         || :
-
     if [ "$osType" != "Darwin" ]; then
+        ${csudo} rm -f ${bin_link_dir}/taos     || :
         ${csudo} rm -f ${bin_link_dir}/taosd    || :
         ${csudo} rm -f ${bin_link_dir}/taosdemo || :
         ${csudo} rm -f ${bin_link_dir}/perfMonitor || :
         ${csudo} rm -f ${bin_link_dir}/taosdump || :
         ${csudo} rm -f ${bin_link_dir}/set_core || :
+        ${csudo} rm -f ${bin_link_dir}/rmtaos   || :
     fi
-
-    ${csudo} rm -f ${bin_link_dir}/rmtaos       || :
-
+    
     ${csudo} cp -r ${binary_dir}/build/bin/* ${install_main_dir}/bin
     ${csudo} cp -r ${script_dir}/taosd-dump-cfg.gdb   ${install_main_dir}/bin
 
@@ -162,20 +167,17 @@ function install_bin() {
     ${csudo} chmod 0555 ${install_main_dir}/bin/*
 
     #Make link
-    [ -x ${install_main_dir}/bin/taos ]      && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos         || :
-
     if [ "$osType" != "Darwin" ]; then
+        [ -x ${install_main_dir}/bin/taos ]      && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos    || :
         [ -x ${install_main_dir}/bin/taosd ]     && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd   || :
         [ -x ${install_main_dir}/bin/taosdump ]  && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
         [ -x ${install_main_dir}/bin/taosdemo ]  && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
         [ -x ${install_main_dir}/bin/perfMonitor ]  && ${csudo} ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || :
         [ -x ${install_main_dir}/set_core.sh ]  && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
     fi
-
+    
     if [ "$osType" != "Darwin" ]; then
-        [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos  || :
-    else
-        [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/rmtaos  || :
+       [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos  || :
     fi
 }
 
@@ -222,7 +224,7 @@ function install_jemalloc() {
         fi
 
         if [ -d /etc/ld.so.conf.d ]; then
-            ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf
+            echo "/usr/local/lib" | ${csudo} tee /etc/ld.so.conf.d/jemalloc.conf
             ${csudo} ldconfig
         else
             echo "/etc/ld.so.conf.d not found!"
@@ -248,10 +250,8 @@ function install_lib() {
         fi
     else
         ${csudo} cp -Rf ${binary_dir}/build/lib/libtaos.* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/*
-        ${csudo} ln -sf ${install_main_dir}/driver/libtaos.1.dylib ${lib_link_dir}/libtaos.1.dylib
-        ${csudo} ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib
     fi
-
+    
     install_jemalloc
 
     if [ "$osType" != "Darwin" ]; then
@@ -261,10 +261,14 @@ function install_lib() {
 
 function install_header() {
 
-    ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h    || :
+    if [ "$osType" != "Darwin" ]; then
+        ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h    || :
+    fi
     ${csudo} cp -f ${source_dir}/src/inc/taos.h ${source_dir}/src/inc/taoserror.h ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/*
-    ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
-    ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
+    if [ "$osType" != "Darwin" ]; then
+        ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
+        ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
+    fi
 }
 
 function install_config() {
@@ -272,29 +276,30 @@ function install_config() {
 
     if [ ! -f ${cfg_install_dir}/taos.cfg ]; then
         ${csudo} mkdir -p ${cfg_install_dir}
-        [ -f ${script_dir}/../cfg/taos.cfg ] && ${csudo} cp ${script_dir}/../cfg/taos.cfg ${cfg_install_dir}
+        [ -f ${script_dir}/../cfg/taos.cfg ] &&
+        ${csudo} cp ${script_dir}/../cfg/taos.cfg ${cfg_install_dir}
         ${csudo} chmod 644 ${cfg_install_dir}/*
     fi
 
     ${csudo} cp -f ${script_dir}/../cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org
-    ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg
+    
+    if [ "$osType" != "Darwin" ]; then ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg
+    fi
 }
 
 function install_log() {
-    ${csudo} rm -rf ${log_dir}  || :
-
     if [ "$osType" != "Darwin" ]; then
-        ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir}
-    else
-        mkdir -p ${log_dir} && chmod 777 ${log_dir}
+            ${csudo} rm -rf ${log_dir}  || :
+            ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir}
+            ${csudo} ln -s ${log_dir} ${install_main_dir}/log
     fi
-
-    ${csudo} ln -s ${log_dir} ${install_main_dir}/log
 }
 
 function install_data() {
-    ${csudo} mkdir -p ${data_dir}
-    ${csudo} ln -s ${data_dir} ${install_main_dir}/data
+    if [ "$osType" != "Darwin" ]; then
+        ${csudo} mkdir -p ${data_dir}
+        ${csudo} ln -s ${data_dir} ${install_main_dir}/data
+    fi
 }
 
 function install_connector() {
@@ -309,7 +314,6 @@ function install_connector() {
         echo "WARNING: go connector not found, please check if want to use it!"
     fi
     ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector
-
     ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null
 }
 
@@ -489,24 +493,24 @@ function install_TDengine() {
     else
         echo -e "${GREEN}Start to install TDEngine Client ...${NC}"
     fi
-
+    
     install_main_path
 
     if [ "$osType" != "Darwin" ]; then
         install_data
     fi
+
     install_log
     install_header
     install_lib
     install_connector
     install_examples
-
     install_bin
-
+    
     if [ "$osType" != "Darwin" ]; then
         install_service
     fi
-
+    
     install_config
 
     if [ "$osType" != "Darwin" ]; then

src/kit/taosdemo/taosdemo.c

@@ -5976,12 +5976,85 @@ static int64_t generateData(char *recBuf, char **data_type,
     return (int32_t)strlen(recBuf);
 }
 
+static int generateSampleMemoryFromRand(SSuperTable *stbInfo)
+{
+    char data[MAX_DATA_SIZE];
+    memset(data, 0, MAX_DATA_SIZE);
+
+    char *buff = malloc(stbInfo->lenOfOneRow);
+    if (NULL == buff) {
+        errorPrint2("%s() LN%d, memory allocation %"PRId64" bytes failed\n",
+                __func__, __LINE__, stbInfo->lenOfOneRow);
+        exit(EXIT_FAILURE);
+    }
+
+    for (int i=0; i < MAX_SAMPLES_ONCE_FROM_FILE; i++) {
+        uint64_t pos = 0;
+        memset(buff, 0, stbInfo->lenOfOneRow);
+
+        for (int c = 0; c < stbInfo->columnCount; c++) {
+            char *tmp;
+            if (0 == strncasecmp(stbInfo->columns[c].dataType,
+                        "BINARY", strlen("BINARY"))) {
+                rand_string(data, stbInfo->columns[c].dataLen);
+                pos += sprintf(buff + pos, "%s,", data);
+            } else if (0 == strncasecmp(stbInfo->columns[c].dataType,
+                        "NCHAR", strlen("NCHAR"))) {
+                rand_string(data, stbInfo->columns[c].dataLen);
+                pos += sprintf(buff + pos, "%s,", data);
+            } else if (0 == strncasecmp(stbInfo->columns[c].dataType,
+                        "INT", strlen("INT"))) {
+                if ((g_args.demo_mode) && (c == 1)) {
+                    tmp = demo_voltage_int_str();
+                } else {
+                    tmp = rand_int_str();
+                }
+                pos += sprintf(buff + pos, "%s,", tmp);
+            } else if (0 == strncasecmp(stbInfo->columns[c].dataType,
+                        "BIGINT", strlen("BIGINT"))) {
+                pos += sprintf(buff + pos, "%s,", rand_bigint_str());
+            } else if (0 == strncasecmp(stbInfo->columns[c].dataType,
+                        "FLOAT", strlen("FLOAT"))) {
+                if (g_args.demo_mode) {
+                    if (c == 0) {
+                        tmp = demo_current_float_str();
+                    } else {
+                        tmp = demo_phase_float_str();
+                    }
+                } else {
+                    tmp = rand_float_str();
+                }
+                pos += sprintf(buff + pos, "%s,", tmp);
+            } else if (0 == strncasecmp(stbInfo->columns[c].dataType,
+                        "DOUBLE", strlen("DOUBLE"))) {
+                pos += sprintf(buff + pos, "%s,", rand_double_str());
+            } else if (0 == strncasecmp(stbInfo->columns[c].dataType,
+                        "SMALLINT", strlen("SMALLINT"))) {
+                pos += sprintf(buff + pos, "%s,", rand_smallint_str());
+            } else if (0 == strncasecmp(stbInfo->columns[c].dataType,
+                        "TINYINT", strlen("TINYINT"))) {
+                pos += sprintf(buff + pos, "%s,", rand_tinyint_str());
+            } else if (0 == strncasecmp(stbInfo->columns[c].dataType,
+                        "BOOL", strlen("BOOL"))) {
+                pos += sprintf(buff + pos, "%s,", rand_bool_str());
+            } else if (0 == strncasecmp(stbInfo->columns[c].dataType,
+                        "TIMESTAMP", strlen("TIMESTAMP"))) {
+                pos += sprintf(buff + pos, "%s,", rand_bigint_str());
+            }
+        }
+        *(buff + pos - 1) = 0;
+        memcpy(stbInfo->sampleDataBuf + i * stbInfo->lenOfOneRow, buff, pos);
+    }
+
+    free(buff);
+    return 0;
+}
+
 static int prepareSampleDataForSTable(SSuperTable *stbInfo) {
-    char* sampleDataBuf = NULL;
 
-    sampleDataBuf = calloc(
+    stbInfo->sampleDataBuf = calloc(
             stbInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
-    if (sampleDataBuf == NULL) {
+    if (NULL == stbInfo->sampleDataBuf) {
         errorPrint2("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n",
                 __func__, __LINE__,
                 stbInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE,
@@ -5989,13 +6062,16 @@ static int prepareSampleDataForSTable(SSuperTable *stbInfo) {
         return -1;
     }
 
-    stbInfo->sampleDataBuf = sampleDataBuf;
-    int ret = readSampleFromCsvFileToMem(stbInfo);
+    int ret;
+    if (0 == strncasecmp(stbInfo->dataSource, "sample", strlen("sample"))) 
+        ret = readSampleFromCsvFileToMem(stbInfo);
+    else
+        ret = generateSampleMemoryFromRand(stbInfo);
 
     if (0 != ret) {
         errorPrint2("%s() LN%d, read sample from csv file failed.\n",
                 __func__, __LINE__);
-        tmfree(sampleDataBuf);
+        tmfree(stbInfo->sampleDataBuf);
         stbInfo->sampleDataBuf = NULL;
         return -1;
     }
@@ -7713,11 +7789,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) {
             pstr += len;
             remainderBufLen -= len;
 
+            // measure prepare + insert
+            startTs = taosGetTimestampUs();
+
             int32_t generated;
             if (stbInfo) {
                 if (stbInfo->iface == STMT_IFACE) {
                     if (sourceRand) {
-                        generated = prepareStbStmtRand(
+/*                        generated = prepareStbStmtRand(
                                 pThreadInfo,
                                 tableName,
                                 tableSeq,
@@ -7725,6 +7804,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) {
                                 insertRows,
                                 i, start_time
                                 );
+				*/
+                        generated = prepareStbStmtWithSample(
+                                pThreadInfo,
+                                tableName,
+                                tableSeq,
+                                g_args.num_of_RPR,
+                                insertRows, i, start_time,
+                                &(pThreadInfo->samplePos));
                     } else {
                         generated = prepareStbStmtWithSample(
                                 pThreadInfo,
@@ -7769,7 +7856,8 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) {
             start_time +=  generated * timeStampStep;
             pThreadInfo->totalInsertRows += generated;
 
-            startTs = taosGetTimestampUs();
+            // only measure insert
+            // startTs = taosGetTimestampUs();
 
             int32_t affectedRows = execInsert(pThreadInfo, generated);
 
@@ -7979,7 +8067,6 @@ static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec)
         return -1;
     }
 
-
     for (int i=0; i < MAX_SAMPLES_ONCE_FROM_FILE; i++) {
         char *bindArray = calloc(1, sizeof(TAOS_BIND) * (stbInfo->columnCount + 1));
         if (bindArray == NULL) {
@@ -7988,7 +8075,6 @@ static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec)
             return -1;
         }
 
-
         TAOS_BIND *bind;
         int cursor = 0;
 
@@ -8077,11 +8163,8 @@ static void startMultiThreadInsertData(int threads, char* db_name,
     debugPrint("%s() LN%d, start_time= %"PRId64"\n",
             __func__, __LINE__, start_time);
 
-    int64_t start = taosGetTimestampMs();
-
     // read sample data from file first
-    if ((stbInfo) && (0 == strncasecmp(stbInfo->dataSource,
-                    "sample", strlen("sample")))) {
+    if (stbInfo) {
         if (0 != prepareSampleDataForSTable(stbInfo)) {
             errorPrint2("%s() LN%d, prepare sample data for stable failed!\n",
                     __func__, __LINE__);
@@ -8229,8 +8312,7 @@ static void startMultiThreadInsertData(int threads, char* db_name,
 
         debugPrint("%s() LN%d, stmtBuffer: %s", __func__, __LINE__, stmtBuffer);
 
-        if ((stbInfo) && (0 == strncasecmp(stbInfo->dataSource,
-                        "sample", strlen("sample")))) {
+        if (stbInfo) {
             parseSampleFileToStmt(stbInfo, timePrec);
         }
     }
@@ -8315,6 +8397,8 @@ static void startMultiThreadInsertData(int threads, char* db_name,
 
     free(stmtBuffer);
 
+    int64_t start = taosGetTimestampMs();
+
     for (int i = 0; i < threads; i++) {
         pthread_join(pids[i], NULL);
     }
@@ -8357,22 +8441,22 @@ static void startMultiThreadInsertData(int threads, char* db_name,
     if (cntDelay == 0)    cntDelay = 1;
     avgDelay = (double)totalDelay / cntDelay;
 
-    int64_t end = taosGetTimestampMs();
+   int64_t end = taosGetTimestampMs();
     int64_t t = end - start;
 
-    double tInMs = t/1000.0;
+    double tInMs = (double) t / 1000.0;
 
     if (stbInfo) {
-        fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
+        fprintf(stderr, "Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
                 tInMs, stbInfo->totalInsertRows,
                 stbInfo->totalAffectedRows,
                 threads, db_name, stbInfo->sTblName,
-                (tInMs)?
+                (double) tInMs?
                 (double)(stbInfo->totalInsertRows/tInMs):FLT_MAX);
 
         if (g_fpOfInsertResult) {
             fprintf(g_fpOfInsertResult,
-                    "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
+                    "Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
                     tInMs, stbInfo->totalInsertRows,
                     stbInfo->totalAffectedRows,
                     threads, db_name, stbInfo->sTblName,
@@ -8380,7 +8464,7 @@ static void startMultiThreadInsertData(int threads, char* db_name,
                     (double)(stbInfo->totalInsertRows/tInMs):FLT_MAX);
         }
     } else {
-        fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
+        fprintf(stderr, "Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
                 tInMs, g_args.totalInsertRows,
                 g_args.totalAffectedRows,
                 threads, db_name,
@@ -8388,7 +8472,7 @@ static void startMultiThreadInsertData(int threads, char* db_name,
                 (double)(g_args.totalInsertRows/tInMs):FLT_MAX);
         if (g_fpOfInsertResult) {
             fprintf(g_fpOfInsertResult,
-                    "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
+                    "Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
                     tInMs, g_args.totalInsertRows,
                     g_args.totalAffectedRows,
                     threads, db_name,