From 29526b2b052f26baba66d4ad26d871ba91f3c247 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 29 Aug 2021 09:32:37 +0800 Subject: [PATCH] Hotfix/sangshuduo/td 5702 taosdemo remove memop for master (#7682) * [TD-5702]: taosdemo remove memory operation. * [TD-5702]: taosdemo remove memory operation. * add remainderBufLen to check row data generation. * row data generation with remainder buffer length checking. * git checkout --patch hotfix/sangshuduo/TD-5702-taosdemo-remove-memop taosdemo.c * optimized memop for rand too. * fix stmt rand file generation bug. * merge make_install.sh for homebrew. Co-authored-by: Shuduo Sang --- packaging/tools/make_install.sh | 96 ++++++++++++----------- src/kit/taosdemo/taosdemo.c | 130 ++++++++++++++++++++++++++------ 2 files changed, 157 insertions(+), 69 deletions(-) diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 55ca1174c9..07f1d61fad 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -19,6 +19,7 @@ else fi # Dynamic directory + data_dir="/var/lib/taos" if [ "$osType" != "Darwin" ]; then @@ -29,25 +30,32 @@ fi data_link_dir="/usr/local/taos/data" log_link_dir="/usr/local/taos/log" - -cfg_install_dir="/etc/taos" +if [ "$osType" != "Darwin" ]; then + cfg_install_dir="/etc/taos" +else + cfg_install_dir="/usr/local/Cellar/tdengine/${verNumber}/taos" +fi if [ "$osType" != "Darwin" ]; then bin_link_dir="/usr/bin" lib_link_dir="/usr/lib" lib64_link_dir="/usr/lib64" inc_link_dir="/usr/include" -else - bin_link_dir="/usr/local/bin" - lib_link_dir="/usr/local/lib" - inc_link_dir="/usr/local/include" fi #install main path -install_main_dir="/usr/local/taos" +if [ "$osType" != "Darwin" ]; then + install_main_dir="/usr/local/taos" +else + install_main_dir="/usr/local/Cellar/tdengine/${verNumber}" +fi # old bin dir -bin_dir="/usr/local/taos/bin" +if [ "$osType" != "Darwin" ]; then + bin_dir="/usr/local/taos/bin" +else + bin_dir="/usr/local/Cellar/tdengine/${verNumber}/bin" +fi service_config_dir="/etc/systemd/system" @@ -59,12 +67,11 @@ GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' csudo="" -if command -v sudo > /dev/null; then - csudo="sudo" -fi if [ "$osType" != "Darwin" ]; then - + if command -v sudo > /dev/null; then + csudo="sudo" + fi initd_mod=0 service_mod=2 if pidof systemd &> /dev/null; then @@ -137,18 +144,16 @@ function install_main_path() { function install_bin() { # Remove links - ${csudo} rm -f ${bin_link_dir}/taos || : - if [ "$osType" != "Darwin" ]; then + ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : ${csudo} rm -f ${bin_link_dir}/perfMonitor || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/set_core || : + ${csudo} rm -f ${bin_link_dir}/rmtaos || : fi - - ${csudo} rm -f ${bin_link_dir}/rmtaos || : - + ${csudo} cp -r ${binary_dir}/build/bin/* ${install_main_dir}/bin ${csudo} cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin @@ -162,20 +167,17 @@ function install_bin() { ${csudo} chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : - if [ "$osType" != "Darwin" ]; then + [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo} ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || : [ -x ${install_main_dir}/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : fi - + if [ "$osType" != "Darwin" ]; then - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : - else - [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/rmtaos || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : fi } @@ -222,7 +224,7 @@ function install_jemalloc() { fi if [ -d /etc/ld.so.conf.d ]; then - ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf + echo "/usr/local/lib" | ${csudo} tee /etc/ld.so.conf.d/jemalloc.conf ${csudo} ldconfig else echo "/etc/ld.so.conf.d not found!" @@ -248,10 +250,8 @@ function install_lib() { fi else ${csudo} cp -Rf ${binary_dir}/build/lib/libtaos.* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* - ${csudo} ln -sf ${install_main_dir}/driver/libtaos.1.dylib ${lib_link_dir}/libtaos.1.dylib - ${csudo} ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib fi - + install_jemalloc if [ "$osType" != "Darwin" ]; then @@ -261,10 +261,14 @@ function install_lib() { function install_header() { - ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + if [ "$osType" != "Darwin" ]; then + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + fi ${csudo} cp -f ${source_dir}/src/inc/taos.h ${source_dir}/src/inc/taoserror.h ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* - ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h - ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h + if [ "$osType" != "Darwin" ]; then + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h + fi } function install_config() { @@ -272,29 +276,30 @@ function install_config() { if [ ! -f ${cfg_install_dir}/taos.cfg ]; then ${csudo} mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/../cfg/taos.cfg ] && ${csudo} cp ${script_dir}/../cfg/taos.cfg ${cfg_install_dir} + [ -f ${script_dir}/../cfg/taos.cfg ] && + ${csudo} cp ${script_dir}/../cfg/taos.cfg ${cfg_install_dir} ${csudo} chmod 644 ${cfg_install_dir}/* fi ${csudo} cp -f ${script_dir}/../cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org - ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + + if [ "$osType" != "Darwin" ]; then ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg + fi } function install_log() { - ${csudo} rm -rf ${log_dir} || : - if [ "$osType" != "Darwin" ]; then - ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} - else - mkdir -p ${log_dir} && chmod 777 ${log_dir} + ${csudo} rm -rf ${log_dir} || : + ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir} + ${csudo} ln -s ${log_dir} ${install_main_dir}/log fi - - ${csudo} ln -s ${log_dir} ${install_main_dir}/log } function install_data() { - ${csudo} mkdir -p ${data_dir} - ${csudo} ln -s ${data_dir} ${install_main_dir}/data + if [ "$osType" != "Darwin" ]; then + ${csudo} mkdir -p ${data_dir} + ${csudo} ln -s ${data_dir} ${install_main_dir}/data + fi } function install_connector() { @@ -309,7 +314,6 @@ function install_connector() { echo "WARNING: go connector not found, please check if want to use it!" fi ${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector - ${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null } @@ -489,24 +493,24 @@ function install_TDengine() { else echo -e "${GREEN}Start to install TDEngine Client ...${NC}" fi - + install_main_path if [ "$osType" != "Darwin" ]; then install_data fi + install_log install_header install_lib install_connector install_examples - install_bin - + if [ "$osType" != "Darwin" ]; then install_service fi - + install_config if [ "$osType" != "Darwin" ]; then diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index d6cb0c5477..8c21f38e82 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -5976,12 +5976,85 @@ static int64_t generateData(char *recBuf, char **data_type, return (int32_t)strlen(recBuf); } +static int generateSampleMemoryFromRand(SSuperTable *stbInfo) +{ + char data[MAX_DATA_SIZE]; + memset(data, 0, MAX_DATA_SIZE); + + char *buff = malloc(stbInfo->lenOfOneRow); + if (NULL == buff) { + errorPrint2("%s() LN%d, memory allocation %"PRId64" bytes failed\n", + __func__, __LINE__, stbInfo->lenOfOneRow); + exit(EXIT_FAILURE); + } + + for (int i=0; i < MAX_SAMPLES_ONCE_FROM_FILE; i++) { + uint64_t pos = 0; + memset(buff, 0, stbInfo->lenOfOneRow); + + for (int c = 0; c < stbInfo->columnCount; c++) { + char *tmp; + if (0 == strncasecmp(stbInfo->columns[c].dataType, + "BINARY", strlen("BINARY"))) { + rand_string(data, stbInfo->columns[c].dataLen); + pos += sprintf(buff + pos, "%s,", data); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "NCHAR", strlen("NCHAR"))) { + rand_string(data, stbInfo->columns[c].dataLen); + pos += sprintf(buff + pos, "%s,", data); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "INT", strlen("INT"))) { + if ((g_args.demo_mode) && (c == 1)) { + tmp = demo_voltage_int_str(); + } else { + tmp = rand_int_str(); + } + pos += sprintf(buff + pos, "%s,", tmp); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "BIGINT", strlen("BIGINT"))) { + pos += sprintf(buff + pos, "%s,", rand_bigint_str()); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "FLOAT", strlen("FLOAT"))) { + if (g_args.demo_mode) { + if (c == 0) { + tmp = demo_current_float_str(); + } else { + tmp = demo_phase_float_str(); + } + } else { + tmp = rand_float_str(); + } + pos += sprintf(buff + pos, "%s,", tmp); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "DOUBLE", strlen("DOUBLE"))) { + pos += sprintf(buff + pos, "%s,", rand_double_str()); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "SMALLINT", strlen("SMALLINT"))) { + pos += sprintf(buff + pos, "%s,", rand_smallint_str()); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "TINYINT", strlen("TINYINT"))) { + pos += sprintf(buff + pos, "%s,", rand_tinyint_str()); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "BOOL", strlen("BOOL"))) { + pos += sprintf(buff + pos, "%s,", rand_bool_str()); + } else if (0 == strncasecmp(stbInfo->columns[c].dataType, + "TIMESTAMP", strlen("TIMESTAMP"))) { + pos += sprintf(buff + pos, "%s,", rand_bigint_str()); + } + } + *(buff + pos - 1) = 0; + memcpy(stbInfo->sampleDataBuf + i * stbInfo->lenOfOneRow, buff, pos); + } + + free(buff); + return 0; +} + static int prepareSampleDataForSTable(SSuperTable *stbInfo) { - char* sampleDataBuf = NULL; - sampleDataBuf = calloc( + stbInfo->sampleDataBuf = calloc( stbInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1); - if (sampleDataBuf == NULL) { + if (NULL == stbInfo->sampleDataBuf) { errorPrint2("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n", __func__, __LINE__, stbInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, @@ -5989,13 +6062,16 @@ static int prepareSampleDataForSTable(SSuperTable *stbInfo) { return -1; } - stbInfo->sampleDataBuf = sampleDataBuf; - int ret = readSampleFromCsvFileToMem(stbInfo); + int ret; + if (0 == strncasecmp(stbInfo->dataSource, "sample", strlen("sample"))) + ret = readSampleFromCsvFileToMem(stbInfo); + else + ret = generateSampleMemoryFromRand(stbInfo); if (0 != ret) { errorPrint2("%s() LN%d, read sample from csv file failed.\n", __func__, __LINE__); - tmfree(sampleDataBuf); + tmfree(stbInfo->sampleDataBuf); stbInfo->sampleDataBuf = NULL; return -1; } @@ -7713,11 +7789,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { pstr += len; remainderBufLen -= len; + // measure prepare + insert + startTs = taosGetTimestampUs(); + int32_t generated; if (stbInfo) { if (stbInfo->iface == STMT_IFACE) { if (sourceRand) { - generated = prepareStbStmtRand( +/* generated = prepareStbStmtRand( pThreadInfo, tableName, tableSeq, @@ -7725,6 +7804,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { insertRows, i, start_time ); + */ + generated = prepareStbStmtWithSample( + pThreadInfo, + tableName, + tableSeq, + g_args.num_of_RPR, + insertRows, i, start_time, + &(pThreadInfo->samplePos)); } else { generated = prepareStbStmtWithSample( pThreadInfo, @@ -7769,7 +7856,8 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { start_time += generated * timeStampStep; pThreadInfo->totalInsertRows += generated; - startTs = taosGetTimestampUs(); + // only measure insert + // startTs = taosGetTimestampUs(); int32_t affectedRows = execInsert(pThreadInfo, generated); @@ -7979,7 +8067,6 @@ static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec) return -1; } - for (int i=0; i < MAX_SAMPLES_ONCE_FROM_FILE; i++) { char *bindArray = calloc(1, sizeof(TAOS_BIND) * (stbInfo->columnCount + 1)); if (bindArray == NULL) { @@ -7988,7 +8075,6 @@ static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec) return -1; } - TAOS_BIND *bind; int cursor = 0; @@ -8077,11 +8163,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, debugPrint("%s() LN%d, start_time= %"PRId64"\n", __func__, __LINE__, start_time); - int64_t start = taosGetTimestampMs(); - // read sample data from file first - if ((stbInfo) && (0 == strncasecmp(stbInfo->dataSource, - "sample", strlen("sample")))) { + if (stbInfo) { if (0 != prepareSampleDataForSTable(stbInfo)) { errorPrint2("%s() LN%d, prepare sample data for stable failed!\n", __func__, __LINE__); @@ -8229,8 +8312,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, debugPrint("%s() LN%d, stmtBuffer: %s", __func__, __LINE__, stmtBuffer); - if ((stbInfo) && (0 == strncasecmp(stbInfo->dataSource, - "sample", strlen("sample")))) { + if (stbInfo) { parseSampleFileToStmt(stbInfo, timePrec); } } @@ -8315,6 +8397,8 @@ static void startMultiThreadInsertData(int threads, char* db_name, free(stmtBuffer); + int64_t start = taosGetTimestampMs(); + for (int i = 0; i < threads; i++) { pthread_join(pids[i], NULL); } @@ -8357,22 +8441,22 @@ static void startMultiThreadInsertData(int threads, char* db_name, if (cntDelay == 0) cntDelay = 1; avgDelay = (double)totalDelay / cntDelay; - int64_t end = taosGetTimestampMs(); + int64_t end = taosGetTimestampMs(); int64_t t = end - start; - double tInMs = t/1000.0; + double tInMs = (double) t / 1000.0; if (stbInfo) { - fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", + fprintf(stderr, "Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", tInMs, stbInfo->totalInsertRows, stbInfo->totalAffectedRows, threads, db_name, stbInfo->sTblName, - (tInMs)? + (double) tInMs? (double)(stbInfo->totalInsertRows/tInMs):FLT_MAX); if (g_fpOfInsertResult) { fprintf(g_fpOfInsertResult, - "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", + "Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n", tInMs, stbInfo->totalInsertRows, stbInfo->totalAffectedRows, threads, db_name, stbInfo->sTblName, @@ -8380,7 +8464,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, (double)(stbInfo->totalInsertRows/tInMs):FLT_MAX); } } else { - fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", + fprintf(stderr, "Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", tInMs, g_args.totalInsertRows, g_args.totalAffectedRows, threads, db_name, @@ -8388,7 +8472,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, (double)(g_args.totalInsertRows/tInMs):FLT_MAX); if (g_fpOfInsertResult) { fprintf(g_fpOfInsertResult, - "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", + "Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n", tInMs, g_args.totalInsertRows, g_args.totalAffectedRows, threads, db_name, -- GitLab