Skip to content

T
TDengine

taosdata / TDengine
1 年多 前同步成功

通知 1185
Star 22016
Fork 4786
T
TDengine
未验证 提交 29526b2b 编写于 作者: sangshuduo's avatar sangshuduo 提交者： GitHub
浏览文件
操作

Hotfix/sangshuduo/td 5702 taosdemo remove memop for master (#7682) 

* [TD-5702]<fix>: taosdemo remove memory operation.

* [TD-5702]<fix>: taosdemo remove memory operation.

* add remainderBufLen to check row data generation.

* row data generation with remainder buffer length checking.

* git checkout --patch hotfix/sangshuduo/TD-5702-taosdemo-remove-memop taosdemo.c

* optimized memop for rand too.

* fix stmt rand file generation bug.

* merge make_install.sh for homebrew.
Co-authored-by: NShuduo Sang <sdsang@taosdata.com>
上级 b5bea0f3
隐藏空白更改
内联 并排
Showing with 157 addition and 69 deletion
packaging/tools/make_install.sh
浏览文件 @ 29526b2b
......@@ -19,6 +19,7 @@ else
fi
# Dynamic directory
data_dir="/var/lib/taos"
if [ "$osType" != "Darwin" ]; then
......@@ -29,25 +30,32 @@ fi
data_link_dir="/usr/local/taos/data"
log_link_dir="/usr/local/taos/log"
cfg_install_dir="/etc/taos"
if [ "$osType" != "Darwin" ]; then
cfg_install_dir="/etc/taos"
else
cfg_install_dir="/usr/local/Cellar/tdengine/${verNumber}/taos"
fi
if [ "$osType" != "Darwin" ]; then
bin_link_dir="/usr/bin"
lib_link_dir="/usr/lib"
lib64_link_dir="/usr/lib64"
inc_link_dir="/usr/include"
else
bin_link_dir="/usr/local/bin"
lib_link_dir="/usr/local/lib"
inc_link_dir="/usr/local/include"
fi
#install main path
install_main_dir="/usr/local/taos"
if [ "$osType" != "Darwin" ]; then
install_main_dir="/usr/local/taos"
else
install_main_dir="/usr/local/Cellar/tdengine/${verNumber}"
fi
# old bin dir
bin_dir="/usr/local/taos/bin"
if [ "$osType" != "Darwin" ]; then
bin_dir="/usr/local/taos/bin"
else
bin_dir="/usr/local/Cellar/tdengine/${verNumber}/bin"
fi
service_config_dir="/etc/systemd/system"
......@@ -59,12 +67,11 @@ GREEN_UNDERLINE='\033[4;32m'
NC='\033[0m'
csudo=""
if command -v sudo > /dev/null; then
csudo="sudo"
fi
if [ "$osType" != "Darwin" ]; then
if command -v sudo > /dev/null; then
csudo="sudo"
fi
initd_mod=0
service_mod=2
if pidof systemd &> /dev/null; then
......@@ -137,18 +144,16 @@ function install_main_path() {
function install_bin() {
# Remove links
${csudo} rm -f ${bin_link_dir}/taos || :
if [ "$osType" != "Darwin" ]; then
${csudo} rm -f ${bin_link_dir}/taos || :
${csudo} rm -f ${bin_link_dir}/taosd || :
${csudo} rm -f ${bin_link_dir}/taosdemo || :
${csudo} rm -f ${bin_link_dir}/perfMonitor || :
${csudo} rm -f ${bin_link_dir}/taosdump || :
${csudo} rm -f ${bin_link_dir}/set_core || :
${csudo} rm -f ${bin_link_dir}/rmtaos || :
fi
${csudo} rm -f ${bin_link_dir}/rmtaos || :
${csudo} cp -r ${binary_dir}/build/bin/* ${install_main_dir}/bin
${csudo} cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin
......@@ -162,20 +167,17 @@ function install_bin() {
${csudo} chmod 0555 ${install_main_dir}/bin/*
#Make link
[ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || :
if [ "$osType" != "Darwin" ]; then
[ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || :
[ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || :
[ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || :
[ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || :
[ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo} ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || :
[ -x ${install_main_dir}/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || :
fi
if [ "$osType" != "Darwin" ]; then
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || :
else
[ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/rmtaos || :
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || :
fi
}
......@@ -222,7 +224,7 @@ function install_jemalloc() {
fi
if [ -d /etc/ld.so.conf.d ]; then
${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf
echo "/usr/local/lib" | ${csudo} tee /etc/ld.so.conf.d/jemalloc.conf
${csudo} ldconfig
else
echo "/etc/ld.so.conf.d not found!"
......@@ -248,10 +250,8 @@ function install_lib() {
fi
else
${csudo} cp -Rf ${binary_dir}/build/lib/libtaos.* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/*
${csudo} ln -sf ${install_main_dir}/driver/libtaos.1.dylib ${lib_link_dir}/libtaos.1.dylib
${csudo} ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib
fi
install_jemalloc
if [ "$osType" != "Darwin" ]; then
......@@ -261,10 +261,14 @@ function install_lib() {
function install_header() {
${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || :
if [ "$osType" != "Darwin" ]; then
${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || :
fi
${csudo} cp -f ${source_dir}/src/inc/taos.h ${source_dir}/src/inc/taoserror.h ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/*
${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
if [ "$osType" != "Darwin" ]; then
${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h
${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h
fi
}
function install_config() {
......@@ -272,29 +276,30 @@ function install_config() {
if [ ! -f ${cfg_install_dir}/taos.cfg ]; then
${csudo} mkdir -p ${cfg_install_dir}
[ -f ${script_dir}/../cfg/taos.cfg ] && ${csudo} cp ${script_dir}/../cfg/taos.cfg ${cfg_install_dir}
[ -f ${script_dir}/../cfg/taos.cfg ] &&
${csudo} cp ${script_dir}/../cfg/taos.cfg ${cfg_install_dir}
${csudo} chmod 644 ${cfg_install_dir}/*
fi
${csudo} cp -f ${script_dir}/../cfg/taos.cfg ${install_main_dir}/cfg/taos.cfg.org
${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg
if [ "$osType" != "Darwin" ]; then ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${install_main_dir}/cfg
fi
}
function install_log() {
${csudo} rm -rf ${log_dir} || :
if [ "$osType" != "Darwin" ]; then
${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir}
else
mkdir -p ${log_dir} && chmod 777 ${log_dir}
${csudo} rm -rf ${log_dir} || :
${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir}
${csudo} ln -s ${log_dir} ${install_main_dir}/log
fi
${csudo} ln -s ${log_dir} ${install_main_dir}/log
}
function install_data() {
${csudo} mkdir -p ${data_dir}
${csudo} ln -s ${data_dir} ${install_main_dir}/data
if [ "$osType" != "Darwin" ]; then
${csudo} mkdir -p ${data_dir}
${csudo} ln -s ${data_dir} ${install_main_dir}/data
fi
}
function install_connector() {
......@@ -309,7 +314,6 @@ function install_connector() {
echo "WARNING: go connector not found, please check if want to use it!"
fi
${csudo} cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector
${csudo} cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &> /dev/null && ${csudo} chmod 777 ${install_main_dir}/connector/*.jar || echo &> /dev/null
}
......@@ -489,24 +493,24 @@ function install_TDengine() {
else
echo -e "${GREEN}Start to install TDEngine Client ...${NC}"
fi
install_main_path
if [ "$osType" != "Darwin" ]; then
install_data
fi
install_log
install_header
install_lib
install_connector
install_examples
install_bin
if [ "$osType" != "Darwin" ]; then
install_service
fi
install_config
if [ "$osType" != "Darwin" ]; then
......
src/kit/taosdemo/taosdemo.c
浏览文件 @ 29526b2b
......@@ -5976,12 +5976,85 @@ static int64_t generateData(char *recBuf, char **data_type,
return (int32_t)strlen(recBuf);
}
static int generateSampleMemoryFromRand(SSuperTable *stbInfo)
{
char data[MAX_DATA_SIZE];
memset(data, 0, MAX_DATA_SIZE);
char *buff = malloc(stbInfo->lenOfOneRow);
if (NULL == buff) {
errorPrint2("%s() LN%d, memory allocation %"PRId64" bytes failed\n",
__func__, __LINE__, stbInfo->lenOfOneRow);
exit(EXIT_FAILURE);
}
for (int i=0; i < MAX_SAMPLES_ONCE_FROM_FILE; i++) {
uint64_t pos = 0;
memset(buff, 0, stbInfo->lenOfOneRow);
for (int c = 0; c < stbInfo->columnCount; c++) {
char *tmp;
if (0 == strncasecmp(stbInfo->columns[c].dataType,
"BINARY", strlen("BINARY"))) {
rand_string(data, stbInfo->columns[c].dataLen);
pos += sprintf(buff + pos, "%s,", data);
} else if (0 == strncasecmp(stbInfo->columns[c].dataType,
"NCHAR", strlen("NCHAR"))) {
rand_string(data, stbInfo->columns[c].dataLen);
pos += sprintf(buff + pos, "%s,", data);
} else if (0 == strncasecmp(stbInfo->columns[c].dataType,
"INT", strlen("INT"))) {
if ((g_args.demo_mode) && (c == 1)) {
tmp = demo_voltage_int_str();
} else {
tmp = rand_int_str();
}
pos += sprintf(buff + pos, "%s,", tmp);
} else if (0 == strncasecmp(stbInfo->columns[c].dataType,
"BIGINT", strlen("BIGINT"))) {
pos += sprintf(buff + pos, "%s,", rand_bigint_str());
} else if (0 == strncasecmp(stbInfo->columns[c].dataType,
"FLOAT", strlen("FLOAT"))) {
if (g_args.demo_mode) {
if (c == 0) {
tmp = demo_current_float_str();
} else {
tmp = demo_phase_float_str();
}
} else {
tmp = rand_float_str();
}
pos += sprintf(buff + pos, "%s,", tmp);
} else if (0 == strncasecmp(stbInfo->columns[c].dataType,
"DOUBLE", strlen("DOUBLE"))) {
pos += sprintf(buff + pos, "%s,", rand_double_str());
} else if (0 == strncasecmp(stbInfo->columns[c].dataType,
"SMALLINT", strlen("SMALLINT"))) {
pos += sprintf(buff + pos, "%s,", rand_smallint_str());
} else if (0 == strncasecmp(stbInfo->columns[c].dataType,
"TINYINT", strlen("TINYINT"))) {
pos += sprintf(buff + pos, "%s,", rand_tinyint_str());
} else if (0 == strncasecmp(stbInfo->columns[c].dataType,
"BOOL", strlen("BOOL"))) {
pos += sprintf(buff + pos, "%s,", rand_bool_str());
} else if (0 == strncasecmp(stbInfo->columns[c].dataType,
"TIMESTAMP", strlen("TIMESTAMP"))) {
pos += sprintf(buff + pos, "%s,", rand_bigint_str());
}
}
*(buff + pos - 1) = 0;
memcpy(stbInfo->sampleDataBuf + i * stbInfo->lenOfOneRow, buff, pos);
}
free(buff);
return 0;
}
static int prepareSampleDataForSTable(SSuperTable *stbInfo) {
char* sampleDataBuf = NULL;
sampleDataBuf = calloc(
stbInfo->sampleDataBuf = calloc(
stbInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1);
if (sampleDataBuf == NULL) {
if (NULL == stbInfo->sampleDataBuf) {
errorPrint2("%s() LN%d, Failed to calloc %"PRIu64" Bytes, reason:%s\n",
__func__, __LINE__,
stbInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE,
......@@ -5989,13 +6062,16 @@ static int prepareSampleDataForSTable(SSuperTable *stbInfo) {
return -1;
}
stbInfo->sampleDataBuf = sampleDataBuf;
int ret = readSampleFromCsvFileToMem(stbInfo);
int ret;
if (0 == strncasecmp(stbInfo->dataSource, "sample", strlen("sample")))
ret = readSampleFromCsvFileToMem(stbInfo);
else
ret = generateSampleMemoryFromRand(stbInfo);
if (0 != ret) {
errorPrint2("%s() LN%d, read sample from csv file failed.\n",
__func__, __LINE__);
tmfree(sampleDataBuf);
tmfree(stbInfo->sampleDataBuf);
stbInfo->sampleDataBuf = NULL;
return -1;
}
......@@ -7713,11 +7789,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) {
pstr += len;
remainderBufLen -= len;
// measure prepare + insert
startTs = taosGetTimestampUs();
int32_t generated;
if (stbInfo) {
if (stbInfo->iface == STMT_IFACE) {
if (sourceRand) {
generated = prepareStbStmtRand(
/* generated = prepareStbStmtRand(
pThreadInfo,
tableName,
tableSeq,
......@@ -7725,6 +7804,14 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) {
insertRows,
i, start_time
);
*/
generated = prepareStbStmtWithSample(
pThreadInfo,
tableName,
tableSeq,
g_args.num_of_RPR,
insertRows, i, start_time,
&(pThreadInfo->samplePos));
} else {
generated = prepareStbStmtWithSample(
pThreadInfo,
......@@ -7769,7 +7856,8 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) {
start_time += generated * timeStampStep;
pThreadInfo->totalInsertRows += generated;
startTs = taosGetTimestampUs();
// only measure insert
// startTs = taosGetTimestampUs();
int32_t affectedRows = execInsert(pThreadInfo, generated);
......@@ -7979,7 +8067,6 @@ static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec)
return -1;
}
for (int i=0; i < MAX_SAMPLES_ONCE_FROM_FILE; i++) {
char *bindArray = calloc(1, sizeof(TAOS_BIND) * (stbInfo->columnCount + 1));
if (bindArray == NULL) {
......@@ -7988,7 +8075,6 @@ static int parseSampleFileToStmt(SSuperTable *stbInfo, uint32_t timePrec)
return -1;
}
TAOS_BIND *bind;
int cursor = 0;
......@@ -8077,11 +8163,8 @@ static void startMultiThreadInsertData(int threads, char* db_name,
debugPrint("%s() LN%d, start_time= %"PRId64"\n",
__func__, __LINE__, start_time);
int64_t start = taosGetTimestampMs();
// read sample data from file first
if ((stbInfo) && (0 == strncasecmp(stbInfo->dataSource,
"sample", strlen("sample")))) {
if (stbInfo) {
if (0 != prepareSampleDataForSTable(stbInfo)) {
errorPrint2("%s() LN%d, prepare sample data for stable failed!\n",
__func__, __LINE__);
......@@ -8229,8 +8312,7 @@ static void startMultiThreadInsertData(int threads, char* db_name,
debugPrint("%s() LN%d, stmtBuffer: %s", __func__, __LINE__, stmtBuffer);
if ((stbInfo) && (0 == strncasecmp(stbInfo->dataSource,
"sample", strlen("sample")))) {
if (stbInfo) {
parseSampleFileToStmt(stbInfo, timePrec);
}
}
......@@ -8315,6 +8397,8 @@ static void startMultiThreadInsertData(int threads, char* db_name,
free(stmtBuffer);
int64_t start = taosGetTimestampMs();
for (int i = 0; i < threads; i++) {
pthread_join(pids[i], NULL);
}
......@@ -8357,22 +8441,22 @@ static void startMultiThreadInsertData(int threads, char* db_name,
if (cntDelay == 0) cntDelay = 1;
avgDelay = (double)totalDelay / cntDelay;
int64_t end = taosGetTimestampMs();
int64_t end = taosGetTimestampMs();
int64_t t = end - start;
double tInMs = t/1000.0;
double tInMs = (double) t / 1000.0;
if (stbInfo) {
fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
fprintf(stderr, "Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
tInMs, stbInfo->totalInsertRows,
stbInfo->totalAffectedRows,
threads, db_name, stbInfo->sTblName,
(tInMs)?
(double) tInMs?
(double)(stbInfo->totalInsertRows/tInMs):FLT_MAX);
if (g_fpOfInsertResult) {
fprintf(g_fpOfInsertResult,
"Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
"Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s.%s. %.2f records/second\n\n",
tInMs, stbInfo->totalInsertRows,
stbInfo->totalAffectedRows,
threads, db_name, stbInfo->sTblName,
......@@ -8380,7 +8464,7 @@ static void startMultiThreadInsertData(int threads, char* db_name,
(double)(stbInfo->totalInsertRows/tInMs):FLT_MAX);
}
} else {
fprintf(stderr, "Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
fprintf(stderr, "Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
tInMs, g_args.totalInsertRows,
g_args.totalAffectedRows,
threads, db_name,
......@@ -8388,7 +8472,7 @@ static void startMultiThreadInsertData(int threads, char* db_name,
(double)(g_args.totalInsertRows/tInMs):FLT_MAX);
if (g_fpOfInsertResult) {
fprintf(g_fpOfInsertResult,
"Spent %.2f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
"Spent %.4f seconds to insert rows: %"PRIu64", affected rows: %"PRIu64" with %d thread(s) into %s %.2f records/second\n\n",
tInMs, g_args.totalInsertRows,
g_args.totalAffectedRows,
threads, db_name,
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或