Skip to content

T
TDengine

慢慢CG / TDengine
与 Fork 源项目一致

Fork自 taosdata / TDengine

通知 1
Star 0
Fork 0
T
TDengine
提交 57c1f20f 编写于 作者: S Shuduo Sang
浏览文件
操作

[TD-3192] <feature>: support stb limit and offset. create tables works.

上级 a9e5d2b7
显示空白变更内容
内联 并排
Showing with 123 addition and 100 deletion
src/kit/taosdemo/taosdemo.c
浏览文件 @ 57c1f20f
......@@ -1797,8 +1797,7 @@ int postProceSql(char* host, uint16_t port, char* sqlstr)
return 0;
}
char* getTagValueFromTagSample( SSuperTable* stbInfo, int tagUsePos) {
static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) {
char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
if (NULL == dataBuf) {
printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
......@@ -1806,12 +1805,13 @@ char* getTagValueFromTagSample( SSuperTable* stbInfo, int tagUsePos) {
}
int dataLen = 0;
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
"(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos);
return dataBuf;
}
char* generateTagVaulesForStb(SSuperTable* stbInfo) {
static char* generateTagVaulesForStb(SSuperTable* stbInfo) {
char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1);
if (NULL == dataBuf) {
printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1);
......@@ -1821,9 +1821,11 @@ char* generateTagVaulesForStb(SSuperTable* stbInfo) {
int dataLen = 0;
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(");
for (int i = 0; i < stbInfo->tagCount; i++) {
if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", 6)) || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", 5))) {
if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary")))
|| (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) {
if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) {
printf("binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN);
printf("binary or nchar length overflow, max size:%u\n",
(uint32_t)TSDB_MAX_BINARY_LEN);
tmfree(dataBuf);
return NULL;
}
......@@ -1835,30 +1837,48 @@ char* generateTagVaulesForStb(SSuperTable* stbInfo) {
return NULL;
}
rand_string(buf, stbInfo->tags[i].dataLen);
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "\'%s\', ", buf);
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
"\'%s\', ", buf);
tmfree(buf);
} else if (0 == strncasecmp(stbInfo->tags[i].dataType, "int", 3)) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_int());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bigint", 6)) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64", ", rand_bigint());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType, "float", 5)) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%f, ", rand_float());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType, "double", 6)) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%f, ", rand_double());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType, "smallint", 8)) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_smallint());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType, "tinyint", 7)) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_tinyint());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bool", 4)) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_bool());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType, "timestamp", 4)) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64", ", rand_bigint());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType,
"int", strlen("int"))) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
"%d, ", rand_int());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType,
"bigint", strlen("bigint"))) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
"%"PRId64", ", rand_bigint());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType,
"float", strlen("float"))) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
"%f, ", rand_float());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType,
"double", strlen("double"))) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
"%f, ", rand_double());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType,
"smallint", strlen("smallint"))) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
"%d, ", rand_smallint());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType,
"tinyint", strlen("tinyint"))) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
"%d, ", rand_tinyint());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType,
"bool", strlen("bool"))) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
"%d, ", rand_bool());
} else if (0 == strncasecmp(stbInfo->tags[i].dataType,
"timestamp", strlen("timestamp"))) {
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen,
"%"PRId64", ", rand_bigint());
} else {
printf("No support data type: %s\n", stbInfo->tags[i].dataType);
tmfree(dataBuf);
return NULL;
}
}
dataLen -= 2;
dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")");
return dataBuf;
......@@ -2423,7 +2443,7 @@ static void* createTable(void *sarg)
}
int startMultiThreadCreateChildTable(
char* cols, int threads, int ntables,
char* cols, int threads, int startFrom, int ntables,
char* db_name, SSuperTable* superTblInfo) {
pthread_t *pids = malloc(threads * sizeof(pthread_t));
threadInfo *infos = malloc(threads * sizeof(threadInfo));
......@@ -2446,7 +2466,7 @@ int startMultiThreadCreateChildTable(
int b = 0;
b = ntables % threads;
int last = 0;
int last = startFrom;
for (int i = 0; i < threads; i++) {
threadInfo *t_info = infos + i;
t_info->threadID = i;
......@@ -2505,12 +2525,23 @@ static void createChildTables() {
verbosePrint("%s() LN%d: %s\n", __func__, __LINE__,
g_Dbs.db[i].superTbls[j].colsOfCreateChildTable);
int startFrom;
if (g_Dbs.db[i].superTbls[j].childTblOffset) {
startFrom = g_Dbs.db[i].superTbls[j].childTblOffset;
g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblLimit;
} else {
startFrom = 0;
g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;
}
verbosePrint("%s() LN%d: create %d child tables from %d\n", __func__, __LINE__,
g_totalChildTables, startFrom);
startMultiThreadCreateChildTable(
g_Dbs.db[i].superTbls[j].colsOfCreateChildTable,
g_Dbs.threadCountByCreateTbl,
g_Dbs.db[i].superTbls[j].childTblCount,
startFrom,
g_totalChildTables,
g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j]));
g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount;
}
} else {
// normal table
......@@ -2538,6 +2569,7 @@ static void createChildTables() {
startMultiThreadCreateChildTable(
tblColsBuf,
g_Dbs.threadCountByCreateTbl,
0,
g_args.num_of_tables,
g_Dbs.db[i].dbName,
NULL);
......@@ -2545,30 +2577,6 @@ static void createChildTables() {
}
}
/*
static int taosGetLineNum(const char *fileName)
{
int lineNum = 0;
char cmd[1024] = { 0 };
char buf[1024] = { 0 };
sprintf(cmd, "wc -l %s", fileName);
FILE *fp = popen(cmd, "r");
if (fp == NULL) {
fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
return lineNum;
}
if (fgets(buf, sizeof(buf), fp)) {
int index = strchr((const char*)buf, ' ') - buf;
buf[index] = '\0';
lineNum = atoi(buf);
}
pclose(fp);
return lineNum;
}
*/
/*
Read 10000 lines at most. If more than 10000 lines, continue to read after using
*/
......@@ -4010,7 +4018,7 @@ static void syncWriteForNumberOfTblInOneSql(
int64_t et = 0xffffffff;
for (int i = 0; i < superTblInfo->insertRows;) {
int32_t tbl_id = 0;
for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; ) {
for (int tableID = winfo->start_table_id; tableID <= winfo->end_table_id; ) {
int64_t start_time = 0;
int inserted = i;
......@@ -4019,12 +4027,12 @@ static void syncWriteForNumberOfTblInOneSql(
memset(buffer, 0, superTblInfo->maxSqlLen);
char *pstr = buffer;
int32_t end_tbl_id = tID + numberOfTblInOneSql;
int32_t end_tbl_id = tableID + numberOfTblInOneSql;
if (end_tbl_id > winfo->end_table_id) {
end_tbl_id = winfo->end_table_id+1;
}
for (tbl_id = tID; tbl_id < end_tbl_id; tbl_id++) {
for (tbl_id = tableID; tbl_id < end_tbl_id; tbl_id++) {
sampleUsePos = samplePos;
if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) {
char* tagsValBuf = NULL;
......@@ -4132,7 +4140,7 @@ static void syncWriteForNumberOfTblInOneSql(
if (inserted >= superTblInfo->insertRows ||
(superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128)) {
tID = tbl_id + 1;
tableID = tbl_id + 1;
printf("config rowsPerTbl and numberOfTblInOneSql not match with max_sql_lenth, please reconfig![lenOfOneRow:%d]\n",
superTblInfo->lenOfOneRow);
goto send_to_server;
......@@ -4140,7 +4148,7 @@ static void syncWriteForNumberOfTblInOneSql(
}
}
tID = tbl_id;
tableID = tbl_id;
inserted += superTblInfo->rowsPerTbl;
send_to_server:
......@@ -4209,7 +4217,7 @@ send_to_server:
break;
}
if (tID > winfo->end_table_id) {
if (tableID > winfo->end_table_id) {
if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) {
samplePos = sampleUsePos;
}
......@@ -4346,7 +4354,7 @@ static int execInsert(threadInfo *winfo, char *buffer, int k)
return affectedRows;
}
static int generateDataBuffer(int32_t threadID, threadInfo *pThreadInfo, char *buffer,
static int generateDataBuffer(int32_t tableID, threadInfo *pThreadInfo, char *buffer,
int64_t insertRows,
int64_t startFrom, int64_t startTime, int *pSampleUsePos)
{
......@@ -4362,6 +4370,8 @@ static int generateDataBuffer(int32_t threadID, threadInfo *pThreadInfo, char *b
}
}
assert(buffer != NULL);
memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len);
char *pstr = buffer;
......@@ -4374,7 +4384,7 @@ static int generateDataBuffer(int32_t threadID, threadInfo *pThreadInfo, char *b
} else {
tagsValBuf = getTagValueFromTagSample(
superTblInfo,
threadID % superTblInfo->tagSampleCount);
tableID % superTblInfo->tagSampleCount);
}
if (NULL == tagsValBuf) {
fprintf(stderr, "tag buf failed to allocate memory\n");
......@@ -4386,7 +4396,7 @@ static int generateDataBuffer(int32_t threadID, threadInfo *pThreadInfo, char *b
"insert into %s.%s%d using %s.%s tags %s values",
pThreadInfo->db_name,
superTblInfo->childTblPrefix,
threadID,
tableID,
pThreadInfo->db_name,
superTblInfo->sTblName,
tagsValBuf);
......@@ -4396,14 +4406,14 @@ static int generateDataBuffer(int32_t threadID, threadInfo *pThreadInfo, char *b
superTblInfo->maxSqlLen,
"insert into %s.%s values",
pThreadInfo->db_name,
superTblInfo->childTblName + threadID * TSDB_TABLE_NAME_LEN);
superTblInfo->childTblName + tableID * TSDB_TABLE_NAME_LEN);
} else {
pstr += snprintf(pstr,
(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len),
"insert into %s.%s%d values",
pThreadInfo->db_name,
superTblInfo?superTblInfo->childTblPrefix:g_args.tb_prefix,
threadID);
tableID);
}
} else {
pstr += snprintf(pstr,
......@@ -4411,7 +4421,7 @@ static int generateDataBuffer(int32_t threadID, threadInfo *pThreadInfo, char *b
"insert into %s.%s%d values",
pThreadInfo->db_name,
superTblInfo?superTblInfo->childTblPrefix:g_args.tb_prefix,
threadID);
tableID);
}
int k;
......@@ -4486,7 +4496,6 @@ static int generateDataBuffer(int32_t threadID, threadInfo *pThreadInfo, char *b
k++;
startFrom ++;
debugPrint("%s() LN%d k=%d startFrom=%ld insertRows=%ld\n", __func__, __LINE__, k, startFrom, insertRows);
if (startFrom >= insertRows)
break;
}
......@@ -4523,7 +4532,7 @@ static void* syncWrite(void *sarg) {
char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1);
if (NULL == buffer) {
fprintf(stderr, "Failed to calloc %d Bytes, reason:%s\n",
fprintf(stderr, "Failed to alloc %d Bytes, reason:%s\n",
superTblInfo->maxSqlLen,
strerror(errno));
tmfree(superTblInfo->sampleDataBuf);
......@@ -4534,7 +4543,8 @@ static void* syncWrite(void *sarg) {
int64_t startTs = taosGetTimestampUs();
int64_t endTs;
int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval;
int insert_interval = superTblInfo?superTblInfo->insertInterval:
g_args.insert_interval;
uint64_t st = 0;
uint64_t et = 0xffffffff;
......@@ -4543,12 +4553,12 @@ static void* syncWrite(void *sarg) {
int sampleUsePos;
if (superTblInfo && superTblInfo->childTblLimit ) {
if (superTblInfo && superTblInfo->childTblOffset) {
// TODO
}
for (uint32_t tID = winfo->start_table_id; tID <= winfo->end_table_id;
tID++) {
for (uint32_t tableID = winfo->start_table_id; tableID <= winfo->end_table_id;
tableID++) {
int64_t start_time = winfo->start_time;
int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT;
......@@ -4561,7 +4571,7 @@ static void* syncWrite(void *sarg) {
sampleUsePos = samplePos;
int generated = generateDataBuffer(tID, winfo, buffer, insertRows,
int generated = generateDataBuffer(tableID, winfo, buffer, insertRows,
i, start_time, &sampleUsePos);
if (generated > 0)
i += generated;
......@@ -4605,12 +4615,12 @@ static void* syncWrite(void *sarg) {
}
} // num_of_DPT
if ((tID == winfo->end_table_id) && superTblInfo &&
if ((tableID == winfo->end_table_id) && superTblInfo &&
(0 == strncasecmp(
superTblInfo->dataSource, "sample", strlen("sample")))) {
samplePos = sampleUsePos;
}
} // tID
} // tableID
free_and_statistics_2:
tmfree(buffer);
......@@ -4705,12 +4715,19 @@ static void startMultiThreadInsertData(int threads, char* db_name,
char* precision,SSuperTable* superTblInfo) {
pthread_t *pids = malloc(threads * sizeof(pthread_t));
assert(pids != NULL);
threadInfo *infos = malloc(threads * sizeof(threadInfo));
assert(infos != NULL);
memset(pids, 0, threads * sizeof(pthread_t));
memset(infos, 0, threads * sizeof(threadInfo));
int ntables = 0;
if (superTblInfo)
if (superTblInfo->childTblOffset)
ntables = superTblInfo->childTblLimit;
else
ntables = superTblInfo->childTblCount;
else
ntables = g_args.num_of_tables;
......@@ -4767,7 +4784,13 @@ static void startMultiThreadInsertData(int threads, char* db_name,
double start = getCurrentTime();
int last = 0;
int last;
if ((superTblInfo) && (superTblInfo->childTblOffset))
last = superTblInfo->childTblOffset;
else
last = 0;
for (int i = 0; i < threads; i++) {
threadInfo *t_info = infos + i;
t_info->threadID = i;
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或