Skip to content

T
TDengine

taosdata / TDengine
1 年多 前同步成功

通知 1185
Star 22016
Fork 4786
T
TDengine
未验证 提交 6cb5af80 编写于 作者: sangshuduo's avatar sangshuduo 提交者： GitHub
浏览文件
操作

Hotfix/sangshuduo/td 3607 taosdemo buffer overflow (#5636) 

* [TD-3607] <fix>: fix taosdemo buffer overflow.

* [TD-3607] <fix>: taosdemo buffer overflow.

add tmp buffer.

* [TD-3607] <fix>: taosdemo buffer overflow.

fix data generation.

* [TD-3607] <fix>: taosdemo buffer overflow.

fix normal table writting.

* [TD-3607] <fix>: taosdemo buffer overflow.

remove tail spaces.

* [TD-3607] <fix>: taosdemo buffer overflow.

fix taosdemo alter table test case.

* [TD-3607] <fix>: taosdemo buffer overflow.

fix taosdemo alter table case.
Co-authored-by: NShuduo Sang <sdsang@taosdata.com>
上级 ff2f32ad
显示空白变更内容
内联 并排
Showing with 185 addition and 129 deletion
src/kit/taosdemo/taosdemo.c
浏览文件 @ 6cb5af80
......@@ -3166,7 +3166,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
if (numRecPerReq && numRecPerReq->type == cJSON_Number) {
g_args.num_of_RPR = numRecPerReq->valueint;
} else if (!numRecPerReq) {
g_args.num_of_RPR = 100;
g_args.num_of_RPR = 0xffff;
} else {
errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", __func__, __LINE__);
goto PARSE_OVER;
......@@ -3615,7 +3615,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
}
g_Dbs.db[i].superTbls[j].maxSqlLen = len;
} else if (!maxSqlLen) {
g_Dbs.db[i].superTbls[j].maxSqlLen = TSDB_MAX_SQL_LEN;
g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len;
} else {
printf("ERROR: failed to read json, maxSqlLen not found\n");
goto PARSE_OVER;
......@@ -4204,67 +4204,73 @@ static int getRowDataFromSample(char* dataBuf, int maxLen, int64_t timestamp,
return dataLen;
}
static int generateRowData(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* stbInfo) {
static int generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stbInfo) {
int dataLen = 0;
dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp);
char *pstr = recBuf;
int maxLen = MAX_DATA_SIZE;
dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp);
for (int i = 0; i < stbInfo->columnCount; i++) {
if ((0 == strncasecmp(stbInfo->columns[i].dataType, "binary", 6))
|| (0 == strncasecmp(stbInfo->columns[i].dataType, "nchar", 5))) {
if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) {
errorPrint( "binary or nchar length overflow, max size:%u\n",
(uint32_t)TSDB_MAX_BINARY_LEN);
return (-1);
return -1;
}
char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1);
if (NULL == buf) {
errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen);
return (-1);
return -1;
}
rand_string(buf, stbInfo->columns[i].dataLen);
dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "\'%s\', ", buf);
dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\', ", buf);
tmfree(buf);
} else if (0 == strncasecmp(stbInfo->columns[i].dataType,
"int", 3)) {
dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
"%d, ", rand_int());
} else if (0 == strncasecmp(stbInfo->columns[i].dataType,
"bigint", 6)) {
dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
"%"PRId64", ", rand_bigint());
} else if (0 == strncasecmp(stbInfo->columns[i].dataType,
"float", 5)) {
dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
"%f, ", rand_float());
} else if (0 == strncasecmp(stbInfo->columns[i].dataType,
"double", 6)) {
dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen,
dataLen += snprintf(pstr + dataLen, maxLen - dataLen,
"%f, ", rand_double());
} else if (0 == strncasecmp(stbInfo->columns[i].dataType,
"smallint", 8)) {
dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_smallint());
dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_smallint());
} else if (0 == strncasecmp(stbInfo->columns[i].dataType, "tinyint", 7)) {
dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_tinyint());
dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_tinyint());
} else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bool", 4)) {
dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_bool());
dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_bool());
} else if (0 == strncasecmp(stbInfo->columns[i].dataType, "timestamp", 9)) {
dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint());
dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint());
} else {
errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType);
return (-1);
return -1;
}
}
dataLen -= 2;
dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")");
dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")");
return dataLen;
verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf);
return strlen(recBuf);
}
static int32_t generateData(char *res, char **data_type,
static int32_t generateData(char *recBuf, char **data_type,
int num_of_cols, int64_t timestamp, int lenOfBinary) {
memset(res, 0, MAX_DATA_SIZE);
char *pstr = res;
memset(recBuf, 0, MAX_DATA_SIZE);
char *pstr = recBuf;
pstr += sprintf(pstr, "(%" PRId64, timestamp);
int c = 0;
......@@ -4308,7 +4314,7 @@ static int32_t generateData(char *res, char **data_type,
free(s);
}
if (pstr - res > MAX_DATA_SIZE) {
if (strlen(recBuf) > MAX_DATA_SIZE) {
perror("column length too long, abort");
exit(-1);
}
......@@ -4316,7 +4322,7 @@ static int32_t generateData(char *res, char **data_type,
pstr += sprintf(pstr, ")");
return (int32_t)(pstr - res);
return (int32_t)strlen(recBuf);
}
static int prepareSampleDataForSTable(SSuperTable *superTblInfo) {
......@@ -4396,7 +4402,7 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, int tableSeq)
static int generateDataTail(char *tableName, int32_t tableSeq,
threadInfo* pThreadInfo, SSuperTable* superTblInfo,
int batch, char* buffer, int64_t insertRows,
int batch, char* buffer, int remainderBufLen, int64_t insertRows,
int64_t startFrom, uint64_t startTime, int *pSamplePos, int *dataLen) {
int len = 0;
int ncols_per_record = 1; // count first col ts
......@@ -4413,14 +4419,15 @@ static int generateDataTail(char *tableName, int32_t tableSeq,
int k = 0;
for (k = 0; k < batch;) {
if (superTblInfo) {
char data[MAX_DATA_SIZE];
int retLen = 0;
if (superTblInfo) {
if (0 == strncasecmp(superTblInfo->dataSource,
"sample", strlen("sample"))) {
retLen = getRowDataFromSample(
buffer + len,
superTblInfo->maxSqlLen - len,
data,
remainderBufLen,
startTime + superTblInfo->timeStampStep * k,
superTblInfo,
pSamplePos);
......@@ -4433,32 +4440,27 @@ static int generateDataTail(char *tableName, int32_t tableSeq,
+ superTblInfo->timeStampStep * k
- taosRandom() % superTblInfo->disorderRange;
retLen = generateRowData(
buffer + len,
superTblInfo->maxSqlLen - len,
data,
d,
superTblInfo);
} else {
retLen = generateRowData(
buffer + len,
superTblInfo->maxSqlLen - len,
data,
startTime + superTblInfo->timeStampStep * k,
superTblInfo);
}
}
if (retLen < 0) {
return -1;
if (retLen > remainderBufLen) {
break;
}
len += retLen;
if (len >= (superTblInfo->maxSqlLen - 256)) { // reserve for overwrite
buffer += sprintf(buffer, " %s", data);
k++;
break;
}
len += retLen;
remainderBufLen -= retLen;
} else {
int rand_num = taosRandom() % 100;
char data[MAX_DATA_SIZE];
char **data_type = g_args.datatype;
int lenOfBinary = g_args.len_of_binary;
......@@ -4467,26 +4469,27 @@ static int generateDataTail(char *tableName, int32_t tableSeq,
int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k
- taosRandom() % 1000000 + rand_num;
len = generateData(data, data_type,
retLen = generateData(data, data_type,
ncols_per_record, d, lenOfBinary);
} else {
len = generateData(data, data_type,
retLen = generateData(data, data_type,
ncols_per_record,
startTime + DEFAULT_TIMESTAMP_STEP * k,
lenOfBinary);
}
if (len > remainderBufLen)
break;
buffer += sprintf(buffer, " %s", data);
if (strlen(buffer) >= (g_args.max_sql_len - 256)) { // too long
k++;
break;
}
len += retLen;
remainderBufLen -= retLen;
}
verbosePrint("%s() LN%d len=%d k=%d \nbuffer=%s\n",
__func__, __LINE__, len, k, buffer);
k++;
startFrom ++;
if (startFrom >= insertRows) {
......@@ -4570,20 +4573,25 @@ static int generateProgressiveDataBuffer(char *pTblName,
assert(buffer != NULL);
memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len);
int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
int remainderBufLen = maxSqlLen;
memset(buffer, 0, maxSqlLen);
char *pstr = buffer;
int headLen = generateSQLHead(pTblName, tableSeq, pThreadInfo, superTblInfo,
buffer);
pstr += headLen;
remainderBufLen -= headLen;
int k;
int dataLen;
k = generateDataTail(pTblName, tableSeq, pThreadInfo, superTblInfo,
g_args.num_of_RPR, pstr, insertRows, startFrom,
g_args.num_of_RPR, pstr, remainderBufLen, insertRows, startFrom,
startTime,
pSamplePos, &dataLen);
return k;
}
......@@ -4656,13 +4664,18 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
int generatedRecPerTbl = 0;
bool flagSleep = true;
int sleepTimeTotal = 0;
int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len;
int remainderBufLen;
while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) {
if ((flagSleep) && (insert_interval)) {
st = taosGetTimestampUs();
flagSleep = false;
}
// generate data
memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len);
memset(buffer, 0, maxSqlLen);
remainderBufLen = maxSqlLen;
char *pstr = buffer;
int recOfBatch = 0;
......@@ -4685,6 +4698,8 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
pThreadInfo->threadID, __func__, __LINE__, i, buffer);
pstr += headLen;
remainderBufLen -= headLen;
int dataLen = 0;
verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n",
......@@ -4698,13 +4713,20 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
} else {
startTime = 1500000000000;
}
generateDataTail(
int generated = generateDataTail(
tableName, tableSeq, pThreadInfo, superTblInfo,
batchPerTbl, pstr, insertRows, 0,
batchPerTbl, pstr, remainderBufLen, insertRows, 0,
startTime,
&(pThreadInfo->samplePos), &dataLen);
if (generated < 0) {
debugPrint("[%d] %s() LN%d, generated data is %d\n",
pThreadInfo->threadID, __func__, __LINE__, generated);
goto free_and_statistics_interlace;
}
pstr += dataLen;
remainderBufLen -= dataLen;
recOfBatch += batchPerTbl;
startTime += batchPerTbl * superTblInfo->timeStampStep;
pThreadInfo->totalInsertRows += batchPerTbl;
......@@ -4966,7 +4988,7 @@ static void callBack(void *param, TAOS_RES *res, int code) {
}
char *buffer = calloc(1, winfo->superTblInfo->maxSqlLen);
char *data = calloc(1, MAX_DATA_SIZE);
char data[MAX_DATA_SIZE];
char *pstr = buffer;
pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix,
winfo->start_table_from);
......@@ -4978,7 +5000,6 @@ static void callBack(void *param, TAOS_RES *res, int code) {
if (winfo->start_table_from > winfo->end_table_to) {
tsem_post(&winfo->lock_sem);
free(buffer);
free(data);
taos_free_result(res);
return;
}
......@@ -4988,11 +5009,9 @@ static void callBack(void *param, TAOS_RES *res, int code) {
if (0 != winfo->superTblInfo->disorderRatio
&& rand_num < winfo->superTblInfo->disorderRatio) {
int64_t d = winfo->lastTs - taosRandom() % 1000000 + rand_num;
//generateData(data, datatype, ncols_per_record, d, len_of_binary);
generateRowData(data, MAX_DATA_SIZE, d, winfo->superTblInfo);
generateRowData(data, d, winfo->superTblInfo);
} else {
//generateData(data, datatype, ncols_per_record, start_time += 1000, len_of_binary);
generateRowData(data, MAX_DATA_SIZE, winfo->lastTs += 1000, winfo->superTblInfo);
generateRowData(data, winfo->lastTs += 1000, winfo->superTblInfo);
}
pstr += sprintf(pstr, "%s", data);
winfo->counter++;
......@@ -5007,7 +5026,6 @@ static void callBack(void *param, TAOS_RES *res, int code) {
}
taos_query_a(winfo->taos, buffer, callBack, winfo);
free(buffer);
free(data);
taos_free_result(res);
}
......@@ -6256,7 +6274,7 @@ static void setParaFromArg(){
g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP;
g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT;
g_Dbs.db[0].superTbls[0].maxSqlLen = TSDB_PAYLOAD_SIZE;
g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len;
g_Dbs.db[0].superTbls[0].columnCount = 0;
for (int i = 0; i < MAX_NUM_DATATYPE; i++) {
......
tests/pytest/fulltest.sh
浏览文件 @ 6cb5af80
......@@ -139,6 +139,18 @@ python3 ./test.py -f import_merge/importInsertThenImport.py
python3 ./test.py -f import_merge/importCSV.py
#======================p1-end===============
#======================p2-start===============
# tools
python3 test.py -f tools/taosdumpTest.py
python3 test.py -f tools/taosdemoTest.py
python3 test.py -f tools/taosdemoTestWithoutMetric.py
python3 test.py -f tools/taosdemoTestWithJson.py
python3 test.py -f tools/taosdemoTestLimitOffset.py
python3 test.py -f tools/taosdemoTestTblAlt.py
python3 test.py -f tools/taosdemoTestSampleData.py
python3 test.py -f tools/taosdemoTestInterlace.py
python3 test.py -f tools/taosdemoTestQuery.py
# update
python3 ./test.py -f update/allow_update.py
python3 ./test.py -f update/allow_update-0.py
......@@ -247,18 +259,6 @@ python3 test.py -f subscribe/supertable.py
#======================p3-end===============
#======================p4-start===============
# tools
python3 test.py -f tools/taosdumpTest.py
python3 test.py -f tools/taosdemoTest.py
python3 test.py -f tools/taosdemoTestWithoutMetric.py
python3 test.py -f tools/taosdemoTestWithJson.py
python3 test.py -f tools/taosdemoTestLimitOffset.py
python3 test.py -f tools/taosdemoTest2.py
python3 test.py -f tools/taosdemoTestSampleData.py
python3 test.py -f tools/taosdemoTestInterlace.py
python3 test.py -f tools/taosdemoTestQuery.py
python3 ./test.py -f update/merge_commit_data-0.py
# wal
python3 ./test.py -f wal/addOldWalTest.py
......
tests/pytest/tools/taosdemoTest2.py → tests/pytest/tools/taosdemoTestTblAlt.py
浏览文件 @ 6cb5af80
......@@ -29,10 +29,33 @@ class TDTestCase:
self.numberOfTables = 10
self.numberOfRecords = 1000000
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosd" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
break
return buildPath
def insertDataAndAlterTable(self, threadID):
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosd not found!")
else:
tdLog.info("taosd found in %s" % buildPath)
binPath = buildPath + "/build/bin/"
if(threadID == 0):
os.system("taosdemo -y -t %d -n %d" %
(self.numberOfTables, self.numberOfRecords))
os.system("%staosdemo -y -t %d -n %d" %
(binPath, self.numberOfTables, self.numberOfRecords))
if(threadID == 1):
time.sleep(2)
print("use test")
......@@ -47,7 +70,13 @@ class TDTestCase:
# check if all the tables have heen created
while True:
try:
tdSql.query("show tables")
except Exception as e:
tdLog.info("show tables test failed")
time.sleep(1)
continue
rows = tdSql.queryRows
print("number of tables: %d" % rows)
if(rows == self.numberOfTables):
......@@ -56,16 +85,23 @@ class TDTestCase:
# check if there are any records in the last created table
while True:
print("query started")
try:
tdSql.query("select * from test.t9")
except Exception as e:
tdLog.info("select * test failed")
time.sleep(2)
continue
rows = tdSql.queryRows
print("number of records: %d" % rows)
if(rows > 0):
break
time.sleep(1)
print("alter table test.meters add column col10 int")
tdSql.execute("alter table test.meters add column col10 int")
print("insert into test.t0 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)")
tdSql.execute("insert into test.t0 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)")
print("insert into test.t9 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)")
tdSql.execute("insert into test.t9 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)")
def run(self):
tdSql.prepare()
......@@ -78,6 +114,8 @@ class TDTestCase:
t1.join()
t2.join()
time.sleep(3)
tdSql.query("select count(*) from test.meters")
tdSql.checkData(0, 0, self.numberOfRecords * self.numberOfTables + 1)
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或