From 868daf4f0623d26bd76e164555a93e79b1f12d2e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 23 Jul 2021 06:36:52 +0800 Subject: [PATCH] Hotfix/sangshuduo/td 5445 taosdemo stmt sample (#6977) * [TD-5445]: taosdemo bug for stmt interface with sample data. * refactoring some macros. * fix mac compile error. --- src/kit/taosdemo/taosdemo.c | 133 +++++++++++++++++++++--------------- 1 file changed, 77 insertions(+), 56 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 5776e6279e..f9b147d43d 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -66,10 +66,6 @@ extern char configDir[]; -#define INSERT_JSON_NAME "insert.json" -#define QUERY_JSON_NAME "query.json" -#define SUBSCRIBE_JSON_NAME "subscribe.json" - #define STR_INSERT_INTO "INSERT INTO " #define MAX_RECORDS_PER_REQ 32766 @@ -79,9 +75,10 @@ extern char configDir[]; #define BUFFER_SIZE TSDB_MAX_ALLOWED_SQL_LEN #define COND_BUF_LEN (BUFFER_SIZE - 30) #define COL_BUFFER_LEN ((TSDB_COL_NAME_LEN + 15) * TSDB_MAX_COLUMNS) + #define MAX_USERNAME_SIZE 64 #define MAX_PASSWORD_SIZE 64 -#define MAX_HOSTNAME_SIZE 64 +#define MAX_HOSTNAME_SIZE 253 // https://man7.org/linux/man-pages/man7/hostname.7.html #define MAX_TB_NAME_SIZE 64 #define MAX_DATA_SIZE (16*TSDB_MAX_COLUMNS)+20 // max record len: 16*MAX_COLUMNS, timestamp string and ,('') need extra space #define OPT_ABORT 1 /* –abort */ @@ -100,6 +97,11 @@ extern char configDir[]; #define MAX_DATABASE_COUNT 256 #define INPUT_BUF_LEN 256 +#define TBNAME_PREFIX_LEN (TSDB_TABLE_NAME_LEN - 20) // 20 characters reserved for seq +#define SMALL_BUFF_LEN 8 +#define DATATYPE_BUFF_LEN (SMALL_BUFF_LEN*3) +#define NOTE_BUFF_LEN (SMALL_BUFF_LEN*16) + #define DEFAULT_TIMESTAMP_STEP 1 @@ -244,16 +246,15 @@ typedef struct SArguments_S { typedef struct SColumn_S { char field[TSDB_COL_NAME_LEN]; - char dataType[16]; + char dataType[DATATYPE_BUFF_LEN]; uint32_t dataLen; - char note[128]; + char note[NOTE_BUFF_LEN]; } StrColumn; typedef struct SSuperTable_S { char sTblName[TSDB_TABLE_NAME_LEN]; - char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample - char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq - char insertMode[MAX_TB_NAME_SIZE]; // taosc, rest + char dataSource[SMALL_BUFF_LEN]; // rand_gen or sample + char childTblPrefix[TBNAME_PREFIX_LEN]; uint16_t childTblExists; int64_t childTblCount; uint64_t batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql @@ -272,7 +273,7 @@ typedef struct SSuperTable_S { int64_t insertRows; int64_t timeStampStep; char startTimestamp[MAX_TB_NAME_SIZE]; - char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json + char sampleFormat[SMALL_BUFF_LEN]; // csv, json char sampleFile[MAX_FILE_NAME_LEN]; char tagsFile[MAX_FILE_NAME_LEN]; @@ -308,7 +309,7 @@ typedef struct { int16_t replica; int16_t quorum; int16_t days; - char keeplist[32]; + char keeplist[64]; int32_t cache; //MB int32_t blocks; int32_t minrows; @@ -317,7 +318,7 @@ typedef struct { int32_t fsync; int8_t comp; int8_t cachelast; - char precision[8]; // time resolution + char precision[SMALL_BUFF_LEN]; // time resolution int8_t update; char status[16]; } SDbInfo; @@ -337,7 +338,7 @@ typedef struct SDbCfg_S { int cache; int blocks; int quorum; - char precision[8]; + char precision[SMALL_BUFF_LEN]; } SDbCfg; typedef struct SDataBase_S { @@ -403,7 +404,7 @@ typedef struct SuperQueryInfo_S { int subscribeKeepProgress; uint64_t queryTimes; int64_t childTblCount; - char childTblPrefix[TSDB_TABLE_NAME_LEN - 20]; // 20 characters reserved for seq + char childTblPrefix[TBNAME_PREFIX_LEN]; // 20 characters reserved for seq int sqlCount; char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; @@ -423,7 +424,7 @@ typedef struct SQueryMetaInfo_S { char user[MAX_USERNAME_SIZE]; char password[MAX_PASSWORD_SIZE]; char dbName[TSDB_DB_NAME_LEN]; - char queryMode[MAX_TB_NAME_SIZE]; // taosc, rest + char queryMode[SMALL_BUFF_LEN]; // taosc, rest SpecifiedQueryInfo specifiedQueryInfo; SuperQueryInfo superQueryInfo; @@ -2680,12 +2681,14 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); tstrncpy(superTbls->tags[tagIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); + min(DATATYPE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); superTbls->tags[tagIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); tstrncpy(superTbls->tags[tagIndex].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], - fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + min(NOTE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes) + 1); tagIndex++; } else { tstrncpy(superTbls->columns[columnIndex].field, @@ -2693,12 +2696,14 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); tstrncpy(superTbls->columns[columnIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); + min(DATATYPE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes) + 1); superTbls->columns[columnIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); tstrncpy(superTbls->columns[columnIndex].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], - fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + min(NOTE_BUFF_LEN, + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes) + 1); columnIndex++; } count++; @@ -3461,8 +3466,9 @@ static bool getColumnAndTagTypeFromInsertJsonFile( __func__, __LINE__); goto PARSE_OVER; } - //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE); - tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1); + //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, DATATYPE_BUFF_LEN); + tstrncpy(columnCase.dataType, dataType->valuestring, + min(DATATYPE_BUFF_LEN, strlen(dataType->valuestring) + 1)); cJSON* dataLen = cJSON_GetObjectItem(column, "len"); if (dataLen && dataLen->type == cJSON_Number) { @@ -3472,12 +3478,13 @@ static bool getColumnAndTagTypeFromInsertJsonFile( __func__, __LINE__); goto PARSE_OVER; } else { - columnCase.dataLen = 8; + columnCase.dataLen = SMALL_BUFF_LEN; } for (int n = 0; n < count; ++n) { tstrncpy(superTbls->columns[index].dataType, - columnCase.dataType, strlen(columnCase.dataType) + 1); + columnCase.dataType, + min(DATATYPE_BUFF_LEN, strlen(columnCase.dataType) + 1)); superTbls->columns[index].dataLen = columnCase.dataLen; index++; } @@ -3533,7 +3540,8 @@ static bool getColumnAndTagTypeFromInsertJsonFile( __func__, __LINE__); goto PARSE_OVER; } - tstrncpy(columnCase.dataType, dataType->valuestring, strlen(dataType->valuestring) + 1); + tstrncpy(columnCase.dataType, dataType->valuestring, + min(DATATYPE_BUFF_LEN, strlen(dataType->valuestring) + 1)); cJSON* dataLen = cJSON_GetObjectItem(tag, "len"); if (dataLen && dataLen->type == cJSON_Number) { @@ -3548,7 +3556,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( for (int n = 0; n < count; ++n) { tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, - strlen(columnCase.dataType) + 1); + min(DATATYPE_BUFF_LEN, strlen(columnCase.dataType) + 1)); superTbls->tags[index].dataLen = columnCase.dataLen; index++; } @@ -3792,9 +3800,9 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (precision && precision->type == cJSON_String && precision->valuestring != NULL) { tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring, - 8); + SMALL_BUFF_LEN); } else if (!precision) { - memset(g_Dbs.db[i].dbCfg.precision, 0, 8); + memset(g_Dbs.db[i].dbCfg.precision, 0, SMALL_BUFF_LEN); } else { printf("ERROR: failed to read json, precision not found\n"); goto PARSE_OVER; @@ -3979,7 +3987,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, - TSDB_TABLE_NAME_LEN - 20); + TBNAME_PREFIX_LEN); cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); if (autoCreateTbl @@ -4047,9 +4055,11 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (dataSource && dataSource->type == cJSON_String && dataSource->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, - dataSource->valuestring, TSDB_DB_NAME_LEN); + dataSource->valuestring, + min(SMALL_BUFF_LEN, strlen(dataSource->valuestring) + 1)); } else if (!dataSource) { - tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", TSDB_DB_NAME_LEN); + tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", + min(SMALL_BUFF_LEN, strlen("rand") + 1)); } else { errorPrint("%s() LN%d, failed to read json, data_source not found\n", __func__, __LINE__); @@ -4130,9 +4140,12 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, - sampleFormat->valuestring, TSDB_DB_NAME_LEN); + sampleFormat->valuestring, + min(SMALL_BUFF_LEN, + strlen(sampleFormat->valuestring) + 1)); } else if (!sampleFormat) { - tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", TSDB_DB_NAME_LEN); + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", + SMALL_BUFF_LEN); } else { printf("ERROR: failed to read json, sample_format not found\n"); goto PARSE_OVER; @@ -4142,9 +4155,12 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) { tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile, - sampleFile->valuestring, MAX_FILE_NAME_LEN); + sampleFile->valuestring, + min(MAX_FILE_NAME_LEN, + strlen(sampleFile->valuestring) + 1)); } else if (!sampleFile) { - memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN); + memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, + MAX_FILE_NAME_LEN); } else { printf("ERROR: failed to read json, sample_file not found\n"); goto PARSE_OVER; @@ -4384,10 +4400,14 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode"); - if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) { - tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE); + if (queryMode + && queryMode->type == cJSON_String + && queryMode->valuestring != NULL) { + tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, + min(SMALL_BUFF_LEN, strlen(queryMode->valuestring) + 1)); } else if (!queryMode) { - tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE); + tstrncpy(g_queryInfo.queryMode, "taosc", + min(SMALL_BUFF_LEN, strlen("taosc") + 1)); } else { printf("ERROR: failed to read json, query_mode not found\n"); goto PARSE_OVER; @@ -6524,9 +6544,10 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { } // num_of_DPT if ((g_args.verbose_print) && - (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) && - (0 == strncasecmp( - superTblInfo->dataSource, "sample", strlen("sample")))) { + (tableSeq == pThreadInfo->ntables - 1) && (superTblInfo) + && (0 == strncasecmp( + superTblInfo->dataSource, + "sample", strlen("sample")))) { verbosePrint("%s() LN%d samplePos=%"PRId64"\n", __func__, __LINE__, pThreadInfo->samplePos); } @@ -7364,7 +7385,7 @@ static void *specifiedTableQuery(void *sarg) { static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) { char sourceString[32] = "xxxx"; - char subTblName[MAX_TB_NAME_SIZE*3]; + char subTblName[TSDB_TABLE_NAME_LEN]; sprintf(subTblName, "%s.%s", g_queryInfo.dbName, g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN); @@ -7523,8 +7544,8 @@ static int queryTestProcess() { if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); + char sqlStr[TSDB_DB_NAME_LEN + 5]; + sprintf(sqlStr, "USE %s", g_queryInfo.dbName); if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { taos_close(taos); free(infos); @@ -7713,8 +7734,8 @@ static void *superSubscribe(void *sarg) { } } - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); + char sqlStr[TSDB_DB_NAME_LEN + 5]; + sprintf(sqlStr, "USE %s", g_queryInfo.dbName); if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { taos_close(pThreadInfo->taos); errorPrint( "use database %s failed!\n\n", @@ -7855,8 +7876,8 @@ static void *specifiedSubscribe(void *sarg) { } } - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); + char sqlStr[TSDB_DB_NAME_LEN + 5]; + sprintf(sqlStr, "USE %s", g_queryInfo.dbName); if (0 != queryDbExec(pThreadInfo->taos, sqlStr, NO_INSERT_TYPE, false)) { taos_close(pThreadInfo->taos); return NULL; @@ -8151,7 +8172,7 @@ static void setParaFromArg() { tstrncpy(g_Dbs.db[0].dbName, g_args.database, TSDB_DB_NAME_LEN); g_Dbs.db[0].dbCfg.replica = g_args.replica; - tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", 8); + tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", SMALL_BUFF_LEN); tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); @@ -8184,8 +8205,8 @@ static void setParaFromArg() { g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, - g_args.tb_prefix, TSDB_TABLE_NAME_LEN - 20); - tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); + g_args.tb_prefix, TBNAME_PREFIX_LEN); + tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", SMALL_BUFF_LEN); if (g_args.iface == INTERFACE_BUT) { g_Dbs.db[0].superTbls[0].iface = TAOSC_IFACE; @@ -8206,7 +8227,7 @@ static void setParaFromArg() { } tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, - data_type[i], strlen(data_type[i]) + 1); + data_type[i], min(DATATYPE_BUFF_LEN, strlen(data_type[i]) + 1)); g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].columnCount++; } @@ -8217,18 +8238,18 @@ static void setParaFromArg() { for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, - "INT", strlen("INT") + 1); + "INT", min(DATATYPE_BUFF_LEN, strlen("INT") + 1)); g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; g_Dbs.db[0].superTbls[0].columnCount++; } } tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, - "INT", strlen("INT") + 1); + "INT", min(DATATYPE_BUFF_LEN, strlen("INT") + 1)); g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, - "BINARY", strlen("BINARY") + 1); + "BINARY", min(DATATYPE_BUFF_LEN, strlen("BINARY") + 1)); g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; g_Dbs.db[0].superTbls[0].tagCount = 2; } else { @@ -8364,7 +8385,7 @@ static void queryResult() { pThreadInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; pThreadInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; tstrncpy(pThreadInfo->tb_prefix, - g_Dbs.db[0].superTbls[0].childTblPrefix, TSDB_TABLE_NAME_LEN - 20); + g_Dbs.db[0].superTbls[0].childTblPrefix, TBNAME_PREFIX_LEN); } else { pThreadInfo->ntables = g_args.num_of_tables; pThreadInfo->end_table_to = g_args.num_of_tables -1; -- GitLab