diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c7e31bfbb891cc23c2cd5e788772d6f33bc329e..e534f925aa1254136d183c42281bf237bc8ab480 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,8 @@ endif(${BUILD_TEST}) add_subdirectory(source) add_subdirectory(tools) -add_subdirectory(tests) +#add_subdirectory(tests) +add_subdirectory(utils) add_subdirectory(examples/c) # docs diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 31331b52651fee79c837a077869bc45ec7acfe6c..839c65b633b17872051075be33ce291fbeaefd5a 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -1,4 +1,5 @@ add_executable(tmq_demo tmqDemo.c) +add_dependencies(tmq_demo taos) add_executable(tmq_sim tmqSim.c) add_executable(create_table createTable.c) add_executable(tmq_taosx_ci tmq_taosx_ci.c) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..59cbbb31471acc377dd614610aa715ae72c9ec31 --- /dev/null +++ b/utils/CMakeLists.txt @@ -0,0 +1,4 @@ +#ADD_SUBDIRECTORY(examples/c) +ADD_SUBDIRECTORY(tsim) +ADD_SUBDIRECTORY(test/c) +#ADD_SUBDIRECTORY(comparisonTest/tdengine) diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..839c65b633b17872051075be33ce291fbeaefd5a --- /dev/null +++ b/utils/test/c/CMakeLists.txt @@ -0,0 +1,59 @@ +add_executable(tmq_demo tmqDemo.c) +add_dependencies(tmq_demo taos) +add_executable(tmq_sim tmqSim.c) +add_executable(create_table createTable.c) +add_executable(tmq_taosx_ci tmq_taosx_ci.c) +add_executable(sml_test sml_test.c) +target_link_libraries( + create_table + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) +target_link_libraries( + tmq_demo + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) +target_link_libraries( + tmq_sim + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) +target_link_libraries( + tmq_taosx_ci + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) + +target_link_libraries( + sml_test + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) + +add_executable(sdbDump sdbDump.c) +target_link_libraries( + sdbDump + PUBLIC dnode + PUBLIC mnode + PUBLIC stream + PUBLIC sdb + PUBLIC os +) +target_include_directories( + sdbDump + PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode" + PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc" + PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/sdb/inc" + PRIVATE "${TD_SOURCE_DIR}/source/dnode/mgmt/node_mgmt/inc" +) diff --git a/utils/test/c/createTable.c b/utils/test/c/createTable.c new file mode 100644 index 0000000000000000000000000000000000000000..6a0f8e244ebcce79ac60a353df5d5343d58b4b34 --- /dev/null +++ b/utils/test/c/createTable.c @@ -0,0 +1,471 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taos.h" +#include "taoserror.h" +#include "tlog.h" + +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +char dbName[32] = "db"; +char stbName[64] = "st"; +int32_t numOfThreads = 1; +int64_t numOfTables = 200000; +int64_t startOffset = 0; +int32_t createTable = 1; +int32_t insertData = 0; +int32_t batchNumOfTbl = 100; +int32_t batchNumOfRow = 1; +int32_t totalRowsOfPerTbl = 1; +int32_t numOfVgroups = 2; +int32_t showTablesFlag = 0; +int32_t queryFlag = 0; + +int64_t startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000 + +typedef struct { + int64_t tableBeginIndex; + int64_t tableEndIndex; + int32_t threadIndex; + char dbName[32]; + char stbName[64]; + float createTableSpeed; + float insertDataSpeed; + int64_t startMs; + int64_t maxDelay; + int64_t minDelay; + TdThread thread; +} SThreadInfo; + +// void parseArgument(int32_t argc, char *argv[]); +// void *threadFunc(void *param); +// void createDbAndStb(); + +void createDbAndStb() { + pPrint("start to create db and stable"); + char qstr[64000]; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("failed to connect to DB, reason:%s", taos_errstr(NULL)); + exit(1); + } + + sprintf(qstr, "create database if not exists %s vgroups %d", dbName, numOfVgroups); + TAOS_RES *pRes = taos_query(con, qstr); + int32_t code = taos_errno(pRes); + if (code != 0) { + pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(pRes), + taos_errstr(pRes)); + exit(0); + } + taos_free_result(pRes); + + sprintf(qstr, "use %s", dbName); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if (code != 0) { + pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + exit(0); + } + taos_free_result(pRes); + + sprintf(qstr, "create table if not exists %s (ts timestamp, i int) tags (j bigint)", stbName); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if (code != 0) { + pError("failed to create stable, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + exit(0); + } + taos_free_result(pRes); + + taos_close(con); +} + +void printCreateProgress(SThreadInfo *pInfo, int64_t t) { + int64_t endMs = taosGetTimestampMs(); + int64_t totalTables = t - pInfo->tableBeginIndex; + float seconds = (endMs - pInfo->startMs) / 1000.0; + float speed = totalTables / seconds; + pInfo->createTableSpeed = speed; + pPrint("thread:%d, %" PRId64 " tables created, time:%.2f sec, speed:%.1f tables/second, ", pInfo->threadIndex, + totalTables, seconds, speed); +} + +void printInsertProgress(SThreadInfo *pInfo, int64_t insertTotalRows) { + int64_t endMs = taosGetTimestampMs(); + //int64_t totalTables = t - pInfo->tableBeginIndex; + float seconds = (endMs - pInfo->startMs) / 1000.0; + float speed = insertTotalRows / seconds; + pInfo->insertDataSpeed = speed; + pPrint("thread:%d, %" PRId64 " rows inserted, time:%.2f sec, speed:%.1f rows/second, ", pInfo->threadIndex, + insertTotalRows, seconds, speed); +} + +static int64_t getResult(TAOS_RES *tres) { + TAOS_ROW row = taos_fetch_row(tres); + if (row == NULL) { + return 0; + } + + int num_fields = taos_num_fields(tres); + TAOS_FIELD *fields = taos_fetch_fields(tres); + int precision = taos_result_precision(tres); + + int64_t numOfRows = 0; + do { + numOfRows++; + row = taos_fetch_row(tres); + } while (row != NULL); + + return numOfRows; +} + +void showTables() { + pPrint("start to show tables"); + char qstr[128]; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("failed to connect to DB, reason:%s", taos_errstr(NULL)); + exit(1); + } + + snprintf(qstr, 128, "use %s", dbName); + TAOS_RES *pRes = taos_query(con, qstr); + int code = taos_errno(pRes); + if (code != 0) { + pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + exit(1); + } + taos_free_result(pRes); + + sprintf(qstr, "show tables"); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if (code != 0) { + pError("failed to show tables, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + exit(0); + } + + int64_t totalTableNum = getResult(pRes); + taos_free_result(pRes); + + pPrint("%s database: %s, total %" PRId64 " tables %s", GREEN, dbName, totalTableNum, NC); + + taos_close(con); +} + +void *threadFunc(void *param) { + SThreadInfo *pInfo = (SThreadInfo *)param; + char *qstr = taosMemoryMalloc(batchNumOfTbl * batchNumOfRow * 128); + int32_t code = 0; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(NULL)); + exit(1); + } + + //pPrint("====before thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, + // pInfo->tableEndIndex); + + pInfo->tableBeginIndex += startOffset; + pInfo->tableEndIndex += startOffset; + + pPrint("====thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, pInfo->tableEndIndex); + + sprintf(qstr, "use %s", pInfo->dbName); + TAOS_RES *pRes = taos_query(con, qstr); + taos_free_result(pRes); + + if (createTable) { + int64_t curMs = 0; + int64_t beginMs = taosGetTimestampMs(); + pInfo->startMs = beginMs; + int64_t t = pInfo->tableBeginIndex; + for (; t <= pInfo->tableEndIndex;) { + // int64_t batch = (pInfo->tableEndIndex - t); + // batch = MIN(batch, batchNum); + + int32_t len = sprintf(qstr, "create table"); + for (int32_t i = 0; i < batchNumOfTbl;) { + len += sprintf(qstr + len, " %s_t%" PRId64 " using %s tags(%" PRId64 ")", stbName, t, stbName, t); + t++; + i++; + if (t > pInfo->tableEndIndex) { + break; + } + } + + int64_t startTs = taosGetTimestampUs(); + TAOS_RES *pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if (code != 0) { + pError("failed to create table reason:%s, sql: %s", tstrerror(code), qstr); + } + taos_free_result(pRes); + int64_t endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + // printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); + if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; + if (delay < pInfo->minDelay) pInfo->minDelay = delay; + + curMs = taosGetTimestampMs(); + if (curMs - beginMs > 10000) { + beginMs = curMs; + // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); + printCreateProgress(pInfo, t); + } + } + printCreateProgress(pInfo, t); + } + + if (insertData) { + int64_t insertTotalRows = 0; + int64_t curMs = 0; + int64_t beginMs = taosGetTimestampMs(); + pInfo->startMs = beginMs; + int64_t t = pInfo->tableBeginIndex; + for (; t <= pInfo->tableEndIndex; t++) { + //printf("table name: %"PRId64"\n", t); + int64_t ts = startTimestamp; + for (int32_t i = 0; i < totalRowsOfPerTbl;) { + int32_t len = sprintf(qstr, "insert into "); + len += sprintf(qstr + len, "%s_t%" PRId64 " values ", stbName, t); + for (int32_t j = 0; j < batchNumOfRow; j++) { + len += sprintf(qstr + len, "(%" PRId64 ", 6666) ", ts++); + i++; + insertTotalRows++; + if (i >= totalRowsOfPerTbl) { + break; + } + } + + #if 1 + int64_t startTs = taosGetTimestampUs(); + TAOS_RES *pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if (code != 0) { + pError("failed to insert %s_t%" PRId64 ", reason:%s", stbName, t, tstrerror(code)); + } + taos_free_result(pRes); + int64_t endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + // printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); + if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; + if (delay < pInfo->minDelay) pInfo->minDelay = delay; + + curMs = taosGetTimestampMs(); + if (curMs - beginMs > 10000) { + beginMs = curMs; + // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); + printInsertProgress(pInfo, insertTotalRows); + } + #endif + } + } + printInsertProgress(pInfo, insertTotalRows); + } + + taos_close(con); + taosMemoryFree(qstr); + return 0; +} + +void printHelp() { + char indent[10] = " "; + printf("Used to test the performance while create table\n"); + + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-d"); + printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stbName); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", numOfThreads); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "numOfTables, default is ", numOfTables); + printf("%s%s\n", indent, "-g"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "startOffset, default is ", startOffset); + printf("%s%s\n", indent, "-v"); + printf("%s%s%s%d\n", indent, indent, "numOfVgroups, default is ", numOfVgroups); + printf("%s%s\n", indent, "-a"); + printf("%s%s%s%d\n", indent, indent, "createTable, default is ", createTable); + printf("%s%s\n", indent, "-i"); + printf("%s%s%s%d\n", indent, indent, "insertData, default is ", insertData); + printf("%s%s\n", indent, "-b"); + printf("%s%s%s%d\n", indent, indent, "batchNumOfTbl, default is ", batchNumOfTbl); + printf("%s%s\n", indent, "-w"); + printf("%s%s%s%d\n", indent, indent, "showTablesFlag, default is ", showTablesFlag); + printf("%s%s\n", indent, "-q"); + printf("%s%s%s%d\n", indent, indent, "queryFlag, default is ", queryFlag); + printf("%s%s\n", indent, "-l"); + printf("%s%s%s%d\n", indent, indent, "batchNumOfRow, default is ", batchNumOfRow); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "totalRowsOfPerTbl, default is ", totalRowsOfPerTbl); + + exit(EXIT_SUCCESS); +} + +void parseArgument(int32_t argc, char *argv[]) { + for (int32_t i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(dbName, argv[++i]); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + strcpy(stbName, argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + numOfTables = atoll(argv[++i]); + } else if (strcmp(argv[i], "-g") == 0) { + startOffset = atoll(argv[++i]); + } else if (strcmp(argv[i], "-v") == 0) { + numOfVgroups = atoi(argv[++i]); + } else if (strcmp(argv[i], "-a") == 0) { + createTable = atoi(argv[++i]); + } else if (strcmp(argv[i], "-i") == 0) { + insertData = atoi(argv[++i]); + } else if (strcmp(argv[i], "-b") == 0) { + batchNumOfTbl = atoi(argv[++i]); + } else if (strcmp(argv[i], "-l") == 0) { + batchNumOfRow = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + totalRowsOfPerTbl = atoi(argv[++i]); + } else if (strcmp(argv[i], "-w") == 0) { + showTablesFlag = atoi(argv[++i]); + } else if (strcmp(argv[i], "-q") == 0) { + queryFlag = atoi(argv[++i]); + } else { + pPrint("%s unknow para: %s %s", GREEN, argv[++i], NC); + } + } + + pPrint("%s dbName:%s %s", GREEN, dbName, NC); + pPrint("%s stbName:%s %s", GREEN, stbName, NC); + pPrint("%s configDir:%s %s", GREEN, configDir, NC); + pPrint("%s numOfTables:%" PRId64 " %s", GREEN, numOfTables, NC); + pPrint("%s startOffset:%" PRId64 " %s", GREEN, startOffset, NC); + pPrint("%s numOfThreads:%d %s", GREEN, numOfThreads, NC); + pPrint("%s numOfVgroups:%d %s", GREEN, numOfVgroups, NC); + pPrint("%s createTable:%d %s", GREEN, createTable, NC); + pPrint("%s insertData:%d %s", GREEN, insertData, NC); + pPrint("%s batchNumOfTbl:%d %s", GREEN, batchNumOfTbl, NC); + pPrint("%s batchNumOfRow:%d %s", GREEN, batchNumOfRow, NC); + pPrint("%s totalRowsOfPerTbl:%d %s", GREEN, totalRowsOfPerTbl, NC); + pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC); + pPrint("%s queryFlag:%d %s", GREEN, queryFlag, NC); + + pPrint("%s start create table performace test %s", GREEN, NC); +} + +int32_t main(int32_t argc, char *argv[]) { + parseArgument(argc, argv); + + if (showTablesFlag) { + showTables(); + return 0; + } + + if (queryFlag) { + // selectRowsFromTable(); + return 0; + } + + if (createTable) { + createDbAndStb(); + } + + pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables, + startOffset); + + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + SThreadInfo *pInfo = (SThreadInfo *)taosMemoryCalloc(numOfThreads, sizeof(SThreadInfo)); + + // int64_t numOfTablesPerThread = numOfTables / numOfThreads; + // numOfTables = numOfTablesPerThread * numOfThreads; + + if (numOfThreads < 1) { + numOfThreads = 1; + } + + int64_t a = numOfTables / numOfThreads; + if (a < 1) { + numOfThreads = numOfTables; + a = 1; + } + + int64_t b = 0; + b = numOfTables % numOfThreads; + + int64_t tableFrom = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + pInfo[i].tableBeginIndex = tableFrom; + pInfo[i].tableEndIndex = (i < b ? tableFrom + a : tableFrom + a - 1); + tableFrom = pInfo[i].tableEndIndex + 1; + pInfo[i].threadIndex = i; + pInfo[i].minDelay = INT64_MAX; + strcpy(pInfo[i].dbName, dbName); + strcpy(pInfo[i].stbName, stbName); + taosThreadCreate(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); + } + + taosMsleep(300); + for (int32_t i = 0; i < numOfThreads; i++) { + taosThreadJoin(pInfo[i].thread, NULL); + taosThreadClear(&pInfo[i].thread); + } + + int64_t maxDelay = 0; + int64_t minDelay = INT64_MAX; + + float createTableSpeed = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + createTableSpeed += pInfo[i].createTableSpeed; + + if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay; + if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; + } + + float insertDataSpeed = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + insertDataSpeed += pInfo[i].insertDataSpeed; + } + + if (createTable) { + pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 + "us %s", + GREEN, numOfTables, createTableSpeed, numOfThreads, maxDelay, minDelay, NC); + } + + if (insertData) { + pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, + numOfThreads, NC); + } + + taosThreadAttrDestroy(&thattr); + taosMemoryFree(pInfo); +} diff --git a/utils/test/c/sdbDump.c b/utils/test/c/sdbDump.c new file mode 100644 index 0000000000000000000000000000000000000000..b90b781e4469f004df742b047d046ff7574601ba --- /dev/null +++ b/utils/test/c/sdbDump.c @@ -0,0 +1,475 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmMgmt.h" +#include "mndInt.h" +#include "sdb.h" +#include "tconfig.h" +#include "tjson.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-result" + +#define TMP_DNODE_DIR TD_TMP_DIR_PATH "dumpsdb" +#define TMP_MNODE_DIR TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" +#define TMP_SDB_DATA_DIR TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "data" +#define TMP_SDB_SYNC_DIR TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync" +#define TMP_SDB_MNODE_JSON TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "mnode.json" +#define TMP_SDB_DATA_FILE TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "data" TD_DIRSEP "sdb.data" +#define TMP_SDB_RAFT_CFG_FILE TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_config.json" +#define TMP_SDB_RAFT_STORE_FILE TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_store.json" + +void reportStartup(const char *name, const char *desc) {} + +void sendRsp(SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); } + +int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { + terrno = TSDB_CODE_INVALID_PTR; + return -1; +} + +char *i642str(int64_t val) { + static char str[24] = {0}; + snprintf(str, sizeof(str), "%" PRId64, val); + return str; +} + +void dumpFunc(SSdb *pSdb, SJson *json) {} + +void dumpDb(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "dbs", items); + + while (1) { + SDbObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "db", item); + + tjsonAddStringToObject(item, "name", pObj->name); + tjsonAddStringToObject(item, "acct", pObj->acct); + tjsonAddStringToObject(item, "createUser", pObj->createUser); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); + tjsonAddStringToObject(item, "uid", i642str(pObj->uid)); + tjsonAddIntegerToObject(item, "cfgVersion", pObj->cfgVersion); + tjsonAddIntegerToObject(item, "vgVersion", pObj->vgVersion); + tjsonAddIntegerToObject(item, "numOfVgroups", pObj->cfg.numOfVgroups); + tjsonAddIntegerToObject(item, "numOfStables", pObj->cfg.numOfStables); + tjsonAddIntegerToObject(item, "buffer", pObj->cfg.buffer); + tjsonAddIntegerToObject(item, "pageSize", pObj->cfg.pageSize); + tjsonAddIntegerToObject(item, "pages", pObj->cfg.pages); + tjsonAddIntegerToObject(item, "cacheLastSize", pObj->cfg.cacheLastSize); + tjsonAddIntegerToObject(item, "daysPerFile", pObj->cfg.daysPerFile); + tjsonAddIntegerToObject(item, "daysToKeep0", pObj->cfg.daysToKeep0); + tjsonAddIntegerToObject(item, "daysToKeep1", pObj->cfg.daysToKeep1); + tjsonAddIntegerToObject(item, "daysToKeep2", pObj->cfg.daysToKeep2); + tjsonAddIntegerToObject(item, "minRows", pObj->cfg.minRows); + tjsonAddIntegerToObject(item, "maxRows", pObj->cfg.maxRows); + tjsonAddIntegerToObject(item, "precision", pObj->cfg.precision); + tjsonAddIntegerToObject(item, "compression", pObj->cfg.compression); + tjsonAddIntegerToObject(item, "replications", pObj->cfg.replications); + tjsonAddIntegerToObject(item, "strict", pObj->cfg.strict); + tjsonAddIntegerToObject(item, "cacheLast", pObj->cfg.cacheLast); + tjsonAddIntegerToObject(item, "hashMethod", pObj->cfg.hashMethod); + tjsonAddIntegerToObject(item, "numOfRetensions", pObj->cfg.numOfRetensions); + tjsonAddIntegerToObject(item, "schemaless", pObj->cfg.schemaless); + tjsonAddIntegerToObject(item, "walLevel", pObj->cfg.walLevel); + tjsonAddIntegerToObject(item, "walFsyncPeriod", pObj->cfg.walFsyncPeriod); + tjsonAddIntegerToObject(item, "walRetentionPeriod", pObj->cfg.walRetentionPeriod); + tjsonAddIntegerToObject(item, "walRetentionSize", pObj->cfg.walRetentionSize); + tjsonAddIntegerToObject(item, "walRollPeriod", pObj->cfg.walRollPeriod); + tjsonAddIntegerToObject(item, "walSegmentSize", pObj->cfg.walSegmentSize); + + sdbRelease(pSdb, pObj); + } +} + +void dumpStb(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "stbs", items); + + while (1) { + SStbObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "stb", item); + + tjsonAddStringToObject(item, "name", pObj->name); + tjsonAddStringToObject(item, "db", pObj->db); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); + tjsonAddStringToObject(item, "uid", i642str(pObj->uid)); + tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid)); + tjsonAddIntegerToObject(item, "tagVer", pObj->tagVer); + tjsonAddIntegerToObject(item, "colVer", pObj->colVer); + tjsonAddIntegerToObject(item, "nextColId", pObj->nextColId); + tjsonAddIntegerToObject(item, "watermark1", pObj->watermark[0]); + tjsonAddIntegerToObject(item, "watermark2", pObj->watermark[1]); + tjsonAddIntegerToObject(item, "maxdelay1", pObj->maxdelay[0]); + tjsonAddIntegerToObject(item, "maxdelay2", pObj->maxdelay[1]); + tjsonAddIntegerToObject(item, "ttl", pObj->ttl); + tjsonAddIntegerToObject(item, "numOfColumns", pObj->numOfColumns); + tjsonAddIntegerToObject(item, "numOfTags", pObj->numOfTags); + tjsonAddIntegerToObject(item, "commentLen", pObj->commentLen); + tjsonAddIntegerToObject(item, "ast1Len", pObj->ast1Len); + tjsonAddIntegerToObject(item, "ast2Len", pObj->ast2Len); + + sdbRelease(pSdb, pObj); + } +} + +void dumpSma(SSdb *pSdb, SJson *json) {} + +void dumpVgroup(SSdb *pSdb, SJson *json) {} + +void dumpTopic(SSdb *pSdb, SJson *json) {} + +void dumpConsumber(SSdb *pSdb, SJson *json) {} + +void dumpSubscribe(SSdb *pSdb, SJson *json) {} + +void dumpOffset(SSdb *pSdb, SJson *json) {} + +void dumpStream(SSdb *pSdb, SJson *json) {} + +void dumpAcct(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "accts", items); + + while (1) { + SAcctObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_ACCT, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "acct", item); + + tjsonAddStringToObject(item, "acct", pObj->acct); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); + tjsonAddIntegerToObject(item, "acctId", pObj->acctId); + + sdbRelease(pSdb, pObj); + } +} + +void dumpAuth(SSdb *pSdb, SJson *json) {} + +void dumpUser(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "users", items); + + while (1) { + SUserObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "user", item); + + tjsonAddStringToObject(item, "name", pObj->user); + tjsonAddStringToObject(item, "pass", pObj->pass); + tjsonAddStringToObject(item, "acct", pObj->acct); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); + tjsonAddIntegerToObject(item, "superUser", pObj->superUser); + tjsonAddIntegerToObject(item, "authVersion", pObj->authVersion); + tjsonAddIntegerToObject(item, "numOfReadDbs", taosHashGetSize(pObj->readDbs)); + tjsonAddIntegerToObject(item, "numOfWriteDbs", taosHashGetSize(pObj->writeDbs)); + + sdbRelease(pSdb, pObj); + } +} + +void dumpDnode(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "dnodes", items); + + while (1) { + SDnodeObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "dnode", item); + + tjsonAddIntegerToObject(item, "id", pObj->id); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); + tjsonAddIntegerToObject(item, "port", pObj->port); + tjsonAddStringToObject(item, "fqdn", pObj->fqdn); + + sdbRelease(pSdb, pObj); + } +} + +void dumpBnode(SSdb *pSdb, SJson *json) {} + +void dumpSnode(SSdb *pSdb, SJson *json) {} + +void dumpQnode(SSdb *pSdb, SJson *json) {} + +void dumpMnode(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "mnodes", items); + + while (1) { + SMnodeObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "mnode", item); + + tjsonAddIntegerToObject(item, "id", pObj->id); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); + + sdbRelease(pSdb, pObj); + } +} + +void dumpCluster(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "clusters", items); + + while (1) { + SClusterObj *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "cluster", item); + + tjsonAddStringToObject(item, "id", i642str(pObj->id)); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); + tjsonAddStringToObject(item, "name", pObj->name); + + sdbRelease(pSdb, pObj); + } +} + +void dumpTrans(SSdb *pSdb, SJson *json) { + void *pIter = NULL; + SJson *items = tjsonCreateObject(); + tjsonAddItemToObject(json, "transactions", items); + + while (1) { + STrans *pObj = NULL; + pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SJson *item = tjsonCreateObject(); + tjsonAddItemToObject(items, "trans", item); + + tjsonAddIntegerToObject(item, "id", pObj->id); + tjsonAddIntegerToObject(item, "stage", pObj->stage); + tjsonAddIntegerToObject(item, "policy", pObj->policy); + tjsonAddIntegerToObject(item, "conflict", pObj->conflict); + tjsonAddIntegerToObject(item, "exec", pObj->exec); + tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime)); + tjsonAddStringToObject(item, "dbname1", pObj->dbname1); + tjsonAddStringToObject(item, "dbname2", pObj->dbname2); + tjsonAddIntegerToObject(item, "commitLogNum", taosArrayGetSize(pObj->commitActions)); + tjsonAddIntegerToObject(item, "redoActionNum", taosArrayGetSize(pObj->redoActions)); + tjsonAddIntegerToObject(item, "undoActionNum", taosArrayGetSize(pObj->undoActions)); + + sdbRelease(pSdb, pObj); + } +} + +void dumpHeader(SSdb *pSdb, SJson *json) { + tjsonAddIntegerToObject(json, "sver", 1); + tjsonAddStringToObject(json, "applyIndex", i642str(pSdb->applyIndex)); + tjsonAddStringToObject(json, "applyTerm", i642str(pSdb->applyTerm)); + tjsonAddStringToObject(json, "applyConfig", i642str(pSdb->applyConfig)); + + SJson *maxIdsJson = tjsonCreateObject(); + tjsonAddItemToObject(json, "maxIds", maxIdsJson); + for (int32_t i = 0; i < SDB_MAX; ++i) { + int64_t maxId = 0; + if (i < SDB_MAX) { + maxId = pSdb->maxId[i]; + } + tjsonAddStringToObject(maxIdsJson, sdbTableName(i), i642str(maxId)); + } + + SJson *tableVersJson = tjsonCreateObject(); + tjsonAddItemToObject(json, "tableVers", tableVersJson); + for (int32_t i = 0; i < SDB_MAX; ++i) { + int64_t tableVer = 0; + if (i < SDB_MAX) { + tableVer = pSdb->tableVer[i]; + } + tjsonAddStringToObject(tableVersJson, sdbTableName(i), i642str(tableVer)); + } +} + +int32_t dumpSdb() { + wDebugFlag = 0; + mDebugFlag = 0; + sDebugFlag = 0; + + SMsgCb msgCb = {0}; + msgCb.reportStartupFp = reportStartup; + msgCb.sendReqFp = sendReq; + msgCb.sendRspFp = sendRsp; + msgCb.mgmt = (SMgmtWrapper *)(&msgCb); // hack + tmsgSetDefault(&msgCb); + walInit(); + syncInit(); + + SMnodeOpt opt = {.msgCb = msgCb}; + SMnode *pMnode = mndOpen(TMP_MNODE_DIR, &opt); + if (pMnode == NULL) return -1; + + SSdb *pSdb = pMnode->pSdb; + SJson *json = tjsonCreateObject(); + dumpHeader(pSdb, json); + dumpFunc(pSdb, json); + dumpDb(pSdb, json); + dumpStb(pSdb, json); + dumpSma(pSdb, json); + dumpVgroup(pSdb, json); + dumpTopic(pSdb, json); + dumpConsumber(pSdb, json); + dumpSubscribe(pSdb, json); + dumpOffset(pSdb, json); + dumpStream(pSdb, json); + dumpAcct(pSdb, json); + dumpAuth(pSdb, json); + dumpUser(pSdb, json); + dumpDnode(pSdb, json); + dumpBnode(pSdb, json); + dumpSnode(pSdb, json); + dumpQnode(pSdb, json); + dumpMnode(pSdb, json); + dumpCluster(pSdb, json); + dumpTrans(pSdb, json); + + char *pCont = tjsonToString(json); + int32_t contLen = strlen(pCont); + char file[] = "sdb.json"; + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to write %s since %s", file, terrstr()); + return -1; + } + taosWriteFile(pFile, pCont, contLen); + taosWriteFile(pFile, "\n", 1); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + tjsonDelete(json); + taosMemoryFree(pCont); + taosRemoveDir(TMP_DNODE_DIR); + return 0; +} + +int32_t parseArgs(int32_t argc, char *argv[]) { + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default is %s\n", configDir); + return -1; + } + } else { + printf("-c Configuration directory. \n"); + return -1; + } + } + + if (taosCreateLog("dumplog", 1, configDir, NULL, NULL, NULL, NULL, 1) != 0) { + printf("failed to dump since init log error\n"); + return -1; + } + + if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) { + printf("failed to dump since read config error\n"); + return -1; + } + + char mnodeJson[PATH_MAX] = {0}; + char dataFile[PATH_MAX] = {0}; + char raftCfgFile[PATH_MAX] = {0}; + char raftStoreFile[PATH_MAX] = {0}; + snprintf(mnodeJson, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "mnode.json", tsDataDir); + snprintf(dataFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "data" TD_DIRSEP "sdb.data", tsDataDir); + snprintf(raftCfgFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_config.json", tsDataDir); + snprintf(raftStoreFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_store.json", tsDataDir); + + char cmd[PATH_MAX * 2] = {0}; + snprintf(cmd, sizeof(cmd), "rm -rf %s", TMP_DNODE_DIR); + + system(cmd); +#ifdef WINDOWS + taosMulMkDir(TMP_SDB_DATA_DIR); + taosMulMkDir(TMP_SDB_SYNC_DIR); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", mnodeJson, TMP_SDB_MNODE_JSON); + system(cmd); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", dataFile, TMP_SDB_DATA_FILE); + system(cmd); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", raftCfgFile, TMP_SDB_RAFT_CFG_FILE); + system(cmd); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", raftStoreFile, TMP_SDB_RAFT_STORE_FILE); + system(cmd); +#else + snprintf(cmd, sizeof(cmd), "mkdir -p %s", TMP_SDB_DATA_DIR); + system(cmd); + snprintf(cmd, sizeof(cmd), "mkdir -p %s", TMP_SDB_SYNC_DIR); + system(cmd); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", mnodeJson, TMP_SDB_MNODE_JSON); + system(cmd); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", dataFile, TMP_SDB_DATA_FILE); + system(cmd); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", raftCfgFile, TMP_SDB_RAFT_CFG_FILE); + system(cmd); + snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", raftStoreFile, TMP_SDB_RAFT_STORE_FILE); + system(cmd); +#endif + + strcpy(tsDataDir, TMP_DNODE_DIR); + return 0; +} + +int32_t main(int32_t argc, char *argv[]) { + if (parseArgs(argc, argv) != 0) { + return -1; + } + + return dumpSdb(); +} + +#pragma GCC diagnostic pop \ No newline at end of file diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c new file mode 100644 index 0000000000000000000000000000000000000000..1fd1def263fb9a142ca7df4be1a8bed839ea098e --- /dev/null +++ b/utils/test/c/sml_test.c @@ -0,0 +1,1191 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "taos.h" +#include "types.h" + +int smlProcess_influx_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 " + "load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=" + "124,velocity=0,heading=221,grade=0 1451606401000000000", + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 " + "load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=" + "124,velocity=0,heading=221,grade=0,fuel_consumption=25 1451607402000000000", + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 " + "load_capacity=1500,fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=" + "124,heading=221,grade=0,fuel_consumption=25 1451608403000000000", + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 " + "fuel_capacity=150,nominal_fuel_consumption=12,latitude=52.31854,longitude=4.72037,elevation=124,velocity=0," + "heading=221,grade=0,fuel_consumption=25 1451609404000000000", + "readings,name=truck_0,fleet=South,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 " + "1451619405000000000", + "readings,name=truck_1,fleet=South,driver=Albert,model=F-150,device_version=v1.5 " + "load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=72.45258,longitude=68.83761,elevation=" + "255,velocity=0,heading=181,grade=0,fuel_consumption=25 1451606406000000000", + "readings,name=truck_2,driver=Derek,model=F-150,device_version=v1.5 " + "load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=" + "428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451606407000000000", + "readings,name=truck_2,fleet=North,driver=Derek,model=F-150 " + "load_capacity=2000,fuel_capacity=200,nominal_fuel_consumption=15,latitude=24.5208,longitude=28.09377,elevation=" + "428,velocity=0,heading=304,grade=0,fuel_consumption=25 1451609408000000000", + "readings,fleet=South,name=truck_0,driver=Trish,model=H-2,device_version=v2.3 fuel_consumption=25,grade=0 " + "1451629409000000000", + "stable,t1=t1,t2=t2,t3=t3 c1=1,c2=2,c3=\"kk\",c4=4 1451629501000000000", + "stable,t2=t2,t1=t1,t3=t3 c1=1,c3=\"\",c4=4 1451629602000000000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + + return code; +} + +int smlProcess_telnet_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", + "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", + "sys.if.bytes.out 1479496102 1.3E3 network=tcp", + " sys.procs.running 1479496100 42 host=web01 "}; + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int smlProcess_json1_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "[" + " {" + " \"metric\": \"sys.cpu.nice\"," + " \"timestamp\": 0," + " \"value\": 18," + " \"tags\": {" + " \"host\": \"web01\"," + " \"id\": \"t1\"," + " \"dc\": \"lga\"" + " }" + " }," + " {" + " \"metric\": \"sys.cpu.nice\"," + " \"timestamp\": 1346846400," + " \"value\": 9," + " \"tags\": {" + " \"host\": \"web02\"," + " \"dc\": \"lga\"" + " }" + " }" + "]"}; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int smlProcess_json2_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "{" + " \"metric\": \"meter_current0\"," + " \"timestamp\": {" + " \"value\" : 1346846400," + " \"type\" : \"s\"" + " }," + " \"value\": {" + " \"value\" : 10.3," + " \"type\" : \"i64\"" + " }," + " \"tags\": {" + " \"groupid\": { " + " \"value\" : 2," + " \"type\" : \"bigint\"" + " }," + " \"location\": { " + " \"value\" : \"北京\"," + " \"type\" : \"binary\"" + " }," + " \"id\": \"d1001\"" + " }" + "}"}; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int smlProcess_json3_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "{" + " \"metric\": \"meter_current1\"," + " \"timestamp\": {" + " \"value\" : 1346846400," + " \"type\" : \"s\"" + " }," + " \"value\": {" + " \"value\" : 10.3," + " \"type\" : \"i64\"" + " }," + " \"tags\": {" + " \"t1\": { " + " \"value\" : 2," + " \"type\" : \"bigint\"" + " }," + " \"t2\": { " + " \"value\" : 2," + " \"type\" : \"int\"" + " }," + " \"t3\": { " + " \"value\" : 2," + " \"type\" : \"i16\"" + " }," + " \"t4\": { " + " \"value\" : 2," + " \"type\" : \"i8\"" + " }," + " \"t5\": { " + " \"value\" : 2," + " \"type\" : \"f32\"" + " }," + " \"t6\": { " + " \"value\" : 2," + " \"type\" : \"double\"" + " }," + " \"t7\": { " + " \"value\" : \"8323\"," + " \"type\" : \"binary\"" + " }," + " \"t8\": { " + " \"value\" : \"北京\"," + " \"type\" : \"nchar\"" + " }," + " \"t9\": { " + " \"value\" : true," + " \"type\" : \"bool\"" + " }," + " \"id\": \"d1001\"" + " }" + "}"}; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int smlProcess_json4_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "{" + " \"metric\": \"meter_current2\"," + " \"timestamp\": {" + " \"value\" : 1346846500000," + " \"type\" : \"ms\"" + " }," + " \"value\": \"ni\"," + " \"tags\": {" + " \"t1\": { " + " \"value\" : 20," + " \"type\" : \"i64\"" + " }," + " \"t2\": { " + " \"value\" : 25," + " \"type\" : \"i32\"" + " }," + " \"t3\": { " + " \"value\" : 2," + " \"type\" : \"smallint\"" + " }," + " \"t4\": { " + " \"value\" : 2," + " \"type\" : \"tinyint\"" + " }," + " \"t5\": { " + " \"value\" : 2," + " \"type\" : \"float\"" + " }," + " \"t6\": { " + " \"value\" : 0.2," + " \"type\" : \"f64\"" + " }," + " \"t7\": \"nsj\"," + " \"t8\": { " + " \"value\" : \"北京\"," + " \"type\" : \"nchar\"" + " }," + " \"t9\": false," + " \"id\": \"d1001\"" + " }" + "}"}; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_TD15662_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db precision 'ns' schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "hetrey c0=f,c1=127i8 1626006833639", + "hetrey,t1=r c0=f,c1=127i8 1626006833640", + }; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_TD15742_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "test_ms,t0=t c0=f 1626006833641", + }; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_16384_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + const char *sql[] = { + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=t,c1=127i8 1626006833639000000", + }; + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, 1, TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + if(code) return code; + + const char *sql1[] = { + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=f,c1=127i8,c11=L\"ncharColValue\",c10=t 1626006833639000000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql1, 1, TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_oom_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + const char *sql[] = { + //"test_ms,t0=t c0=f 1626006833641", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pgxbrbga\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"gviggpmi\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"cexkarjn\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"rzwwuoxu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"xphrlkey\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"llsawebj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"jwpkipff\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"euzzhcvu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"jumhnsvw\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"fnetgdhj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"vrmmpgqe\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"lnpfjapr\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"gvbhmsfr\",t8=L\"ncharTagValue\" " + "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"kydxrxwc\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pfyarryq\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"uxptotap\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"prolhudh\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ttxaxnac\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"dfgvmjmz\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"bloextkn\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"dvjxwzsi\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"aigjomaf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"refbidtf\",t8=L\"ncharTagValue\" " + "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"vuanlfpz\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"nbpajxkx\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ktzzauxh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"prcwdjct\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"vmbhvjtp\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"liuddtuz\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"pddsktow\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"algldlvl\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"mlmnjgdl\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"oiynpcog\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"wmynbagb\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"asvyulrm\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ohaacrkp\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"ytyejhiq\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"bbznuerb\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"lpebcibw\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"xmqrbafv\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"lnmwpdne\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"jpcsjqun\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"mmxqmavz\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"hhsbgaow\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"uwogyuud\",t8=L\"ncharTagValue\" " + "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ytxpaxnk\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"wouwdvtt\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"iitwikkh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"lgyzuyaq\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"bdtiigxi\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"qpnsvdhw\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"pjxihgvu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"ksxkfetn\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ocukufqs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"qzerxmpe\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"qwcfdyxs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"jldrpmmd\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"lucxlfzc\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"rcewrvya\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"dknvaphs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"nxtxgzdr\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"mbvuugwz\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"uikakffu\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"mwmtqsma\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"bfcxrrpa\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ksajygdj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"vmhhszyv\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"urwjgvut\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"jrvytcxy\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"evqkzygh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"zitdznhg\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"tpqekrxa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"yrrbgjtk\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"bnphiuyq\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"huknehjn\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"iudbxfke\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"fjmolwbn\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"gukzgcjs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"bjvdtlgq\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"phxnesxh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"qgpgckvc\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"yechqtfa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pbouxywy\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"kxtuojyo\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"txaniwlj\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"fixgufrj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"okzvalwq\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"iitawgbn\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"gayvmird\",t8=L\"ncharTagValue\" " + "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"dprkfjph\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"kmuccshq\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"vkslsdsd\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"dukccdqk\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"leztxmqf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"kltixbwz\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"xqhkweef\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"idxsimvz\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"vbruvcpk\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"uxandqkd\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"dsiosysh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"kxuyanpp\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"wkrktags\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"yvizzpiv\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ddnefben\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"novmfmbc\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"fnusxsfu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"ouerfjap\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"sigognkf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"slvzhede\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"bknerect\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"tmhcdfjb\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"hpnoanpp\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"okmhelnc\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"xcernjin\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"jdmiismg\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"tmnqozrf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"zgwrftkx\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"zyamlwwh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"nuedqcro\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"lpsvyqaa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"mneitsul\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"vpleinwb\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"njxuaedy\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"sdgxpqmu\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"yjirrebp\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ikqndzfj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"ghnfdxhr\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"hrwczpvo\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"nattumpb\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"zoyfzazn\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"rdwemofy\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"phkgsjeg\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pyhvvjrt\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"zfslyton\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"bxwjzeri\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"uovzzgjv\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"cfjmacvr\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"jefqgzqx\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"njrksxmr\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"mhvabvgn\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"kfekjltr\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"lexfaaby\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"zbblsmwq\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"oqcombkx\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"rcdmhzyw\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"otksuean\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"itbdvowq\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"tswtmhex\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"xoukkzid\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"guangmpq\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"rayxzuky\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"lspwucrv\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pdprzzkf\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"sddqrtza\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"kabndgkx\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"aglnqqxs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"fiwpzmdr\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"hxctooen\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pckjpwyh\",t8=L\"ncharTagValue\" " + "c0=false,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ivmvsbai\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"eljdclst\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"rwgdctie\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"zlnthxoz\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"ljtxelle\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"llfggdpy\",t8=L\"ncharTagValue\" " + "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"tvnridze\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"hxjpgube\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"zmldmquq\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"bggqwcoj\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"drksfofm\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"jcsixens\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"cdwnwhaf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"nngpumuq\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"hylgooci\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"cozeyjys\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"lcgpfcsa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"qdtzhtyd\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"txpubynb\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"gbslzbtu\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"buihcpcl\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"ayqezaiq\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"zgkgtilj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"bcjopqif\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"mfzxiaqt\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"xmnlqxoj\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"reyiklyf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"xssuomhk\",t8=L\"ncharTagValue\" " + "c0=False,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"liazkjll\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"nigjlblo\",t8=L\"ncharTagValue\" " + "c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"vmojyznk\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"dotkbvrz\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"kuwdyydw\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"slsfqydw\",t8=L\"ncharTagValue\" " + "c0=t,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"zyironhd\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pktwfhzi\",t8=L\"ncharTagValue\" " + "c0=T,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"xybavsvh\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"pyrxemvx\",t8=L\"ncharTagValue\" " + "c0=True,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"tlfihwjs\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ogirwqci,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"neumakmg\",t8=L\"ncharTagValue\" " + "c0=F,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"wxqingoa\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + }; + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_16368_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "[{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639000, \"type\": \"us\"}, \"value\": 1, " + "\"tags\": {\"t1\": 3, \"t2\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t3\", \"type\": " + "\"binary\"}}}," + "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833739000, \"type\": \"us\"}, \"value\": 2, " + "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " + "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," + "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639100, \"type\": \"us\"}, \"value\": 3, " + "\"tags\": {\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t3\": {\"value\": \"ste\", \"type\": \"nchar\"}}}," + "{\"metric\": \"stf567890\", \"timestamp\": {\"value\": 1626006833639200, \"type\": \"us\"}, \"value\": 4, " + "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"bigint\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " + "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," + "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639300, \"type\": \"us\"}, \"value\": " + "{\"value\": 5, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t2\": 5.0, " + "\"t3\": {\"value\": \"t4\", \"type\": \"binary\"}}}," + "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639400, \"type\": \"us\"}, \"value\": " + "{\"value\": 6, \"type\": \"double\"}, \"tags\": {\"t2\": 5.0, \"t3\": {\"value\": \"ste2\", \"type\": " + "\"nchar\"}}}," + "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006834639400, \"type\": \"us\"}, \"value\": " + "{\"value\": 7, \"type\": \"double\"}, \"tags\": {\"t2\": {\"value\": 5.0, \"type\": \"double\"}, \"t3\": " + "{\"value\": \"ste2\", \"type\": \"nchar\"}}}," + "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": " + "{\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": " + "{\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, " + "\"type\": \"double\"}}}," + "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": " + "{\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " + "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]"}; + pRes = taos_schemaless_insert(taos, (char **)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_dup_time_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + const char *sql[] = {//"test_ms,t0=t c0=f 1626006833641", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=false,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000"}; + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_16960_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "[" + "{" + "\"timestamp\":" + "" + "{ \"value\": 1349020800000, \"type\": \"ms\" }" + "," + "\"value\":" + "" + "{ \"value\": 830525384, \"type\": \"int\" }" + "," + "\"tags\": {" + "\"id\": \"stb00_0\"," + "\"t0\":" + "" + "{ \"value\": 83972721, \"type\": \"int\" }" + "," + "\"t1\":" + "" + "{ \"value\": 539147525, \"type\": \"int\" }" + "," + "\"t2\":" + "" + "{ \"value\": 618258572, \"type\": \"int\" }" + "," + "\"t3\":" + "" + "{ \"value\": -10536201, \"type\": \"int\" }" + "," + "\"t4\":" + "" + "{ \"value\": 349227409, \"type\": \"int\" }" + "," + "\"t5\":" + "" + "{ \"value\": 249347042, \"type\": \"int\" }" + "}," + "\"metric\": \"stb0\"" + "}," + "{" + "\"timestamp\":" + "" + "{ \"value\": 1349020800001, \"type\": \"ms\" }" + "," + "\"value\":" + "" + "{ \"value\": -588348364, \"type\": \"int\" }" + "," + "\"tags\": {" + "\"id\": \"stb00_0\"," + "\"t0\":" + "" + "{ \"value\": 83972721, \"type\": \"int\" }" + "," + "\"t1\":" + "" + "{ \"value\": 539147525, \"type\": \"int\" }" + "," + "\"t2\":" + "" + "{ \"value\": 618258572, \"type\": \"int\" }" + "," + "\"t3\":" + "" + "{ \"value\": -10536201, \"type\": \"int\" }" + "," + "\"t4\":" + "" + "{ \"value\": 349227409, \"type\": \"int\" }" + "," + "\"t5\":" + "" + "{ \"value\": 249347042, \"type\": \"int\" }" + "}," + "\"metric\": \"stb0\"" + "}," + "{" + "\"timestamp\":" + "" + "{ \"value\": 1349020800002, \"type\": \"ms\" }" + "," + "\"value\":" + "" + "{ \"value\": -370310823, \"type\": \"int\" }" + "," + "\"tags\": {" + "\"id\": \"stb00_0\"," + "\"t0\":" + "" + "{ \"value\": 83972721, \"type\": \"int\" }" + "," + "\"t1\":" + "" + "{ \"value\": 539147525, \"type\": \"int\" }" + "," + "\"t2\":" + "" + "{ \"value\": 618258572, \"type\": \"int\" }" + "," + "\"t3\":" + "" + "{ \"value\": -10536201, \"type\": \"int\" }" + "," + "\"t4\":" + "" + "{ \"value\": 349227409, \"type\": \"int\" }" + "," + "\"t5\":" + "" + "{ \"value\": 249347042, \"type\": \"int\" }" + "}," + "\"metric\": \"stb0\"" + "}," + "{" + "\"timestamp\":" + "" + "{ \"value\": 1349020800003, \"type\": \"ms\" }" + "," + "\"value\":" + "" + "{ \"value\": -811250191, \"type\": \"int\" }" + "," + "\"tags\": {" + "\"id\": \"stb00_0\"," + "\"t0\":" + "" + "{ \"value\": 83972721, \"type\": \"int\" }" + "," + "\"t1\":" + "" + "{ \"value\": 539147525, \"type\": \"int\" }" + "," + "\"t2\":" + "" + "{ \"value\": 618258572, \"type\": \"int\" }" + "," + "\"t3\":" + "" + "{ \"value\": -10536201, \"type\": \"int\" }" + "," + "\"t4\":" + "" + "{ \"value\": 349227409, \"type\": \"int\" }" + "," + "\"t5\":" + "" + "{ \"value\": 249347042, \"type\": \"int\" }" + "}," + "\"metric\": \"stb0\"" + "}," + "{" + "\"timestamp\":" + "" + "{ \"value\": 1349020800004, \"type\": \"ms\" }" + "," + "\"value\":" + "" + "{ \"value\": -330340558, \"type\": \"int\" }" + "," + "\"tags\": {" + "\"id\": \"stb00_0\"," + "\"t0\":" + "" + "{ \"value\": 83972721, \"type\": \"int\" }" + "," + "\"t1\":" + "" + "{ \"value\": 539147525, \"type\": \"int\" }" + "," + "\"t2\":" + "" + "{ \"value\": 618258572, \"type\": \"int\" }" + "," + "\"t3\":" + "" + "{ \"value\": -10536201, \"type\": \"int\" }" + "," + "\"t4\":" + "" + "{ \"value\": 349227409, \"type\": \"int\" }" + "," + "\"t5\":" + "" + "{ \"value\": 249347042, \"type\": \"int\" }" + "}," + "\"metric\": \"stb0\"" + "}" + "]"}; + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + return code; +} + +int sml_add_tag_col_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + const char *sql[] = { + "macylr,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000" + }; + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + if (code) return code; + + const char *sql1[] = { + "macylr,id=macylr_17875_1804,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\",t11=127i8,t10=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c8=L\"ncharColValue\",c9=7u64,c11=L\"ncharColValue\",c10=f 1626006833639000000" + }; + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + code = taos_errno(pRes); + taos_free_result(pRes); + + return code; +} + +int smlProcess_18784_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "disk,device=sdc inodes_used=176059i,total=1081101176832i 1661943960000000000", + "disk,device=sdc inodes_free=66932805i 1661943960000000000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); + int code = taos_errno(pRes); + ASSERT(!code); + ASSERT(taos_affected_rows(pRes) == 2); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from disk"); + ASSERT(pRes); + int fieldNum = taos_field_count(pRes); + ASSERT(fieldNum == 5); + printf("fieldNum:%d\n", fieldNum); + TAOS_ROW row = NULL; + int32_t rowIndex = 0; + while((row = taos_fetch_row(pRes)) != NULL) { + int64_t ts = *(int64_t*)row[0]; + int64_t used = *(int64_t*)row[1]; + int64_t total = *(int64_t*)row[2]; + int64_t freed = *(int64_t*)row[3]; + if(rowIndex == 0){ + ASSERT(ts == 1661943960000); + ASSERT(used == 176059); + ASSERT(total == 1081101176832); + ASSERT(freed == 66932805); +// ASSERT_EQ(latitude, 24.5208); +// ASSERT_EQ(longitude, 28.09377); +// ASSERT_EQ(elevation, 428); +// ASSERT_EQ(velocity, 0); +// ASSERT_EQ(heading, 304); +// ASSERT_EQ(grade, 0); +// ASSERT_EQ(fuel_consumption, 25); + }else{ +// ASSERT(0); + } + rowIndex++; + } + taos_free_result(pRes); + + return code; +} + +int main(int argc, char *argv[]) { + int ret = 0; + ret = smlProcess_influx_Test(); + ASSERT(!ret); + ret = smlProcess_telnet_Test(); + ASSERT(!ret); + ret = smlProcess_json1_Test(); + ASSERT(!ret); + ret = smlProcess_json2_Test(); + ASSERT(!ret); + ret = smlProcess_json3_Test(); + ASSERT(!ret); + ret = smlProcess_json4_Test(); + ASSERT(!ret); + ret = sml_TD15662_Test(); + ASSERT(!ret); + ret = sml_TD15742_Test(); + ASSERT(!ret); + ret = sml_16384_Test(); + ASSERT(!ret); + ret = sml_oom_Test(); + ASSERT(!ret); + ret = sml_16368_Test(); + ASSERT(!ret); + ret = sml_dup_time_Test(); + ASSERT(!ret); + ret = sml_16960_Test(); + ASSERT(!ret); + ret = sml_add_tag_col_Test(); + ASSERT(!ret); + ret = smlProcess_18784_Test(); + ASSERT(!ret); + return ret; +} diff --git a/utils/test/c/tmqDemo.c b/utils/test/c/tmqDemo.c new file mode 100644 index 0000000000000000000000000000000000000000..784b45c92bcb2b1b8e30cfdd7d60b31f279e2e8d --- /dev/null +++ b/utils/test/c/tmqDemo.c @@ -0,0 +1,705 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include +#include +// #include + +#include "taos.h" +#include "taoserror.h" +#include "tlog.h" + +#define GREEN "\033[1;32m" +#define NC "\033[0m" +#define min(a, b) (((a) < (b)) ? (a) : (b)) + +#define MAX_SQL_STR_LEN (1024 * 1024) +#define MAX_ROW_STR_LEN (16 * 1024) + +enum _RUN_MODE { + TMQ_RUN_INSERT_AND_CONSUME, + TMQ_RUN_ONLY_INSERT, + TMQ_RUN_ONLY_CONSUME, + TMQ_RUN_MODE_BUTT, +}; + +typedef struct { + char dbName[32]; + char stbName[64]; + char resultFileName[256]; + char vnodeWalPath[256]; + int32_t numOfThreads; + int32_t numOfTables; + int32_t numOfVgroups; + int32_t runMode; + int32_t numOfColumn; + double ratio; + int32_t batchNumOfRow; + int32_t totalRowsOfPerTbl; + int64_t startTimestamp; + int32_t showMsgFlag; + int32_t simCase; + + int32_t totalRowsOfT2; +} SConfInfo; + +static SConfInfo g_stConfInfo = { + "tmqdb", + "stb", + "./tmqResult.txt", // output_file + "", // /data2/dnode/data/vnode/vnode2/wal", + 1, // threads + 1, // tables + 1, // vgroups + 0, // run mode + 1, // columns + 1, // ratio + 1, // batch size + 10000, // total rows for per table + 0, // 2020-01-01 00:00:00.000 + 0, // show consume msg switch + 0, // if run in sim case + 10000, +}; + +char* g_pRowValue = NULL; +TdFilePtr g_fp = NULL; + +static void printHelp() { + char indent[10] = " "; + printf("Used to test the performance while create table\n"); + + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-d"); + printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", g_stConfInfo.dbName); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", g_stConfInfo.stbName); + printf("%s%s\n", indent, "-f"); + printf("%s%s%s%s\n", indent, indent, "The file of result, default is ", g_stConfInfo.resultFileName); + printf("%s%s\n", indent, "-w"); + printf("%s%s%s%s\n", indent, indent, "The path of vnode of wal, default is ", g_stConfInfo.vnodeWalPath); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", g_stConfInfo.numOfThreads); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s%d\n", indent, indent, "numOfTables, default is ", g_stConfInfo.numOfTables); + printf("%s%s\n", indent, "-v"); + printf("%s%s%s%d\n", indent, indent, "numOfVgroups, default is ", g_stConfInfo.numOfVgroups); + printf("%s%s\n", indent, "-a"); + printf("%s%s%s%d\n", indent, indent, "runMode, default is ", g_stConfInfo.runMode); + printf("%s%s\n", indent, "-l"); + printf("%s%s%s%d\n", indent, indent, "numOfColumn, default is ", g_stConfInfo.numOfColumn); + printf("%s%s\n", indent, "-q"); + printf("%s%s%s%f\n", indent, indent, "ratio, default is ", g_stConfInfo.ratio); + printf("%s%s\n", indent, "-b"); + printf("%s%s%s%d\n", indent, indent, "batchNumOfRow, default is ", g_stConfInfo.batchNumOfRow); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "totalRowsOfPerTbl, default is ", g_stConfInfo.totalRowsOfPerTbl); + printf("%s%s\n", indent, "-m"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "startTimestamp, default is ", g_stConfInfo.startTimestamp); + printf("%s%s\n", indent, "-g"); + printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); + printf("%s%s\n", indent, "-sim"); + printf("%s%s%s%d\n", indent, indent, "simCase, default is ", g_stConfInfo.simCase); + + exit(EXIT_SUCCESS); +} + +void parseArgument(int32_t argc, char* argv[]) { + g_stConfInfo.startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000 + + for (int32_t i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(g_stConfInfo.dbName, argv[++i]); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + strcpy(g_stConfInfo.stbName, argv[++i]); + } else if (strcmp(argv[i], "-w") == 0) { + strcpy(g_stConfInfo.vnodeWalPath, argv[++i]); + } else if (strcmp(argv[i], "-f") == 0) { + strcpy(g_stConfInfo.resultFileName, argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + g_stConfInfo.numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + g_stConfInfo.numOfTables = atoll(argv[++i]); + } else if (strcmp(argv[i], "-v") == 0) { + g_stConfInfo.numOfVgroups = atoi(argv[++i]); + } else if (strcmp(argv[i], "-a") == 0) { + g_stConfInfo.runMode = atoi(argv[++i]); + } else if (strcmp(argv[i], "-b") == 0) { + g_stConfInfo.batchNumOfRow = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + g_stConfInfo.totalRowsOfPerTbl = atoi(argv[++i]); + } else if (strcmp(argv[i], "-l") == 0) { + g_stConfInfo.numOfColumn = atoi(argv[++i]); + } else if (strcmp(argv[i], "-q") == 0) { + g_stConfInfo.ratio = atof(argv[++i]); + } else if (strcmp(argv[i], "-m") == 0) { + g_stConfInfo.startTimestamp = atol(argv[++i]); + } else if (strcmp(argv[i], "-g") == 0) { + g_stConfInfo.showMsgFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-sim") == 0) { + g_stConfInfo.simCase = atol(argv[++i]); + } else { + printf("%s unknow para: %s %s", GREEN, argv[++i], NC); + exit(-1); + } + } + + g_stConfInfo.totalRowsOfT2 = g_stConfInfo.totalRowsOfPerTbl * g_stConfInfo.ratio; + +#if 0 + pPrint("%s configDir:%s %s", GREEN, configDir, NC); + pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC); + pPrint("%s stbName:%s %s", GREEN, g_stConfInfo.stbName, NC); + pPrint("%s resultFileName:%s %s", GREEN, g_stConfInfo.resultFileName, NC); + pPrint("%s vnodeWalPath:%s %s", GREEN, g_stConfInfo.vnodeWalPath, NC); + pPrint("%s numOfTables:%d %s", GREEN, g_stConfInfo.numOfTables, NC); + pPrint("%s numOfThreads:%d %s", GREEN, g_stConfInfo.numOfThreads, NC); + pPrint("%s numOfVgroups:%d %s", GREEN, g_stConfInfo.numOfVgroups, NC); + pPrint("%s runMode:%d %s", GREEN, g_stConfInfo.runMode, NC); + pPrint("%s ratio:%f %s", GREEN, g_stConfInfo.ratio, NC); + pPrint("%s numOfColumn:%d %s", GREEN, g_stConfInfo.numOfColumn, NC); + pPrint("%s batchNumOfRow:%d %s", GREEN, g_stConfInfo.batchNumOfRow, NC); + pPrint("%s totalRowsOfPerTbl:%d %s", GREEN, g_stConfInfo.totalRowsOfPerTbl, NC); + pPrint("%s totalRowsOfT2:%d %s", GREEN, g_stConfInfo.totalRowsOfT2, NC); + pPrint("%s startTimestamp:%" PRId64" %s", GREEN, g_stConfInfo.startTimestamp, NC); + pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); +#endif +} + +static int running = 1; +/*static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }*/ + +// calc dir size (not include itself 4096Byte) +int64_t getDirectorySize(char* dir) { + TdDirPtr pDir; + TdDirEntryPtr pDirEntry; + int64_t totalSize = 0; + + if ((pDir = taosOpenDir(dir)) == NULL) { + fprintf(stderr, "Cannot open dir: %s\n", dir); + return -1; + } + + // lstat(dir, &statbuf); + // totalSize+=statbuf.st_size; + + while ((pDirEntry = taosReadDir(pDir)) != NULL) { + char subdir[1024]; + char* fileName = taosGetDirEntryName(pDirEntry); + sprintf(subdir, "%s/%s", dir, fileName); + + // printf("===d_name: %s\n", entry->d_name); + if (taosIsDir(subdir)) { + if (strcmp(".", fileName) == 0 || strcmp("..", fileName) == 0) { + continue; + } + + int64_t subDirSize = getDirectorySize(subdir); + totalSize += subDirSize; + } else if (0 == strcmp(strchr(fileName, '.'), ".log")) { // only calc .log file size, and not include .idx file + int64_t file_size = 0; + taosStatFile(subdir, &file_size, NULL); + totalSize += file_size; + } + } + + taosCloseDir(&pDir); + return totalSize; +} + +int queryDB(TAOS* taos, char* command) { + TAOS_RES* pRes = taos_query(taos, command); + int code = taos_errno(pRes); + if (code != 0) { + pError("failed to reason:%s, sql: %s", tstrerror(code), command); + taos_free_result(pRes); + return -1; + } + taos_free_result(pRes); + return 0; +} + +int32_t init_env() { + char sqlStr[1024] = {0}; + + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + sprintf(sqlStr, "create database if not exists %s vgroups %d", g_stConfInfo.dbName, g_stConfInfo.numOfVgroups); + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + sprintf(sqlStr, "use %s", g_stConfInfo.dbName); + pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + // create row value + g_pRowValue = (char*)taosMemoryCalloc(1, g_stConfInfo.numOfColumn * 16 + 128); + if (NULL == g_pRowValue) { + return -1; + } + + int32_t dataLen = 0; + int32_t sqlLen = 0; + sqlLen += sprintf(sqlStr + sqlLen, "create stable if not exists %s (ts timestamp, ", g_stConfInfo.stbName); + for (int32_t i = 0; i < g_stConfInfo.numOfColumn; i++) { + if (i == g_stConfInfo.numOfColumn - 1) { + sqlLen += sprintf(sqlStr + sqlLen, "c%d int) ", i); + memcpy(g_pRowValue + dataLen, "66778899", strlen("66778899")); + dataLen += strlen("66778899"); + } else { + sqlLen += sprintf(sqlStr + sqlLen, "c%d int, ", i); + memcpy(g_pRowValue + dataLen, "66778899, ", strlen("66778899, ")); + dataLen += strlen("66778899, "); + } + } + sqlLen += sprintf(sqlStr + sqlLen, "tags (t0 int)"); + + pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("failed to create super table %s, reason:%s\n", g_stConfInfo.stbName, taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + for (int32_t i = 0; i < g_stConfInfo.numOfTables; i++) { + sprintf(sqlStr, "create table if not exists %s%d using %s tags(1)", g_stConfInfo.stbName, i, g_stConfInfo.stbName); + pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("failed to create child table %s%d, reason:%s\n", g_stConfInfo.stbName, i, taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + } + + // const char* sql = "select * from tu1"; + sprintf(sqlStr, "create topic test_stb_topic_1 as select ts,c0 from %s", g_stConfInfo.stbName); + /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sqlStr, strlen(sqlStr));*/ + pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + taos_close(pConn); + return 0; +} + +tmq_t* build_consumer() { +#if 0 + char sqlStr[1024] = {0}; + + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + sprintf(sqlStr, "use %s", g_stConfInfo.dbName); + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); +#endif + + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); + tmq_conf_destroy(conf); + return tmq; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "test_stb_topic_1"); + return topic_list; +} + +void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { + static const int MIN_COMMIT_COUNT = 1000; + + int msg_count = 0; + int32_t err; + + if ((err = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); + return; + } + + while (running) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1); + if (tmqmessage) { + /*msg_process(tmqmessage);*/ + taos_free_result(tmqmessage); + + if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit_sync(tmq, NULL); + } + } + + err = tmq_consumer_close(tmq); + if (err) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLogSize) { + int32_t err; + + if ((err = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); + printf("subscribe err\n"); + return; + } + /*taosSsleep(3);*/ + int32_t batchCnt = 0; + int64_t startTime = taosGetTimestampUs(); + while (running) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 3000); + if (tmqmessage) { + batchCnt++; + if (0 != g_stConfInfo.showMsgFlag) { + /*msg_process(tmqmessage);*/ + } + taos_free_result(tmqmessage); + } else { + break; + } + } + int64_t endTime = taosGetTimestampUs(); + double consumeTime = (double)(endTime - startTime) / 1000000; + + if (batchCnt != totalMsgs) { + printf("%s inserted msgs: %d and consume msgs: %d mismatch %s", GREEN, totalMsgs, batchCnt, NC); + /*exit(-1);*/ + } + + if (0 == g_stConfInfo.simCase) { + printf("consume result: msgs: %d, time used:%.3f second\n", batchCnt, consumeTime); + } else { + printf("{consume success: %d}", totalMsgs); + } + taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.2f| %10.2f |\n", batchCnt, consumeTime, + (double)batchCnt / consumeTime, (double)walLogSize / (1024 * 1024.0) / consumeTime, + (double)walLogSize / 1024.0 / batchCnt); + + err = tmq_consumer_close(tmq); + if (err) { + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + exit(-1); + } +} + +// sync insertion +int32_t syncWriteData() { + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + char sqlStr[1024] = {0}; + sprintf(sqlStr, "use %s", g_stConfInfo.dbName); + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + char* buffer = NULL; + buffer = (char*)taosMemoryMalloc(MAX_SQL_STR_LEN); + if (NULL == buffer) { + return -1; + } + + int32_t totalMsgs = 0; + + int64_t time_counter = g_stConfInfo.startTimestamp; + for (int i = 0; i < g_stConfInfo.totalRowsOfPerTbl;) { + for (int tID = 0; tID <= g_stConfInfo.numOfTables - 1; tID++) { + int inserted = i; + int64_t tmp_time = time_counter; + + int32_t data_len = 0; + data_len += sprintf(buffer + data_len, "insert into %s%d values", g_stConfInfo.stbName, tID); + int k; + for (k = 0; k < g_stConfInfo.batchNumOfRow;) { + data_len += sprintf(buffer + data_len, "(%" PRId64 ", %s) ", tmp_time++, g_pRowValue); + inserted++; + k++; + + if (inserted >= g_stConfInfo.totalRowsOfPerTbl) { + break; + } + + if (data_len > MAX_SQL_STR_LEN - MAX_ROW_STR_LEN) { + break; + } + } + + int code = queryDB(pConn, buffer); + if (0 != code) { + fprintf(stderr, "insert data error!\n"); + taosMemoryFreeClear(buffer); + return -1; + } + + totalMsgs++; + + if (tID == g_stConfInfo.numOfTables - 1) { + i = inserted; + time_counter = tmp_time; + } + } + } + taosMemoryFreeClear(buffer); + return totalMsgs; +} + +// sync insertion +int32_t syncWriteDataByRatio() { + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + char sqlStr[1024] = {0}; + sprintf(sqlStr, "use %s", g_stConfInfo.dbName); + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + char* buffer = NULL; + buffer = (char*)taosMemoryMalloc(MAX_SQL_STR_LEN); + if (NULL == buffer) { + return -1; + } + + int32_t totalMsgs = 0; + + int32_t insertedOfT1 = 0; + int32_t insertedOfT2 = 0; + + int64_t tsOfT1 = g_stConfInfo.startTimestamp; + int64_t tsOfT2 = g_stConfInfo.startTimestamp; + int64_t tmp_time; + + for (;;) { + if ((insertedOfT1 >= g_stConfInfo.totalRowsOfPerTbl) && (insertedOfT2 >= g_stConfInfo.totalRowsOfT2)) { + break; + } + + for (int tID = 0; tID <= g_stConfInfo.numOfTables - 1; tID++) { + if (0 == tID) { + tmp_time = tsOfT1; + if (insertedOfT1 >= g_stConfInfo.totalRowsOfPerTbl) { + continue; + } + } else if (1 == tID) { + tmp_time = tsOfT2; + if (insertedOfT2 >= g_stConfInfo.totalRowsOfT2) { + continue; + } + } + + int32_t data_len = 0; + data_len += sprintf(buffer + data_len, "insert into %s%d values", g_stConfInfo.stbName, tID); + int k; + for (k = 0; k < g_stConfInfo.batchNumOfRow;) { + data_len += sprintf(buffer + data_len, "(%" PRId64 ", %s) ", tmp_time++, g_pRowValue); + k++; + if (0 == tID) { + insertedOfT1++; + if (insertedOfT1 >= g_stConfInfo.totalRowsOfPerTbl) { + break; + } + } else if (1 == tID) { + insertedOfT2++; + if (insertedOfT2 >= g_stConfInfo.totalRowsOfT2) { + break; + } + } + + if (data_len > MAX_SQL_STR_LEN - MAX_ROW_STR_LEN) { + break; + } + } + + int code = queryDB(pConn, buffer); + if (0 != code) { + fprintf(stderr, "insert data error!\n"); + taosMemoryFreeClear(buffer); + return -1; + } + + if (0 == tID) { + tsOfT1 = tmp_time; + } else if (1 == tID) { + tsOfT2 = tmp_time; + } + + totalMsgs++; + } + } + pPrint("expect insert rows: T1[%d] T2[%d], actual insert rows: T1[%d] T2[%d]\n", g_stConfInfo.totalRowsOfPerTbl, + g_stConfInfo.totalRowsOfT2, insertedOfT1, insertedOfT2); + taosMemoryFreeClear(buffer); + return totalMsgs; +} + +void printParaIntoFile() { + // FILE *fp = fopen(g_stConfInfo.resultFileName, "a"); + TdFilePtr pFile = + taosOpenFile(g_stConfInfo.resultFileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM); + if (NULL == pFile) { + fprintf(stderr, "Failed to open %s for save result\n", g_stConfInfo.resultFileName); + exit(-1); + } + g_fp = pFile; + + time_t tTime = taosGetTimestampSec(); + struct tm tm; + taosLocalTime(&tTime, &tm); + + taosFprintfFile(pFile, "###################################################################\n"); + taosFprintfFile(pFile, "# configDir: %s\n", configDir); + taosFprintfFile(pFile, "# dbName: %s\n", g_stConfInfo.dbName); + taosFprintfFile(pFile, "# stbName: %s\n", g_stConfInfo.stbName); + taosFprintfFile(pFile, "# vnodeWalPath: %s\n", g_stConfInfo.vnodeWalPath); + taosFprintfFile(pFile, "# numOfTables: %d\n", g_stConfInfo.numOfTables); + taosFprintfFile(pFile, "# numOfThreads: %d\n", g_stConfInfo.numOfThreads); + taosFprintfFile(pFile, "# numOfVgroups: %d\n", g_stConfInfo.numOfVgroups); + taosFprintfFile(pFile, "# runMode: %d\n", g_stConfInfo.runMode); + taosFprintfFile(pFile, "# ratio: %f\n", g_stConfInfo.ratio); + taosFprintfFile(pFile, "# numOfColumn: %d\n", g_stConfInfo.numOfColumn); + taosFprintfFile(pFile, "# batchNumOfRow: %d\n", g_stConfInfo.batchNumOfRow); + taosFprintfFile(pFile, "# totalRowsOfPerTbl: %d\n", g_stConfInfo.totalRowsOfPerTbl); + taosFprintfFile(pFile, "# totalRowsOfT2: %d\n", g_stConfInfo.totalRowsOfT2); + taosFprintfFile(pFile, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + taosFprintfFile(pFile, "###################################################################\n"); + taosFprintfFile(pFile, + "|-------------------------------insert " + "info-----------------------------|--------------------------------consume " + "info---------------------------------|\n"); + taosFprintfFile(pFile, + "|batch size| insert msgs | insert time(s) | msgs/s | walLogSize(MB) | consume msgs | consume " + "time(s) | msgs/s | MB/s | avg msg size(KB) |\n"); + taosFprintfFile(g_fp, "|%10d", g_stConfInfo.batchNumOfRow); +} + +int main(int32_t argc, char* argv[]) { + parseArgument(argc, argv); + printParaIntoFile(); + + int64_t walLogSize = 0; + + int code; + code = init_env(); + if (code != 0) { + fprintf(stderr, "%% init_env error!\n"); + return -1; + } + + int32_t totalMsgs = 0; + + if (g_stConfInfo.runMode != TMQ_RUN_ONLY_CONSUME) { + int64_t startTs = taosGetTimestampUs(); + if (1 == g_stConfInfo.ratio) { + totalMsgs = syncWriteData(); + } else { + totalMsgs = syncWriteDataByRatio(); + } + + if (totalMsgs <= 0) { + pError("inset data error!\n"); + return -1; + } + int64_t endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + + int32_t totalRows = 0; + if (1 == g_stConfInfo.ratio) { + totalRows = g_stConfInfo.totalRowsOfPerTbl * g_stConfInfo.numOfTables; + } else { + totalRows = g_stConfInfo.totalRowsOfPerTbl * (1 + g_stConfInfo.ratio); + } + + float seconds = delay / 1000000.0; + float rowsSpeed = totalRows / seconds; + float msgsSpeed = totalMsgs / seconds; + + if ((0 == g_stConfInfo.simCase) && (strlen(g_stConfInfo.vnodeWalPath))) { + walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath); + if (walLogSize <= 0) { + printf("%s size incorrect!", g_stConfInfo.vnodeWalPath); + exit(-1); + } else { + pPrint(".log file size in vnode2/wal: %.3f MBytes\n", (double)walLogSize / (1024 * 1024.0)); + } + } + + if (0 == g_stConfInfo.simCase) { + pPrint("insert result: %d rows, %d msgs, time:%.3f sec, speed:%.1f rows/second, %.1f msgs/second\n", totalRows, + totalMsgs, seconds, rowsSpeed, msgsSpeed); + } + taosFprintfFile(g_fp, "|%10d | %10.3f | %8.2f | %10.3f ", totalMsgs, seconds, msgsSpeed, + (double)walLogSize / (1024 * 1024.0)); + } + + if (g_stConfInfo.runMode == TMQ_RUN_ONLY_INSERT) { + return 0; + } + + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + if ((NULL == tmq) || (NULL == topic_list)) { + return -1; + } + + perf_loop(tmq, topic_list, totalMsgs, walLogSize); + + taosMemoryFreeClear(g_pRowValue); + taosFprintfFile(g_fp, "\n"); + taosCloseFile(&g_fp); + return 0; +} + diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c new file mode 100644 index 0000000000000000000000000000000000000000..71b31ba1071c977d9fd3d2ceb046bdff02ca53df --- /dev/null +++ b/utils/test/c/tmqSim.c @@ -0,0 +1,1520 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "taos.h" +#include "taosdef.h" +#include "taoserror.h" +#include "tlog.h" +#include "types.h" + +#define GREEN "\033[1;32m" +#define NC "\033[0m" +#define min(a, b) (((a) < (b)) ? (a) : (b)) + +#define MAX_SQL_STR_LEN (1024 * 1024) +#define MAX_ROW_STR_LEN (16 * 1024) +#define MAX_CONSUMER_THREAD_CNT (16) +#define MAX_VGROUP_CNT (32) +#define SEND_TIME_UNIT 10 // ms +#define MAX_SQL_LEN 1048576 + +typedef enum { + NOTIFY_CMD_START_CONSUM, + NOTIFY_CMD_START_COMMIT, + NOTIFY_CMD_ID_BUTT, +} NOTIFY_CMD_ID; + +typedef enum enumQUERY_TYPE { + NO_INSERT_TYPE, + INSERT_TYPE, + QUERY_TYPE_BUT +} QUERY_TYPE; + +typedef struct { + TdThread thread; + int32_t consumerId; + + int32_t ifManualCommit; + // int32_t autoCommitIntervalMs; // 1000 ms + // char autoCommit[8]; // true, false + // char autoOffsetRest[16]; // none, earliest, latest + + TdFilePtr pConsumeRowsFile; + TdFilePtr pConsumeMetaFile; + int32_t ifCheckData; + int64_t expectMsgCnt; + + int64_t consumeMsgCnt; + int64_t consumeRowCnt; + int64_t consumeLen; + int32_t checkresult; + + char topicString[1024]; + char keyString[1024]; + + int32_t numOfTopic; + char topics[32][64]; + + int32_t numOfKey; + char key[32][64]; + char value[32][64]; + + tmq_t* tmq; + tmq_list_t* topicList; + + int32_t numOfVgroups; + int32_t rowsOfPerVgroups[MAX_VGROUP_CNT][2]; // [i][0]: vgroup id, [i][1]: rows of consume + int64_t ts; + + TAOS* taos; + + // below parameters is used by omb test + int32_t producerRate; // unit: msgs/s + int64_t totalProduceMsgs; + int64_t totalMsgsLen; + +} SThreadInfo; + +typedef struct { + // input from argvs + char cdbName[32]; + char dbName[64]; + int32_t showMsgFlag; + int32_t showRowFlag; + int32_t saveRowFlag; + int32_t consumeDelay; // unit s + int32_t numOfThread; + int32_t useSnapshot; + int64_t nowTime; + SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; + + SThreadInfo stProdThreads[MAX_CONSUMER_THREAD_CNT]; + + // below parameters is used by omb test + char topic[64]; + int32_t producers; + int32_t producerRate; + int32_t runDurationMinutes; + int32_t batchSize; + int32_t payloadLen; +} SConfInfo; + +static SConfInfo g_stConfInfo; +TdFilePtr g_fp = NULL; +static int running = 1; +char* g_payload = NULL; + +// char* g_pRowValue = NULL; +// TdFilePtr g_fp = NULL; + +static void printHelp() { + char indent[10] = " "; + printf("Used to test the tmq feature with sim cases\n"); + + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-d"); + printf("%s%s%s\n", indent, indent, "The name of the database for cosumer, no default "); + printf("%s%s\n", indent, "-g"); + printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "showRowFlag, default is ", g_stConfInfo.showRowFlag); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%d\n", indent, indent, "saveRowFlag, default is ", g_stConfInfo.saveRowFlag); + printf("%s%s\n", indent, "-y"); + printf("%s%s%s%ds\n", indent, indent, "consume delay, default is ", g_stConfInfo.consumeDelay); + printf("%s%s\n", indent, "-e"); + printf("%s%s%s%d\n", indent, indent, "snapshot, default is ", g_stConfInfo.useSnapshot); + + printf("%s%s\n", indent, "-t"); + printf("%s%s%s\n", indent, indent, "topic name, default is null"); + + printf("%s%s\n", indent, "-x"); + printf("%s%s%s\n", indent, indent, "consume thread number, default is 1"); + + + printf("%s%s\n", indent, "-l"); + printf("%s%s%s\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); + printf("%s%s\n", indent, "-p"); + printf("%s%s%s\n", indent, indent, "producer thread number, default is 0"); + printf("%s%s\n", indent, "-b"); + printf("%s%s%s\n", indent, indent, "batch size, default is 1"); + printf("%s%s\n", indent, "-i"); + printf("%s%s%s\n", indent, indent, "produce rate unit is msgs /s, default is 100000"); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s\n", indent, indent, "payload len unit is byte, default is 1000"); + + + exit(EXIT_SUCCESS); +} + +char* getCurrentTimeString(char* timeString) { + time_t tTime = taosGetTimestampSec(); + struct tm tm; + taosLocalTime(&tTime, &tm); + sprintf(timeString, "%d-%02d-%02d %02d:%02d:%02d", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, + tm.tm_min, tm.tm_sec); + + return timeString; +} + +static void tmqStop(int signum, void* info, void* ctx) { + running = 0; + char tmpString[128]; + taosFprintfFile(g_fp, "%s tmqStop() receive stop signal[%d]\n", getCurrentTimeString(tmpString), signum); +} + +static void tmqSetSignalHandle() { taosSetSignal(SIGINT, tmqStop); } + +void initLogFile() { + char filename[256]; + char tmpString[128]; + + pid_t process_id = getpid(); + + if (0 != strlen(g_stConfInfo.topic)) { + sprintf(filename, "/tmp/tmqlog-%d-%s.txt", process_id, getCurrentTimeString(tmpString)); + } else { + sprintf(filename, "%s/../log/tmqlog-%d-%s.txt", configDir, process_id, getCurrentTimeString(tmpString)); + } +#ifdef WINDOWS + for (int i = 2; i < sizeof(filename); i++) { + if (filename[i] == ':') filename[i] = '-'; + if (filename[i] == '\0') break; + } +#endif + TdFilePtr pFile = taosOpenFile(filename, TD_FILE_TEXT | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + if (NULL == pFile) { + fprintf(stderr, "Failed to open %s for save result\n", filename); + exit(-1); + } + g_fp = pFile; +} + +void saveConfigToLogFile() { + taosFprintfFile(g_fp, "###################################################################\n"); + taosFprintfFile(g_fp, "# configDir: %s\n", configDir); + taosFprintfFile(g_fp, "# dbName: %s\n", g_stConfInfo.dbName); + taosFprintfFile(g_fp, "# cdbName: %s\n", g_stConfInfo.cdbName); + taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); + taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); + taosFprintfFile(g_fp, "# saveRowFlag: %d\n", g_stConfInfo.saveRowFlag); + taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); + taosFprintfFile(g_fp, "# numOfThread: %d\n", g_stConfInfo.numOfThread); + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); + // taosFprintfFile(g_fp, " auto commit: %s\n", g_stConfInfo.stThreads[i].autoCommit); + // taosFprintfFile(g_fp, " auto commit interval ms: %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs); + // taosFprintfFile(g_fp, " auto offset rest: %s\n", g_stConfInfo.stThreads[i].autoOffsetRest); + taosFprintfFile(g_fp, " Topics: "); + for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) { + taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]); + } + taosFprintfFile(g_fp, "\n"); + taosFprintfFile(g_fp, " Key: "); + for (int k = 0; k < g_stConfInfo.stThreads[i].numOfKey; k++) { + taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); + } + taosFprintfFile(g_fp, "\n"); + taosFprintfFile(g_fp, " expect rows: %d\n", g_stConfInfo.stThreads[i].expectMsgCnt); + } + + char tmpString[128]; + taosFprintfFile(g_fp, "# Test time: %s\n", getCurrentTimeString(tmpString)); + taosFprintfFile(g_fp, "###################################################################\n"); +} + +void parseArgument(int32_t argc, char* argv[]) { + memset(&g_stConfInfo, 0, sizeof(SConfInfo)); + g_stConfInfo.showMsgFlag = 0; + g_stConfInfo.showRowFlag = 0; + g_stConfInfo.saveRowFlag = 0; + g_stConfInfo.consumeDelay = 5; + g_stConfInfo.numOfThread = 1; + g_stConfInfo.batchSize = 1; + g_stConfInfo.producers = 0; + + g_stConfInfo.nowTime = taosGetTimestampMs(); + + for (int32_t i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(g_stConfInfo.dbName, argv[++i]); + } else if (strcmp(argv[i], "-w") == 0) { + strcpy(g_stConfInfo.cdbName, argv[++i]); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-g") == 0) { + g_stConfInfo.showMsgFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + g_stConfInfo.showRowFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + g_stConfInfo.saveRowFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-y") == 0) { + g_stConfInfo.consumeDelay = atol(argv[++i]); + } else if (strcmp(argv[i], "-e") == 0) { + g_stConfInfo.useSnapshot = atol(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + char tmpBuf[56]; + strcpy(tmpBuf, argv[++i]); + sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); + } else if (strcmp(argv[i], "-x") == 0) { + g_stConfInfo.numOfThread = atol(argv[++i]); + } else if (strcmp(argv[i], "-l") == 0) { + g_stConfInfo.runDurationMinutes = atol(argv[++i]); + } else if (strcmp(argv[i], "-p") == 0) { + g_stConfInfo.producers = atol(argv[++i]); + } else if (strcmp(argv[i], "-b") == 0) { + g_stConfInfo.batchSize = atol(argv[++i]); + } else if (strcmp(argv[i], "-i") == 0) { + g_stConfInfo.producerRate = atol(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + g_stConfInfo.payloadLen = atol(argv[++i]); + } else { + pError("%s unknow para: %s %s", GREEN, argv[++i], NC); + exit(-1); + } + } + + g_payload = taosMemoryCalloc(g_stConfInfo.payloadLen + 1, 1); + if (NULL == g_payload) { + pPrint("%s failed to malloc for payload %s", GREEN, NC); + exit(-1); + } + + for (int32_t i = 0; i < g_stConfInfo.payloadLen; i++) { + strcpy(&g_payload[i], "a"); + } + + initLogFile(); + + taosFprintfFile(g_fp, "====parseArgument() success\n"); + +#if 1 + pPrint("%s configDir:%s %s", GREEN, configDir, NC); + pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC); + pPrint("%s cdbName:%s %s", GREEN, g_stConfInfo.cdbName, NC); + pPrint("%s consumeDelay:%d %s", GREEN, g_stConfInfo.consumeDelay, NC); + pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); + pPrint("%s showRowFlag:%d %s", GREEN, g_stConfInfo.showRowFlag, NC); + pPrint("%s saveRowFlag:%d %s", GREEN, g_stConfInfo.saveRowFlag, NC); + + pPrint("%s snapshot:%d %s", GREEN, g_stConfInfo.useSnapshot, NC); + + pPrint("%s omb topic:%s %s", GREEN, g_stConfInfo.topic, NC); + pPrint("%s numOfThread:%d %s", GREEN, g_stConfInfo.numOfThread, NC); +#endif +} + +void splitStr(char** arr, char* str, const char* del) { + char* s = strtok(str, del); + while (s != NULL) { + *arr++ = s; + s = strtok(NULL, del); + } +} + +void ltrim(char* str) { + if (str == NULL || *str == '\0') { + return; + } + int len = 0; + char* p = str; + while (*p != '\0' && isspace(*p)) { + ++p; + ++len; + } + memmove(str, p, strlen(str) - len + 1); + // return str; +} + +int queryDB(TAOS* taos, char* command) { + int retryCnt = 10; + int code; + TAOS_RES* pRes; + + while (retryCnt--) { + pRes = taos_query(taos, command); + code = taos_errno(pRes); + if (code != 0) { + taosSsleep(1); + taos_free_result(pRes); + continue; + } + taos_free_result(pRes); + return 0; + } + + pError("failed to reason:%s, sql: %s", tstrerror(code), command); + taos_free_result(pRes); + return -1; +} + + +void addRowsToVgroupId(SThreadInfo* pInfo, int32_t vgroupId, int32_t rows) { + int32_t i; + for (i = 0; i < pInfo->numOfVgroups; i++) { + if (vgroupId == pInfo->rowsOfPerVgroups[i][0]) { + pInfo->rowsOfPerVgroups[i][1] += rows; + return; + } + } + + pInfo->rowsOfPerVgroups[pInfo->numOfVgroups][0] = vgroupId; + pInfo->rowsOfPerVgroups[pInfo->numOfVgroups][1] += rows; + pInfo->numOfVgroups++; + + taosFprintfFile(g_fp, "consume id %d, add one new vogroup id: %d\n", pInfo->consumerId, vgroupId); + if (pInfo->numOfVgroups > MAX_VGROUP_CNT) { + taosFprintfFile(g_fp, "====consume id %d, vgroup num %d over than 32. new vgroupId: %d\n", pInfo->consumerId, + pInfo->numOfVgroups, vgroupId); + taosCloseFile(&g_fp); + exit(-1); + } +} + +TAOS* createNewTaosConnect() { + TAOS* taos = NULL; + int32_t retryCnt = 10; + + while (retryCnt--) { + TAOS* taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (NULL != taos) { + return taos; + } + taosSsleep(1); + } + + taosFprintfFile(g_fp, "taos_connect() fail\n"); + return NULL; +} + + +int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { + char sqlStr[1100] = {0}; + + if (strlen(buf) > 1024) { + taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", strlen(buf)); + taosCloseFile(&g_fp); + return -1; + } + + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (pConn == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not save consume result to main script\n"); + return -1; + } + + sprintf(sqlStr, "insert into %s.content_%d values (%" PRId64 ", \'%s\')", g_stConfInfo.cdbName, pInfo->consumerId, + pInfo->ts++, buf); + int retCode = queryDB(pConn, sqlStr); + if (retCode != 0) { + taosFprintfFile(g_fp, "error in save consume content\n"); + taosCloseFile(&g_fp); + taos_close(pConn); + exit(-1); + } + + taos_close(pConn); + + return 0; +} + +static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) { + // if (shell.args.is_raw_time) { + // sprintf(buf, "%" PRId64, val); + // return buf; + // } + + time_t tt; + int32_t ms = 0; + if (precision == TSDB_TIME_PRECISION_NANO) { + tt = (time_t)(val / 1000000000); + ms = val % 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + ms = val % 1000000; + } else { + tt = (time_t)(val / 1000); + ms = val % 1000; + } + + if (tt <= 0 && ms < 0) { + tt--; + if (precision == TSDB_TIME_PRECISION_NANO) { + ms += 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + ms += 1000000; + } else { + ms += 1000; + } + } + + struct tm ptm; + taosLocalTime(&tt, &ptm); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); + + if (precision == TSDB_TIME_PRECISION_NANO) { + sprintf(buf + pos, ".%09d", ms); + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", ms); + } else { + sprintf(buf + pos, ".%03d", ms); + } + + return buf; +} + +static void shellDumpFieldToFile(TdFilePtr pFile, const char* val, TAOS_FIELD* field, int32_t length, + int32_t precision) { + if (val == NULL) { + return; + } + + int n; + char buf[TSDB_MAX_BYTES_PER_ROW]; + switch (field->type) { + case TSDB_DATA_TYPE_BOOL: + taosFprintfFile(pFile, "%d", ((((int32_t)(*((char*)val))) == 1) ? 1 : 0)); + break; + case TSDB_DATA_TYPE_TINYINT: + taosFprintfFile(pFile, "%d", *((int8_t*)val)); + break; + case TSDB_DATA_TYPE_UTINYINT: + taosFprintfFile(pFile, "%u", *((uint8_t*)val)); + break; + case TSDB_DATA_TYPE_SMALLINT: + taosFprintfFile(pFile, "%d", *((int16_t*)val)); + break; + case TSDB_DATA_TYPE_USMALLINT: + taosFprintfFile(pFile, "%u", *((uint16_t*)val)); + break; + case TSDB_DATA_TYPE_INT: + taosFprintfFile(pFile, "%d", *((int32_t*)val)); + break; + case TSDB_DATA_TYPE_UINT: + taosFprintfFile(pFile, "%u", *((uint32_t*)val)); + break; + case TSDB_DATA_TYPE_BIGINT: + taosFprintfFile(pFile, "%" PRId64, *((int64_t*)val)); + break; + case TSDB_DATA_TYPE_UBIGINT: + taosFprintfFile(pFile, "%" PRIu64, *((uint64_t*)val)); + break; + case TSDB_DATA_TYPE_FLOAT: + taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); + break; + case TSDB_DATA_TYPE_DOUBLE: + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val)); + if (n > TMAX(25, length)) { + taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val)); + } else { + taosFprintfFile(pFile, "%s", buf); + } + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_JSON: + { + char quotationStr[2]; + int32_t bufIndex = 0; + quotationStr[0] = 0; + quotationStr[1] = 0; + for (int32_t i = 0; i < length; i++) { + buf[bufIndex] = val[i]; + bufIndex++; + if (val[i] == '\"') { + buf[bufIndex] = val[i]; + bufIndex++; + quotationStr[0] = '\"'; + } + if (val[i] == ',') { + quotationStr[0] = '\"'; + } + } + buf[bufIndex] = 0; + if (length == 0) { + quotationStr[0] = '\"'; + } + + taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr); + } + break; + case TSDB_DATA_TYPE_TIMESTAMP: + shellFormatTimestamp(buf, *(int64_t*)val, precision); + taosFprintfFile(pFile, "%s", buf); + break; + default: + break; + } +} + +static void dumpToFileForCheck(TdFilePtr pFile, TAOS_ROW row, TAOS_FIELD* fields, int32_t* length, int32_t num_fields, + int32_t precision) { + for (int32_t i = 0; i < num_fields; i++) { + if (i > 0) { + taosFprintfFile(pFile, ","); + } + shellDumpFieldToFile(pFile, (const char*)row[i], fields + i, length[i], precision); + } + taosFprintfFile(pFile, "\n"); +} + +static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) { + char buf[1024]; + int32_t totalRows = 0; + + // printf("topic: %s\n", tmq_get_topic_name(msg)); + int32_t vgroupId = tmq_get_vgroup_id(msg); + const char* dbName = tmq_get_db_name(msg); + + taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", + tmq_get_topic_name(msg), vgroupId); + + while (1) { + TAOS_ROW row = taos_fetch_row(msg); + + if (row == NULL) break; + + TAOS_FIELD* fields = taos_fetch_fields(msg); + int32_t numOfFields = taos_field_count(msg); + int32_t* length = taos_fetch_lengths(msg); + int32_t precision = taos_result_precision(msg); + const char* tbName = tmq_get_table_name(msg); + +#if 0 + // get schema + //============================== stub =================================================// + for (int32_t i = 0; i < numOfFields; i++) { + taosFprintfFile(g_fp, "%02d: name: %s, type: %d, len: %d\n", i, fields[i].name, fields[i].type, fields[i].bytes); + } + //============================== stub =================================================// +#endif + + dumpToFileForCheck(pInfo->pConsumeRowsFile, row, fields, length, numOfFields, precision); + + taos_print_row(buf, row, fields, numOfFields); + + if (0 != g_stConfInfo.showRowFlag) { + taosFprintfFile(g_fp, "tbname:%s, rows[%d]: %s\n", (tbName != NULL ? tbName : "null table"), totalRows, buf); + // if (0 != g_stConfInfo.saveRowFlag) { + // saveConsumeContentToTbl(pInfo, buf); + // } + } + + totalRows++; + } + + addRowsToVgroupId(pInfo, vgroupId, totalRows); + + return totalRows; +} + + +static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) { + char buf[1024]; + int32_t totalRows = 0; + + // printf("topic: %s\n", tmq_get_topic_name(msg)); + int32_t vgroupId = tmq_get_vgroup_id(msg); + const char* dbName = tmq_get_db_name(msg); + + taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", + tmq_get_topic_name(msg), vgroupId); + + { + tmq_raw_data raw = {0}; + int32_t code = tmq_get_raw(msg, &raw); + + if(code == TSDB_CODE_SUCCESS){ + int retCode = queryDB(pInfo->taos, "use metadb"); + if (retCode != 0) { + taosFprintfFile(g_fp, "error when use metadb\n"); + taosCloseFile(&g_fp); + exit(-1); + } + taosFprintfFile(g_fp, "raw:%p\n", &raw); + + tmq_write_raw(pInfo->taos, raw); + } + + char* result = tmq_get_json_meta(msg); + if(result){ + //printf("meta result: %s\n", result); + taosFprintfFile(pInfo->pConsumeMetaFile, "%s\n", result); + taosMemoryFree(result); + } + } + + totalRows++; + + return totalRows; +} + +static void appNothing(void* param, TAOS_RES* res, int32_t numOfRows) {} + +int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { + char sqlStr[1024] = {0}; + + // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int + sprintf(sqlStr, "insert into %s.notifyinfo values (%" PRId64 ", %d, %d)", g_stConfInfo.cdbName, atomic_fetch_add_64(&g_stConfInfo.nowTime, 1), cmdId, + pInfo->consumerId); + + taos_query_a(pInfo->taos, sqlStr, appNothing, NULL); + + taosFprintfFile(g_fp, "notifyMainScript success, sql: %s\n", sqlStr); + + return 0; +} + +static int32_t g_once_commit_flag = 0; +static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + taosFprintfFile(g_fp, "tmq_commit_cb_print() commit %d\n", code); + + if (0 == g_once_commit_flag) { + g_once_commit_flag = 1; + notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT); + } + + char tmpString[128]; + taosFprintfFile(g_fp, "%s tmq_commit_cb_print() be called\n", getCurrentTimeString(tmpString)); +} + +void build_consumer(SThreadInfo* pInfo) { + tmq_conf_t* conf = tmq_conf_new(); + + // tmq_conf_set(conf, "td.connect.ip", "localhost"); + // tmq_conf_set(conf, "td.connect.port", "6030"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + + // tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, pInfo); + + // tmq_conf_set(conf, "group.id", "cgrp1"); + for (int32_t i = 0; i < pInfo->numOfKey; i++) { + tmq_conf_set(conf, pInfo->key[i], pInfo->value[i]); + } + + tmq_conf_set(conf, "msg.with.table.name", "true"); + + // tmq_conf_set(conf, "client.id", "c-001"); + + // tmq_conf_set(conf, "enable.auto.commit", "true"); + // tmq_conf_set(conf, "enable.auto.commit", "false"); + + // tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + + // tmq_conf_set(conf, "auto.offset.reset", "none"); + // tmq_conf_set(conf, "auto.offset.reset", "earliest"); + // tmq_conf_set(conf, "auto.offset.reset", "latest"); + // + if (g_stConfInfo.useSnapshot) { + tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + } + + pInfo->tmq = tmq_consumer_new(conf, NULL, 0); + + tmq_conf_destroy(conf); + + return; +} + +void build_topic_list(SThreadInfo* pInfo) { + pInfo->topicList = tmq_list_new(); + // tmq_list_append(topic_list, "test_stb_topic_1"); + for (int32_t i = 0; i < pInfo->numOfTopic; i++) { + tmq_list_append(pInfo->topicList, pInfo->topics[i]); + } + return; +} + +int32_t saveConsumeResult(SThreadInfo* pInfo) { + char sqlStr[1024] = {0}; + // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int + sprintf(sqlStr, "insert into %s.consumeresult values (%" PRId64 ", %d, %" PRId64 ", %" PRId64 ", %d)", + g_stConfInfo.cdbName, atomic_fetch_add_64(&g_stConfInfo.nowTime, 1), pInfo->consumerId, pInfo->consumeMsgCnt, + pInfo->consumeRowCnt, pInfo->checkresult); + + char tmpString[128]; + taosFprintfFile(g_fp, "%s, consume id %d result: %s\n", getCurrentTimeString(tmpString), pInfo->consumerId, sqlStr); + + int retCode = queryDB(pInfo->taos, sqlStr); + if (retCode != 0) { + taosFprintfFile(g_fp, "consume id %d error in save consume result\n", pInfo->consumerId); + return -1; + } + + return 0; +} + +void loop_consume(SThreadInfo* pInfo) { + int32_t code; + + int32_t once_flag = 0; + + int64_t totalMsgs = 0; + int64_t totalRows = 0; + + char tmpString[128]; + taosFprintfFile(g_fp, "%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), + pInfo->consumerId); + + pInfo->ts = taosGetTimestampMs(); + + if (pInfo->ifCheckData) { + char filename[256] = {0}; + char tmpString[128]; + // sprintf(filename, "%s/../log/consumerid_%d_%s.txt", configDir, pInfo->consumerId, + // getCurrentTimeString(tmpString)); + sprintf(filename, "%s/../log/consumerid_%d.txt", configDir, pInfo->consumerId); + pInfo->pConsumeRowsFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + + sprintf(filename, "%s/../log/meta_consumerid_%d.txt", configDir, pInfo->consumerId); + pInfo->pConsumeMetaFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + + if (pInfo->pConsumeRowsFile == NULL || pInfo->pConsumeMetaFile == NULL) { + taosFprintfFile(g_fp, "%s create file fail for save rows or save meta\n", getCurrentTimeString(tmpString)); + return; + } + } + + int64_t lastTotalMsgs = 0; + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + + int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000); + while (running) { + TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, consumeDelay); + if (tmqMsg) { + if (0 != g_stConfInfo.showMsgFlag) { + tmq_res_t msgType = tmq_get_res_type(tmqMsg); + if (msgType == TMQ_RES_TABLE_META) { + totalRows += meta_msg_process(tmqMsg, pInfo, totalMsgs); + } else if (msgType == TMQ_RES_DATA) + totalRows += data_msg_process(tmqMsg, pInfo, totalMsgs); + } + + taos_free_result(tmqMsg); + + totalMsgs++; + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 10 * 1000) { + taosFprintfFile( + g_fp, "consumer id %d has currently poll total msgs: %" PRId64 ", period rate: %.3f msgs/second\n", + pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0 / (currentPrintTime - lastPrintTime)); + lastPrintTime = currentPrintTime; + lastTotalMsgs = totalMsgs; + } + + if (0 == once_flag) { + once_flag = 1; + notifyMainScript(pInfo, NOTIFY_CMD_START_CONSUM); + } + + if ((totalRows >= pInfo->expectMsgCnt) || (totalMsgs >= pInfo->expectMsgCnt)) { + char tmpString[128]; + taosFprintfFile(g_fp, "%s over than expect rows, so break consume\n", getCurrentTimeString(tmpString)); + break; + } + } else { + char tmpString[128]; + taosFprintfFile(g_fp, "%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); + break; + } + } + + if (0 == running) { + taosFprintfFile(g_fp, "receive stop signal and not continue consume\n"); + } + + pInfo->consumeMsgCnt = totalMsgs; + pInfo->consumeRowCnt = totalRows; + + taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %" PRId64 ", consumeRowCnt: %" PRId64 "\n", + pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt); +} + +void* consumeThreadFunc(void* param) { + SThreadInfo* pInfo = (SThreadInfo*)param; + + pInfo->taos = createNewTaosConnect(); + if (pInfo->taos == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not notify and save consume result to main scripte\n"); + return NULL; + } + + build_consumer(pInfo); + build_topic_list(pInfo); + if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { + taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); + taos_close(pInfo->taos); + pInfo->taos = NULL; + return NULL; + } + + int32_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); + if (err != 0) { + pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err)); + taos_close(pInfo->taos); + pInfo->taos = NULL; + return NULL; + } + + tmq_list_destroy(pInfo->topicList); + pInfo->topicList = NULL; + + loop_consume(pInfo); + + if (pInfo->ifManualCommit) { + pPrint("tmq_commit() manual commit when consume end.\n"); + /*tmq_commit(pInfo->tmq, NULL, 0);*/ + tmq_commit_sync(pInfo->tmq, NULL); + taosFprintfFile(g_fp, "tmq_commit() manual commit over.\n"); + pPrint("tmq_commit() manual commit over.\n"); + } + + err = tmq_unsubscribe(pInfo->tmq); + if (err != 0) { + pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_unsubscribe()! reason: %s\n", tmq_err2str(err)); + } + + err = tmq_consumer_close(pInfo->tmq); + if (err != 0) { + pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_consumer_close()! reason: %s\n", tmq_err2str(err)); + } + pInfo->tmq = NULL; + + // save consume result into consumeresult table + saveConsumeResult(pInfo); + + // save rows from per vgroup + taosFprintfFile(g_fp, "======== consumerId: %d, consume rows from per vgroups ========\n", pInfo->consumerId); + for (int32_t i = 0; i < pInfo->numOfVgroups; i++) { + taosFprintfFile(g_fp, "vgroups: %04d, rows: %d\n", pInfo->rowsOfPerVgroups[i][0], pInfo->rowsOfPerVgroups[i][1]); + } + + taos_close(pInfo->taos); + pInfo->taos = NULL; + + return NULL; +} + +void parseConsumeInfo() { + char* token; + const char delim[2] = ","; + const char ch = ':'; + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + token = strtok(g_stConfInfo.stThreads[i].topicString, delim); + while (token != NULL) { + // printf("%s\n", token ); + strcpy(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic], token); + ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]); + // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); + g_stConfInfo.stThreads[i].numOfTopic++; + + token = strtok(NULL, delim); + } + + token = strtok(g_stConfInfo.stThreads[i].keyString, delim); + while (token != NULL) { + // printf("%s\n", token ); + { + char* pstr = token; + ltrim(pstr); + char* ret = strchr(pstr, ch); + memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr); + strcpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1); + // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], + // g_stConfInfo.value[g_stConfInfo.numOfKey]); + g_stConfInfo.stThreads[i].numOfKey++; + } + + token = strtok(NULL, delim); + } + } +} + +int32_t getConsumeInfo() { + char sqlStr[1024] = {0}; + + TAOS* pConn = createNewTaosConnect(); + if (pConn == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not get consume info for start consumer\n"); + return -1; + } + + sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName); + TAOS_RES *pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + taosFprintfFile(g_fp, "error in get consumeinfo for %s\n", taos_errstr(pRes)); + taosCloseFile(&g_fp); + taos_free_result(pRes); + taos_close(pConn); + return -1; + } + + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(pRes); + TAOS_FIELD* fields = taos_fetch_fields(pRes); + + // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, + // ifcheckdata int + + int32_t numOfThread = 0; + while ((row = taos_fetch_row(pRes))) { + int32_t* lengths = taos_fetch_lengths(pRes); + + // set default value + // g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000; + // memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true")); + // memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast")); + + for (int i = 0; i < num_fields; ++i) { + if (row[i] == NULL || 0 == i) { + continue; + } + + if ((1 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { + g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t*)row[i]); + } else if ((2 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { + memcpy(g_stConfInfo.stThreads[numOfThread].topicString, row[i], lengths[i]); + } else if ((3 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { + memcpy(g_stConfInfo.stThreads[numOfThread].keyString, row[i], lengths[i]); + } else if ((4 == i) && (fields[i].type == TSDB_DATA_TYPE_BIGINT)) { + g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]); + } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { + g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]); + } else if ((6 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { + g_stConfInfo.stThreads[numOfThread].ifManualCommit = *((int32_t*)row[i]); + } + } + numOfThread++; + } + g_stConfInfo.numOfThread = numOfThread; + + taos_free_result(pRes); + + parseConsumeInfo(); + taos_close(pConn); + + return 0; +} + + +static int32_t omb_data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex, int64_t* lenOfRows) { + char buf[16*1024]; + int32_t totalRows = 0; + int32_t totalLen = 0; + + // printf("topic: %s\n", tmq_get_topic_name(msg)); + //int32_t vgroupId = tmq_get_vgroup_id(msg); + //const char* dbName = tmq_get_db_name(msg); + + //taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + //taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", + // tmq_get_topic_name(msg), vgroupId); + + while (1) { + TAOS_ROW row = taos_fetch_row(msg); + + if (row == NULL) break; + + TAOS_FIELD* fields = taos_fetch_fields(msg); + int32_t numOfFields = taos_field_count(msg); + //int32_t* length = taos_fetch_lengths(msg); + //int32_t precision = taos_result_precision(msg); + //const char* tbName = tmq_get_table_name(msg); + + taos_print_row(buf, row, fields, numOfFields); + totalLen += strlen(buf); + totalRows++; + } + + *lenOfRows = totalLen; + return totalRows; +} + +void omb_loop_consume(SThreadInfo* pInfo) { + int32_t code; + + int32_t once_flag = 0; + + int64_t totalMsgs = 0; + int64_t totalRows = 0; + + char tmpString[128]; + taosFprintfFile(g_fp, "%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), + pInfo->consumerId); + printf("%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), + pInfo->consumerId); + + pInfo->ts = taosGetTimestampMs(); + + int64_t lastTotalMsgs = 0; + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + + int64_t totalLenOfMsg = 0; + int64_t lastTotalLenOfMsg = 0; + int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000); + while (running) { + TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, consumeDelay); + if (tmqMsg) { + int64_t lenOfMsg = 0; + totalRows += omb_data_msg_process(tmqMsg, pInfo, totalMsgs, &lenOfMsg); + totalLenOfMsg += lenOfMsg; + taos_free_result(tmqMsg); + totalMsgs++; + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 10 * 1000) { + int64_t currentLenOfMsg = totalLenOfMsg - lastTotalLenOfMsg; + int64_t deltaTime = currentPrintTime - lastPrintTime; + printf("consumer id %d has currently cons total rows: %" PRId64 ", msgs: %" PRId64 ", rate: %.3f msgs/s, %.1f MB/s\n", + pInfo->consumerId, totalRows, totalMsgs, + (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, + currentLenOfMsg*1000.0/(1024*1024)/deltaTime); + + taosFprintfFile( + g_fp, "consumer id %d has currently poll total msgs: %" PRId64 ", period cons rate: %.3f msgs/s, %.1f MB/s\n", + pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, currentLenOfMsg*1000.0/deltaTime); + lastPrintTime = currentPrintTime; + lastTotalMsgs = totalMsgs; + lastTotalLenOfMsg = totalLenOfMsg; + } + } else { + char tmpString[128]; + taosFprintfFile(g_fp, "%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); + printf("%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); + int64_t currentPrintTime = taosGetTimestampMs(); + int64_t currentLenOfMsg = totalLenOfMsg - lastTotalLenOfMsg; + int64_t deltaTime = currentPrintTime - lastPrintTime; + printf("consumer id %d has currently cons total rows: %" PRId64 ", msgs: %" PRId64 ", rate: %.3f msgs/s, %.1f MB/s\n", + pInfo->consumerId, totalRows, totalMsgs, + (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, + currentLenOfMsg*1000.0/(1024*1024)/deltaTime); + break; + } + } + + pInfo->consumeMsgCnt = totalMsgs; + pInfo->consumeRowCnt = totalRows; + pInfo->consumeLen = totalLenOfMsg; + +} + + +void* ombConsumeThreadFunc(void* param) { + SThreadInfo* pInfo = (SThreadInfo*)param; + + //################### set key ######################## + tmq_conf_t* conf = tmq_conf_new(); + // tmq_conf_set(conf, "td.connect.ip", "localhost"); + // tmq_conf_set(conf, "td.connect.port", "6030"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + // tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, pInfo); + tmq_conf_set(conf, "group.id", "ombCgrp"); + // tmq_conf_set(conf, "msg.with.table.name", "true"); + // tmq_conf_set(conf, "client.id", "c-001"); + // tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "enable.auto.commit", "false"); + // tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + // tmq_conf_set(conf, "auto.offset.reset", "none"); + // tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + // + if (g_stConfInfo.useSnapshot) { + tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + } + + pInfo->tmq = tmq_consumer_new(conf, NULL, 0); + + tmq_conf_destroy(conf); + + //################### set topic ########################## + pInfo->topicList = tmq_list_new(); + tmq_list_append(pInfo->topicList, g_stConfInfo.topic); + + if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { + taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); + return NULL; + } + + int32_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); + if (err != 0) { + pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err)); + return NULL; + } + + tmq_list_destroy(pInfo->topicList); + pInfo->topicList = NULL; + + omb_loop_consume(pInfo); + + err = tmq_unsubscribe(pInfo->tmq); + if (err != 0) { + pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_unsubscribe()! reason: %s\n", tmq_err2str(err)); + } + + err = tmq_consumer_close(pInfo->tmq); + if (err != 0) { + pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_consumer_close()! reason: %s\n", tmq_err2str(err)); + } + pInfo->tmq = NULL; + + return NULL; +} + + + +static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type) { + TAOS_RES *res = taos_query(taos, command); + int32_t code = taos_errno(res); + + if (code != 0) { + pPrint("%s Failed to execute <%s>, reason: %s %s", GREEN, command, taos_errstr(res), NC); + taos_free_result(res); + return -1; + } + + if (INSERT_TYPE == type) { + int affectedRows = taos_affected_rows(res); + taos_free_result(res); + return affectedRows; + } + + taos_free_result(res); + return 0; +} + +void* ombProduceThreadFunc(void* param) { + SThreadInfo* pInfo = (SThreadInfo*)param; + + pInfo->taos = createNewTaosConnect(); + if (pInfo->taos == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not start producers!\n"); + return NULL; + } + + int64_t affectedRowsTotal = 0; + int64_t sendMsgs = 0; + + uint32_t totalSendLoopTimes = g_stConfInfo.runDurationMinutes * 60 * 1000 / SEND_TIME_UNIT; // send some msgs per 10ms + uint32_t batchPerTblTimes = pInfo->producerRate / 100 / g_stConfInfo.batchSize; + uint32_t remainder = (pInfo->producerRate / 100) % g_stConfInfo.batchSize; + if (remainder) { + batchPerTblTimes += 1; + } + + char* sqlBuf = taosMemoryMalloc(MAX_SQL_LEN); + if (NULL == sqlBuf) { + printf("malloc fail for sqlBuf\n"); + taos_close(pInfo->taos); + pInfo->taos = NULL; + return NULL; + } + + printf("Produce Info: totalSendLoopTimes: %d, batchPerTblTimes: %d, producerRate: %d\n", totalSendLoopTimes, batchPerTblTimes, pInfo->producerRate); + + char ctbName[64] = {0}; + sprintf(ctbName, "%s.ctb%d", g_stConfInfo.dbName, pInfo->consumerId); + + int64_t lastPrintTime = taosGetTimestampUs(); + int64_t totalMsgLen = 0; + //int64_t timeStamp = taosGetTimestampUs(); + while (totalSendLoopTimes) { + int64_t startTs = taosGetTimestampUs(); + for (int i = 0; i < batchPerTblTimes; ++i) { + uint32_t msgsOfSql = g_stConfInfo.batchSize; + if ((i == batchPerTblTimes - 1) && (0 != remainder)) { + msgsOfSql = remainder; + } + int len = 0; + len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "insert into %s values ", ctbName); + for (int j = 0; j < msgsOfSql; j++) { + int64_t timeStamp = taosGetTimestampNs(); + len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); + sendMsgs++; + pInfo->totalProduceMsgs++; + } + + totalMsgLen += len; + pInfo->totalMsgsLen += len; + + int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); + if (affectedRows < 0) { + taos_close(pInfo->taos); + pInfo->taos = NULL; + return NULL; + } + + affectedRowsTotal += affectedRows; + + //printf("Produce Info: affectedRows: %" PRId64 "\n", affectedRows); + } + totalSendLoopTimes -= 1; + + // calc spent time + int64_t currentTs = taosGetTimestampUs(); + int64_t delta = currentTs - startTs; + if (delta < SEND_TIME_UNIT * 1000) { + int64_t sleepLen = (int32_t)(SEND_TIME_UNIT * 1000 - delta); + //printf("sleep %" PRId64 " us, use time: %" PRId64 " us\n", sleepLen, delta); + taosUsleep((int32_t)sleepLen); + } + + currentTs = taosGetTimestampUs(); + delta = currentTs - lastPrintTime; + if (delta > 10 * 1000 * 1000) { + printf("producer[%d] info: %" PRId64 " msgs, %" PRId64 " Byte, %" PRId64 " us, totalSendLoopTimes: %d\n", + pInfo->consumerId, sendMsgs, totalMsgLen, delta, totalSendLoopTimes); + printf("producer[%d] rate: %1.f msgs/s, %1.f KB/s\n", + pInfo->consumerId, + sendMsgs * 1000.0 * 1000 / delta, + (totalMsgLen / 1024.0) / (delta / (1000*1000))); + lastPrintTime = currentTs; + sendMsgs = 0; + totalMsgLen = 0; + } + } + + printf("affectedRowsTotal: %"PRId64"\n", affectedRowsTotal); + taos_close(pInfo->taos); + pInfo->taos = NULL; + return NULL; +} + + +void printProduceInfo(int64_t start) { + int64_t totalMsgs = 0; + int64_t totalLenOfMsgs = 0; + for (int i = 0; i < g_stConfInfo.producers; i++) { + totalMsgs += g_stConfInfo.stProdThreads[i].totalProduceMsgs; + totalLenOfMsgs += g_stConfInfo.stProdThreads[i].totalMsgsLen; + } + + int64_t end = taosGetTimestampUs(); + + int64_t t = end - start; + if (0 == t) t = 1; + + double tInMs = (double)t / 1000000.0; + printf("Spent %.3f seconds to prod %" PRIu64 " msgs, %" PRIu64 " Byte\n\n", tInMs, totalMsgs, totalLenOfMsgs); + + + printf("Spent %.3f seconds to prod %" PRIu64 " msgs with %d producer(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", + tInMs, totalMsgs, g_stConfInfo.producers, + (double)totalMsgs / tInMs, + (double)totalLenOfMsgs/(1024.0*1024)/tInMs); + return; +} + + +void startOmbConsume() { + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + + if (0 != g_stConfInfo.producers) { + TAOS* taos = createNewTaosConnect(); + if (taos == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not create db, stbl, ctbl, topic!\n"); + return ; + } + + char stbName[16] = "stb"; + char ctbPrefix[16] = "ctb"; + + char sql[256] = {0}; + sprintf(sql, "drop database if exists %s", g_stConfInfo.dbName); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + sprintf(sql, "create database if not exists %s precision 'ns' vgroups %d", g_stConfInfo.dbName, g_stConfInfo.producers); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + sprintf(sql, "create stable %s.%s (ts timestamp, payload binary(%d)) tags (t bigint) ", g_stConfInfo.dbName, stbName, g_stConfInfo.payloadLen); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + for (int i = 0; i < g_stConfInfo.producers; i++) { + sprintf(sql, "create table %s.%s%d using %s.stb tags(%d) ", g_stConfInfo.dbName, ctbPrefix, i, g_stConfInfo.dbName, i); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + } + + // create topic + sprintf(sql, "create topic %s as stable %s.%s", g_stConfInfo.topic, g_stConfInfo.dbName, stbName); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + + int32_t producerRate = ceil(g_stConfInfo.producerRate / g_stConfInfo.producers); + + printf("==== create %d produce thread ====\n", g_stConfInfo.producers); + for (int32_t i = 0; i < g_stConfInfo.producers; ++i) { + g_stConfInfo.stProdThreads[i].consumerId = i; + g_stConfInfo.stProdThreads[i].producerRate = producerRate; + taosThreadCreate(&(g_stConfInfo.stProdThreads[i].thread), &thattr, ombProduceThreadFunc, + (void*)(&(g_stConfInfo.stProdThreads[i]))); + } + + if (0 == g_stConfInfo.numOfThread) { + int64_t start = taosGetTimestampUs(); + for (int32_t i = 0; i < g_stConfInfo.producers; i++) { + taosThreadJoin(g_stConfInfo.stProdThreads[i].thread, NULL); + taosThreadClear(&g_stConfInfo.stProdThreads[i].thread); + } + + printProduceInfo(start); + + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); + taosCloseFile(&g_fp); + taos_close(taos); + return; + } + + taos_close(taos); + } + + // pthread_create one thread to consume + taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); + for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { + g_stConfInfo.stThreads[i].consumerId = i; + taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, ombConsumeThreadFunc, + (void*)(&(g_stConfInfo.stThreads[i]))); + } + + int64_t start = taosGetTimestampUs(); + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); + taosThreadClear(&g_stConfInfo.stThreads[i].thread); + } + + int64_t end = taosGetTimestampUs(); + + int64_t totalRows = 0; + int64_t totalMsgs = 0; + int64_t totalLenOfMsgs = 0; + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + totalMsgs += g_stConfInfo.stThreads[i].consumeMsgCnt; + totalLenOfMsgs += g_stConfInfo.stThreads[i].consumeLen; + totalRows += g_stConfInfo.stThreads[i].consumeRowCnt; + } + + int64_t t = end - start; + if (0 == t) t = 1; + + double tInMs = (double)t / 1000000.0; + taosFprintfFile(g_fp, + "Spent %.3f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", + tInMs, totalMsgs, g_stConfInfo.numOfThread, + (double)(totalMsgs / tInMs), + (double)totalLenOfMsgs/(1024*1024)/tInMs); + + printf("Spent %.3f seconds to cons rows: %" PRIu64 " msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", + tInMs, totalRows, totalMsgs, g_stConfInfo.numOfThread, + (double)(totalMsgs / tInMs), + (double)totalLenOfMsgs/(1024*1024)/tInMs); + + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); + taosCloseFile(&g_fp); + + return; +} + + +int main(int32_t argc, char* argv[]) { + parseArgument(argc, argv); + + if (0 != strlen(g_stConfInfo.topic)) { + startOmbConsume(); + return 0; + } + + int32_t retCode = getConsumeInfo(); + if (0 != retCode) { + return -1; + } + + saveConfigToLogFile(); + + tmqSetSignalHandle(); + + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + + // pthread_create one thread to consume + taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); + for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { + taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, + (void*)(&(g_stConfInfo.stThreads[i]))); + } + + int64_t start = taosGetTimestampUs(); + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); + taosThreadClear(&g_stConfInfo.stThreads[i].thread); + } + + int64_t end = taosGetTimestampUs(); + + int64_t totalMsgs = 0; + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + totalMsgs += g_stConfInfo.stThreads[i].consumeMsgCnt; + } + + int64_t t = end - start; + if (0 == t) t = 1; + + double tInMs = (double)t / 1000000.0; + taosFprintfFile(g_fp, + "Spent %.3f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/second\n\n", + tInMs, totalMsgs, g_stConfInfo.numOfThread, (double)(totalMsgs / tInMs)); + + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); + taosCloseFile(&g_fp); + + return 0; +} diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c new file mode 100644 index 0000000000000000000000000000000000000000..f917b9159e9914682c277329ddcfa4e269dc4908 --- /dev/null +++ b/utils/test/c/tmq_taosx_ci.c @@ -0,0 +1,623 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "taos.h" +#include "types.h" + +static int running = 1; +TdFilePtr g_fp = NULL; +typedef struct{ + bool snapShot; + bool dropTable; + bool subTable; + int srcVgroups; + int dstVgroups; + char dir[64]; +}Config; + +Config g_conf = {0}; + +static TAOS* use_db(){ + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return NULL; + } + + TAOS_RES* pRes = taos_query(pConn, "use db_taosx"); + if (taos_errno(pRes) != 0) { + printf("error in use db_taosx, reason:%s\n", taos_errstr(pRes)); + return NULL; + } + taos_free_result(pRes); + return pConn; +} + +static void msg_process(TAOS_RES* msg) { + printf("-----------topic-------------: %s\n", tmq_get_topic_name(msg)); + printf("db: %s\n", tmq_get_db_name(msg)); + printf("vg: %d\n", tmq_get_vgroup_id(msg)); + TAOS *pConn = use_db(); + if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { + char* result = tmq_get_json_meta(msg); + if (result) { + printf("meta result: %s\n", result); + } + if(g_fp){ + taosFprintfFile(g_fp, result); + taosFprintfFile(g_fp, "\n"); + } + + tmq_free_json_meta(result); + } + + tmq_raw_data raw = {0}; + tmq_get_raw(msg, &raw); + int32_t ret = tmq_write_raw(pConn, raw); + printf("write raw data: %s\n", tmq_err2str(ret)); + + taos_close(pConn); +} + +int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct0 values(1626006833400, 1, 2, 'a')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct1 using st1(t1) tags(2000)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct2 using st1(t1) tags(NULL)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct1 values(1626006833600, 3, 4, 'b')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct3 using st1(t1) tags(3000)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct3 values(1626006833600, 5, 6, 'c') ct1 values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, 'ddd') ct0 values(1626006833603, 4, 3, 'hwj') ct1 values(now+5s, 23, 32, 's21ds')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table st1 add column c4 bigint"); + if (taos_errno(pRes) != 0) { + printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table st1 modify column c3 binary(64)"); + if (taos_errno(pRes) != 0) { + printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct3 values(1626006833605, 53, 63, 'cffffffffffffffffffffffffffff', 8989898899999) (1626006833609, 51, 62, 'c333', 940)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct3 select * from ct1"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table st1 add tag t2 binary(64)"); + if (taos_errno(pRes) != 0) { + printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table ct3 set tag t1=5000"); + if (taos_errno(pRes) != 0) { + printf("failed to slter child table ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "delete from abc1 .ct3 where ts < 1626006833606"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + if(g_conf.dropTable){ + pRes = taos_query(pConn, "drop table ct3 ct1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop child table ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop table st1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + } + + pRes = taos_query(pConn, "create table if not exists n1(ts timestamp, c1 int, c2 nchar(4))"); + if (taos_errno(pRes) != 0) { + printf("failed to create normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 add column c3 bigint"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 modify column c2 nchar(8)"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 rename column c3 cc3"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 comment 'hello'"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 drop column c1"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into n1 values(now, 'eeee', 8989898899999) (now+9s, 'c333', 940)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + if(g_conf.dropTable){ + pRes = taos_query(pConn, "drop table n1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + } + + pRes = taos_query(pConn, "create table jt(ts timestamp, i int) tags(t json)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table jt1 using jt tags('{\"k1\":1, \"k2\":\"hello\"}')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table jt2 using jt tags('')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into jt1 values(now, 1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into jt2 values(now, 11)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + if(g_conf.dropTable){ + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop table st1"); + if (taos_errno(pRes) != 0) { + printf("failed to drop super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + } + return 0; +} + +int buildStable(TAOS* pConn, TAOS_RES* pRes){ + pRes = taos_query(pConn, "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` VARCHAR(16))"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table meters, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d0 using meters tags(1, 'San Francisco')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table d0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table d1 using meters tags(2, 'Beijing')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table d1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stream meters_summary_s into meters_summary as select _wstart, max(current) as current, groupid, location from meters partition by groupid, location interval(10m)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table meters_summary, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into d0 (ts, current) values (now, 120)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into table d0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + return 0; +} + +int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_taosx"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + char sql[128] = {0}; + snprintf(sql, 128, "create database if not exists db_taosx vgroups %d", g_conf.dstVgroups); + pRes = taos_query(pConn, sql); + if (taos_errno(pRes) != 0) { + printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop topic if exists topic_db"); + if (taos_errno(pRes) != 0) { + printf("error in drop topic, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop topic if exists meters_summary_t1"); + if (taos_errno(pRes) != 0) { + printf("error in drop topic, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists abc1"); + if (taos_errno(pRes) != 0) { + printf("error in drop db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + snprintf(sql, 128, "create database if not exists abc1 vgroups %d", g_conf.srcVgroups); + pRes = taos_query(pConn, sql); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + if(g_conf.subTable){ + buildStable(pConn, pRes); + }else{ + buildDatabase(pConn, pRes); + } + + taos_close(pConn); + return 0; +} + +int32_t create_topic() { + printf("create topic\n"); + TAOS_RES* pRes; + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + if(g_conf.subTable){ + pRes = taos_query(pConn, "create topic meters_summary_t1 with meta as stable meters_summary"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic meters_summary_t1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + }else{ + pRes = taos_query(pConn, "create topic topic_db with meta as database abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic topic_db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + } + + taos_close(pConn); + return 0; +} + +void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + printf("commit %d tmq %p param %p\n", code, tmq, param); +} + +tmq_t* build_consumer() { + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set(conf, "client.id", "my app 1"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "enable.heartbeat.background", "true"); + + if(g_conf.snapShot){ + tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + } + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); + tmq_conf_destroy(conf); + return tmq; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + if(g_conf.subTable){ + tmq_list_append(topic_list, "meters_summary_t1"); + }else{ + tmq_list_append(topic_list, "topic_db"); + } + return topic_list; +} + +void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { + int32_t code; + + if ((code = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); + printf("subscribe err\n"); + return; + } + int32_t cnt = 0; + while (running) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000); + if (tmqmessage) { + cnt++; + msg_process(tmqmessage); + taos_free_result(tmqmessage); + }else{ + break; + } + } + + code = tmq_consumer_close(tmq); + if (code) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +void initLogFile() { + char f1[256] = {0}; + char f2[256] = {0}; + + if(g_conf.snapShot){ + sprintf(f1, "%s/../log/tmq_taosx_tmp_snapshot.source", g_conf.dir); + sprintf(f2, "%s/../log/tmq_taosx_tmp_snapshot.result", g_conf.dir); + }else{ + sprintf(f1, "%s/../log/tmq_taosx_tmp.source", g_conf.dir); + sprintf(f2, "%s/../log/tmq_taosx_tmp.result", g_conf.dir); + } + + TdFilePtr pFile = taosOpenFile(f1, TD_FILE_TEXT | TD_FILE_TRUNC | TD_FILE_STREAM); + if (NULL == pFile) { + fprintf(stderr, "Failed to open %s for save result\n", f1); + exit(-1); + } + g_fp = pFile; + + TdFilePtr pFile2 = taosOpenFile(f2, TD_FILE_TEXT | TD_FILE_TRUNC | TD_FILE_STREAM); + if (NULL == pFile2) { + fprintf(stderr, "Failed to open %s for save result\n", f2); + exit(-1); + } + + if(g_conf.snapShot){ + char *result[] = { + "{\"type\":\"create\",\"tableName\":\"st1\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":64},{\"name\":\"c4\",\"type\":5}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1},{\"name\":\"t2\",\"type\":8,\"length\":64}]}", + "{\"type\":\"create\",\"tableName\":\"ct0\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}", + "{\"type\":\"create\",\"tableName\":\"ct1\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}]}", + "{\"type\":\"create\",\"tableName\":\"ct2\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[]}", + "{\"type\":\"create\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":5000}]}", + "{\"type\":\"create\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c2\",\"type\":10,\"length\":8},{\"name\":\"cc3\",\"type\":5}],\"tags\":[]}", + "{\"type\":\"create\",\"tableName\":\"jt\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", + "{\"type\":\"create\",\"tableName\":\"jt1\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}]}", + "{\"type\":\"create\",\"tableName\":\"jt2\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]}", + }; + + for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } + }else{ + char *result[] = { + "{\"type\":\"create\",\"tableName\":\"st1\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", + "{\"type\":\"create\",\"tableName\":\"ct0\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}", + "{\"type\":\"create\",\"tableName\":\"ct1\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}]}", + "{\"type\":\"create\",\"tableName\":\"ct2\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[]}", + "{\"type\":\"create\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":3000}]}", + "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}", + "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":7,\"colName\":\"c3\",\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":1,\"colName\":\"t2\",\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"alterType\":4,\"colName\":\"t1\",\"colValue\":\"5000\",\"colValueNull\":false}", + "{\"type\":\"create\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":5,\"colName\":\"c3\",\"colType\":5}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":7,\"colName\":\"c2\",\"colType\":10,\"colLength\":8}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":10,\"colName\":\"c3\",\"colNewName\":\"cc3\"}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":9}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":6,\"colName\":\"c1\"}", + "{\"type\":\"create\",\"tableName\":\"jt\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", + "{\"type\":\"create\",\"tableName\":\"jt1\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}]}", + "{\"type\":\"create\",\"tableName\":\"jt2\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]}" + }; + + for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } + } + + taosCloseFile(&pFile2); +} + +int main(int argc, char* argv[]) { + for (int32_t i = 1; i < argc; i++) { + if(strcmp(argv[i], "-c") == 0){ + strcpy(g_conf.dir, argv[++i]); + }else if(strcmp(argv[i], "-s") == 0){ + g_conf.snapShot = true; + }else if(strcmp(argv[i], "-d") == 0){ + g_conf.dropTable = true; + }else if(strcmp(argv[i], "-sv") == 0){ + g_conf.srcVgroups = atol(argv[++i]); + }else if(strcmp(argv[i], "-dv") == 0){ + g_conf.dstVgroups = atol(argv[++i]); + }else if(strcmp(argv[i], "-t") == 0){ + g_conf.subTable = true; + } + } + + printf("env init\n"); + if(strlen(g_conf.dir) != 0){ + initLogFile(); + } + + if (init_env() < 0) { + return -1; + } + create_topic(); + + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + basic_consume_loop(tmq, topic_list); + taosCloseFile(&g_fp); +} diff --git a/utils/tsim/CMakeLists.txt b/utils/tsim/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..c2cf7ac3c5380c4e116e96753fa78f486d918566 --- /dev/null +++ b/utils/tsim/CMakeLists.txt @@ -0,0 +1,14 @@ +aux_source_directory(src TSIM_SRC) +add_executable(tsim ${TSIM_SRC}) +target_link_libraries( + tsim + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os + PUBLIC cjson +) +target_include_directories( + tsim + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) diff --git a/utils/tsim/inc/simInt.h b/utils/tsim/inc/simInt.h new file mode 100644 index 0000000000000000000000000000000000000000..f9e96fc67b20eee8b05619475c3d577cfa5b659c --- /dev/null +++ b/utils/tsim/inc/simInt.h @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_SIM_INT_H_ +#define _TD_SIM_INT_H_ + +#include "os.h" + +#include "cJSON.h" +#include "tconfig.h" +#include "taos.h" +#include "taoserror.h" +#include "tidpool.h" +#include "tlog.h" +#include "ttimer.h" +#include "ttypes.h" +#include "tutil.h" +#include "tglobal.h" + +#define MAX_MAIN_SCRIPT_NUM 10 +#define MAX_BACKGROUND_SCRIPT_NUM 10 +#define MAX_FILE_NAME_LEN 256 +#define MAX_ERROR_LEN 1024 +#define MAX_QUERY_VALUE_LEN 1024 +#define MAX_QUERY_COL_NUM 100 +#define MAX_QUERY_ROW_NUM 100 +#define MAX_SYSTEM_RESULT_LEN 2048 +#define MAX_VAR_LEN 100 +#define MAX_VAR_NAME_LEN 32 +#define MAX_VAR_VAL_LEN 80 +#define MAX_OPT_NAME_LEN 32 +#define MAX_SIM_CMD_NAME_LEN 40 + +#ifdef LINUX +#define SUCCESS_PREFIX "\033[44;32;1m" +#define SUCCESS_POSTFIX "\033[0m" +#define FAILED_PREFIX "\033[44;31;1m" +#define FAILED_POSTFIX "\033[0m" +#else +#define SUCCESS_PREFIX "" +#define SUCCESS_POSTFIX "" +#define FAILED_PREFIX "" +#define FAILED_POSTFIX "" +#endif + +#define simFatal(...) { if (simDebugFlag & DEBUG_FATAL) { taosPrintLog("SIM FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define simError(...) { if (simDebugFlag & DEBUG_ERROR) { taosPrintLog("SIM ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define simWarn(...) { if (simDebugFlag & DEBUG_WARN) { taosPrintLog("SIM WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define simInfo(...) { if (simDebugFlag & DEBUG_INFO) { taosPrintLog("SIM ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define simDebug(...) { if (simDebugFlag & DEBUG_DEBUG) { taosPrintLog("SIM ", DEBUG_DEBUG, simDebugFlag, __VA_ARGS__); }} +#define simTrace(...) { if (simDebugFlag & DEBUG_TRACE) { taosPrintLog("SIM ", DEBUG_TRACE, simDebugFlag, __VA_ARGS__); }} + +enum { SIM_SCRIPT_TYPE_MAIN, SIM_SCRIPT_TYPE_BACKGROUND }; + +enum { + SIM_CMD_EXP, + SIM_CMD_IF, + SIM_CMD_ELIF, + SIM_CMD_ELSE, + SIM_CMD_ENDI, + SIM_CMD_WHILE, + SIM_CMD_ENDW, + SIM_CMD_SWITCH, + SIM_CMD_CASE, + SIM_CMD_DEFAULT, + SIM_CMD_CONTINUE, + SIM_CMD_BREAK, + SIM_CMD_ENDS, + SIM_CMD_SLEEP, + SIM_CMD_GOTO, + SIM_CMD_RUN, + SIM_CMD_RUN_BACK, + SIM_CMD_PRINT, + SIM_CMD_SYSTEM, + SIM_CMD_SYSTEM_CONTENT, + SIM_CMD_SQL, + SIM_CMD_SQL_ERROR, + SIM_CMD_SQL_SLOW, + SIM_CMD_RESTFUL, + SIM_CMD_TEST, + SIM_CMD_RETURN, + SIM_CMD_LINE_INSERT, + SIM_CMD_LINE_INSERT_ERROR, + SIM_CMD_END +}; + +enum { + SQL_JUMP_FALSE, + SQL_JUMP_TRUE, +}; + +struct _script_t; +typedef struct _cmd_t { + int16_t cmdno; + int16_t nlen; + char name[MAX_SIM_CMD_NAME_LEN]; + bool (*parseCmd)(char *, struct _cmd_t *, int32_t); + bool (*executeCmd)(struct _script_t *script, char *option); + struct _cmd_t *next; +} SCommand; + +typedef struct { + int16_t cmdno; + int16_t jump; // jump position + int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag + // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ + int16_t lineNum; // correspodning line number in original file + int32_t optionOffset; // relative option offset +} SCmdLine; + +typedef struct _var_t { + char varName[MAX_VAR_NAME_LEN]; + char varValue[MAX_VAR_VAL_LEN]; + char varNameLen; +} SVariable; + +typedef struct _script_t { + int32_t type; + bool killed; + void *taos; + char rows[12]; // number of rows data retrieved + char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results + char system_exit_code[12]; + char system_ret_content[MAX_SYSTEM_RESULT_LEN]; + int32_t varLen; + int32_t linePos; // current cmd position + int32_t numOfLines; // number of lines in the script + int32_t bgScriptLen; + char fileName[MAX_FILE_NAME_LEN]; // script file name + char error[MAX_ERROR_LEN]; + char *optionBuffer; + SCmdLine *lines; // command list + SVariable variables[MAX_VAR_LEN]; + TdThread bgPid; + char auth[128]; + struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM]; +} SScript; + +extern SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; +extern SCommand simCmdList[]; +extern int32_t simScriptPos; +extern int32_t simScriptSucced; +extern int32_t simDebugFlag; +extern char simScriptDir[]; +extern bool abortExecution; +extern bool useMultiProcess; +extern bool useValgrind; + +SScript *simParseScript(char *fileName); +SScript *simProcessCallOver(SScript *script); +void *simExecuteScript(void *script); +void simInitsimCmdList(); +bool simSystemInit(); +void simSystemCleanUp(); +char *simGetVariable(SScript *script, char *varName, int32_t varLen); +bool simExecuteExpCmd(SScript *script, char *option); +bool simExecuteTestCmd(SScript *script, char *option); +bool simExecuteGotoCmd(SScript *script, char *option); +bool simExecuteRunCmd(SScript *script, char *option); +bool simExecuteRunBackCmd(SScript *script, char *option); +bool simExecuteSystemCmd(SScript *script, char *option); +bool simExecuteSystemContentCmd(SScript *script, char *option); +bool simExecutePrintCmd(SScript *script, char *option); +bool simExecuteSleepCmd(SScript *script, char *option); +bool simExecuteReturnCmd(SScript *script, char *option); +bool simExecuteSqlCmd(SScript *script, char *option); +bool simExecuteSqlErrorCmd(SScript *script, char *rest); +bool simExecuteSqlSlowCmd(SScript *script, char *option); +bool simExecuteRestfulCmd(SScript *script, char *rest); +bool simExecuteLineInsertCmd(SScript *script, char *option); +bool simExecuteLineInsertErrorCmd(SScript *script, char *option); +void simVisuallizeOption(SScript *script, char *src, char *dst); + +#endif /*_TD_SIM_INT_H_*/ \ No newline at end of file diff --git a/utils/tsim/inc/simParse.h b/utils/tsim/inc/simParse.h new file mode 100644 index 0000000000000000000000000000000000000000..2a683f95613a486291e425c1aac5a80aebe74a75 --- /dev/null +++ b/utils/tsim/inc/simParse.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_SIM_PARSE_H_ +#define _TD_SIM_PARSE_H_ + +#include "simInt.h" + +#define MAX_NUM_CMD 64 +#define MAX_NUM_LABLES 100 +#define MAX_LABEL_LEN 40 +#define MAX_NUM_BLOCK 100 +#define MAX_NUM_JUMP 100 +#define MAX_LINE_LEN 3000 +#define MAX_CMD_LINES 2048 +#define MAX_OPTION_BUFFER 64000 + +enum { + BLOCK_IF, + BLOCK_WHILE, + BLOCK_SWITCH, +}; + +/* label stack */ +typedef struct { + int8_t top; /* number of labels */ + int16_t pos[MAX_NUM_LABLES]; /* the position of the label */ + char label[MAX_NUM_LABLES][MAX_LABEL_LEN]; /* name of the label */ +} SLabel; + +/* block definition */ +typedef struct { + int8_t top; /* the number of blocks stacked */ + char type[MAX_NUM_BLOCK]; /* the block type */ + int16_t *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ + int16_t back[MAX_NUM_BLOCK]; /* go back, endw and continue */ + char numJump[MAX_NUM_BLOCK]; + int16_t *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */ + char sexp[MAX_NUM_BLOCK][40]; /*switch expression */ + char sexpLen[MAX_NUM_BLOCK]; /*switch expression length */ +} SBlock; + +bool simParseExpression(char *token, int32_t lineNum); + +#endif /*_TD_SIM_PARSE_H_*/ \ No newline at end of file diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c new file mode 100644 index 0000000000000000000000000000000000000000..16732ff9a191576d871584253f2c70e9187b6d2f --- /dev/null +++ b/utils/tsim/src/simExe.c @@ -0,0 +1,1016 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "simInt.h" + +void simLogSql(char *sql, bool useSharp) { + static TdFilePtr pFile = NULL; + char filename[256]; + sprintf(filename, "%s/sim.sql", simScriptDir); + if (pFile == NULL) { + // fp = fopen(filename, "w"); + pFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + if (pFile == NULL) { + fprintf(stderr, "ERROR: failed to open file: %s\n", filename); + return; + } + } + if (useSharp) { + taosFprintfFile(pFile, "# %s;\n", sql); + } else { + taosFprintfFile(pFile, "%s;\n", sql); + } + + taosFsyncFile(pFile); +} + +char *simParseArbitratorName(char *varName) { + static char hostName[140]; +#ifdef WINDOWS + taosGetFqdn(hostName); + sprintf(&hostName[strlen(hostName)], ":%d", 8000); +#else + sprintf(hostName, "%s:%d", "localhost", 8000); +#endif + return hostName; +} + +char *simParseHostName(char *varName) { + static char hostName[140]; +#ifdef WINDOWS + hostName[0] = '\"'; + taosGetFqdn(&hostName[1]); + int strEndIndex = strlen(hostName); + hostName[strEndIndex] = '\"'; + hostName[strEndIndex + 1] = '\0'; +#else + sprintf(hostName, "%s", "localhost"); +#endif + return hostName; +} + +static void simFindFirstNum(const char *begin, int32_t beginLen, int32_t *num) { + if (beginLen <= 5) { + *num = 0; + } else { + *num = atoi(begin + 5); + } +} + +static void simFindSecondNum(const char *begin, int32_t beginLen, int32_t *num) { + const char *number = strstr(begin, "]["); + if (number == NULL) { + *num = 0; + } else { + *num = atoi(number + 2); + } +} + +static void simFindFirstKeyVal(const char *begin, int32_t beginLen, char *key, int32_t keyLen) { + key[0] = 0; + for (int32_t i = 5; i < beginLen && i - 5 < keyLen; ++i) { + if (begin[i] != 0 && begin[i] != ']' && begin[i] != ')') { + key[i - 5] = begin[i]; + } + } +} + +static void simFindSecondKeyNum(const char *begin, int32_t beginLen, int32_t *num) { + const char *number = strstr(begin, ")["); + if (number == NULL) { + *num = 0; + } else { + *num = atoi(number + 2); + } +} + +char *simGetVariable(SScript *script, char *varName, int32_t varLen) { + if (strncmp(varName, "hostname", 8) == 0) { + return simParseHostName(varName); + } + + if (strncmp(varName, "arbitrator", 10) == 0) { + return simParseArbitratorName(varName); + } + + if (strncmp(varName, "error", varLen) == 0) return script->error; + + if (strncmp(varName, "rows", varLen) == 0) return script->rows; + + if (strncmp(varName, "system_exit", varLen) == 0) return script->system_exit_code; + + if (strncmp(varName, "system_content", varLen) == 0) return script->system_ret_content; + + if (strncmp(varName, "data", 4) == 0) { + if (varLen < 6) { + return "null"; + } + + int32_t row = 0; + int32_t col = 0; + char keyVal[1024] = {0}; + int32_t keyLen = 1024; + + if (varName[4] == '[') { + // $data[0][1] + simFindFirstNum(varName, varLen, &row); + simFindSecondNum(varName, varLen, &col); + if (row < 0 || row >= MAX_QUERY_ROW_NUM) { + return "null"; + } + if (col < 0 || col >= MAX_QUERY_COL_NUM) { + return "null"; + } + simDebug("script:%s, data[%d][%d]=%s", script->fileName, row, col, script->data[row][col]); + return script->data[row][col]; + } else if (varName[4] == '(') { + // $data(db)[0] + simFindFirstKeyVal(varName, varLen, keyVal, keyLen); + simFindSecondKeyNum(varName, varLen, &col); + for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) { + if (strncmp(keyVal, script->data[i][0], keyLen) == 0) { + simDebug("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); + return script->data[i][col]; + } + } + } else if (varName[5] == '_') { + // data2_db + int32_t col = varName[4] - '0'; + if (col < 0 || col >= MAX_QUERY_COL_NUM) { + return "null"; + } + + char *keyName; + int32_t keyLen; + paGetToken(varName + 6, &keyName, &keyLen); + + for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) { + if (strncmp(keyName, script->data[i][0], keyLen) == 0) { + simDebug("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); + return script->data[i][col]; + } + } + return "null"; + } else if (varName[6] == '_') { + // data21_db + int32_t col = (varName[4] - '0') * 10 + (varName[5] - '0'); + if (col < 0 || col >= MAX_QUERY_COL_NUM) { + return "null"; + } + + char *keyName; + int32_t keyLen; + paGetToken(varName + 7, &keyName, &keyLen); + + for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) { + if (strncmp(keyName, script->data[i][0], keyLen) == 0) { + simTrace("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); + return script->data[i][col]; + } + } + return "null"; + } else { + // $data00 + int32_t row = varName[4] - '0'; + int32_t col = varName[5] - '0'; + if (row < 0 || row >= MAX_QUERY_ROW_NUM) { + return "null"; + } + if (col < 0 || col >= MAX_QUERY_COL_NUM) { + return "null"; + } + + simDebug("script:%s, data[%d][%d]=%s", script->fileName, row, col, script->data[row][col]); + return script->data[row][col]; + } + + return "null"; + } + + for (int32_t i = 0; i < script->varLen; ++i) { + SVariable *var = &script->variables[i]; + if (var->varNameLen != varLen) { + continue; + } + if (strncmp(varName, var->varName, varLen) == 0) { + return var->varValue; + } + } + + if (script->varLen >= MAX_VAR_LEN) { + simError("script:%s, too many varialbes:%d", script->fileName, script->varLen); + exit(0); + } + + SVariable *var = &script->variables[script->varLen]; + script->varLen++; + strncpy(var->varName, varName, varLen); + var->varNameLen = varLen; + var->varValue[0] = 0; + return var->varValue; +} + +int32_t simExecuteExpression(SScript *script, char *exp) { + char *op1, *op2, *var1, *var2, *var3, *rest; + int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1; + char t0[1024], t1[1024], t2[1024], t3[2048]; + int32_t result; + + rest = paGetToken(exp, &var1, &var1Len); + rest = paGetToken(rest, &op1, &op1Len); + rest = paGetToken(rest, &var2, &var2Len); + rest = paGetToken(rest, &op2, &op2Len); + + if (var1[0] == '$') + strcpy(t0, simGetVariable(script, var1 + 1, var1Len - 1)); + else { + memcpy(t0, var1, var1Len); + t0[var1Len] = 0; + } + + if (var2[0] == '$') + strcpy(t1, simGetVariable(script, var2 + 1, var2Len - 1)); + else { + memcpy(t1, var2, var2Len); + t1[var2Len] = 0; + } + + if (op2Len != 0) { + rest = paGetToken(rest, &var3, &var3Len); + + if (var3[0] == '$') + strcpy(t2, simGetVariable(script, var3 + 1, var3Len - 1)); + else { + memcpy(t2, var3, var3Len); + t2[var3Len] = 0; + } + + if (op2[0] == '+') { + sprintf(t3, "%lld", atoll(t1) + atoll(t2)); + } else if (op2[0] == '-') { + sprintf(t3, "%lld", atoll(t1) - atoll(t2)); + } else if (op2[0] == '*') { + sprintf(t3, "%lld", atoll(t1) * atoll(t2)); + } else if (op2[0] == '/') { + sprintf(t3, "%lld", atoll(t1) / atoll(t2)); + } else if (op2[0] == '.') { + sprintf(t3, "%s%s", t1, t2); + } + } else { + strcpy(t3, t1); + } + + result = 0; + + if (op1Len == 1) { + if (op1[0] == '=') { + strcpy(simGetVariable(script, var1 + 1, var1Len - 1), t3); + } else if (op1[0] == '<') { + val0 = atoi(t0); + val1 = atoi(t3); + if (val0 >= val1) result = -1; + } else if (op1[0] == '>') { + val0 = atoi(t0); + val1 = atoi(t3); + if (val0 <= val1) result = -1; + } + } else { + if (op1[0] == '=' && op1[1] == '=') { + if (strcmp(t0, t3) != 0) result = -1; + } else if (op1[0] == '!' && op1[1] == '=') { + if (strcmp(t0, t3) == 0) result = -1; + } else if (op1[0] == '<' && op1[1] == '=') { + val0 = atoi(t0); + val1 = atoi(t3); + if (val0 > val1) result = -1; + } else if (op1[0] == '>' && op1[1] == '=') { + val0 = atoi(t0); + val1 = atoi(t3); + if (val0 < val1) result = -1; + } + } + + return result; +} + +bool simExecuteExpCmd(SScript *script, char *option) { + simExecuteExpression(script, option); + script->linePos++; + return true; +} + +bool simExecuteTestCmd(SScript *script, char *option) { + int32_t result; + result = simExecuteExpression(script, option); + + if (result >= 0) + script->linePos++; + else + script->linePos = script->lines[script->linePos].jump; + + return true; +} + +bool simExecuteGotoCmd(SScript *script, char *option) { + script->linePos = script->lines[script->linePos].jump; + return true; +} + +bool simExecuteRunCmd(SScript *script, char *option) { + char *fileName = option; + if (fileName == NULL || strlen(fileName) == 0) { + sprintf(script->error, "lineNum:%d. script file is null", script->lines[script->linePos].lineNum); + return false; + } + + SScript *newScript = simParseScript(option); + if (newScript == NULL) { + sprintf(script->error, "lineNum:%d. parse file:%s error", script->lines[script->linePos].lineNum, fileName); + return false; + } + + simInfo("script:%s, start to execute", newScript->fileName); + + newScript->type = SIM_SCRIPT_TYPE_MAIN; + simScriptPos++; + simScriptList[simScriptPos] = newScript; + + script->linePos++; + return true; +} + +bool simExecuteRunBackCmd(SScript *script, char *option) { + char *fileName = option; + if (fileName == NULL || strlen(fileName) == 0) { + sprintf(script->error, "lineNum:%d. script file is null", script->lines[script->linePos].lineNum); + return false; + } + + SScript *newScript = simParseScript(option); + if (newScript == NULL) { + sprintf(script->error, "lineNum:%d. parse file:%s error", script->lines[script->linePos].lineNum, fileName); + return false; + } + + newScript->type = SIM_SCRIPT_TYPE_BACKGROUND; + script->bgScripts[script->bgScriptLen++] = newScript; + simInfo("script:%s, start to execute in background,", newScript->fileName); + + if (taosThreadCreate(&newScript->bgPid, NULL, simExecuteScript, (void *)newScript) != 0) { + sprintf(script->error, "lineNum:%d. create background thread failed", script->lines[script->linePos].lineNum); + return false; + } else { + simDebug("script:%s, background thread:0x%08" PRIx64 " is created", newScript->fileName, + taosGetPthreadId(newScript->bgPid)); + } + + script->linePos++; + return true; +} + +bool simReplaceStr(char *buf, char *src, char *dst) { + bool replaced = false; + char *begin = strstr(buf, src); + if (begin != NULL) { + int32_t srcLen = (int32_t)strlen(src); + int32_t dstLen = (int32_t)strlen(dst); + int32_t interval = (dstLen - srcLen); + int32_t remainLen = (int32_t)strlen(buf); + char *end = buf + remainLen; + *(end + interval) = 0; + + for (char *p = end; p >= begin; p--) { + *(p + interval) = *p; + } + + memcpy(begin, dst, dstLen); + replaced = true; + } + + simInfo("system cmd is %s", buf); + return replaced; +} + +bool simExecuteSystemCmd(SScript *script, char *option) { + char buf[4096] = {0}; + bool replaced = false; + +#ifndef WINDOWS + sprintf(buf, "cd %s; ", simScriptDir); + simVisuallizeOption(script, option, buf + strlen(buf)); +#else + sprintf(buf, "%s", simScriptDir); + simVisuallizeOption(script, option, buf + strlen(buf)); + simReplaceStr(buf, ".sh", ".bat"); +#endif + + if (useMultiProcess) { + simReplaceStr(buf, "deploy.sh", "deploy.sh -m"); + } + + if (useValgrind) { + replaced = simReplaceStr(buf, "exec.sh", "exec.sh -v"); + } + + simLogSql(buf, true); + int32_t code = system(buf); + int32_t repeatTimes = 0; + while (code < 0) { + simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d", script->fileName, buf, code, + errno, strerror(errno), repeatTimes); + taosMsleep(1000); + taosDflSignal(SIGCHLD); + if (repeatTimes++ >= 10) { + exit(0); + } + } + + sprintf(script->system_exit_code, "%d", code); + script->linePos++; + if (replaced && strstr(buf, "start") != NULL) { + simInfo("====> startup is slow in valgrind mode, so sleep 5 seconds after exec.sh -s start"); + taosMsleep(5000); + } + + return true; +} + +void simStoreSystemContentResult(SScript *script, char *filename) { + memset(script->system_ret_content, 0, MAX_SYSTEM_RESULT_LEN); + + TdFilePtr pFile; + // if ((fd = fopen(filename, "r")) != NULL) { + if ((pFile = taosOpenFile(filename, TD_FILE_READ)) != NULL) { + taosReadFile(pFile, script->system_ret_content, MAX_SYSTEM_RESULT_LEN - 1); + int32_t len = strlen(script->system_ret_content); + for (int32_t i = 0; i < len; ++i) { + if (script->system_ret_content[i] == '\n' || script->system_ret_content[i] == '\r') { + script->system_ret_content[i] = 0; + } + } + taosCloseFile(&pFile); + char rmCmd[MAX_FILE_NAME_LEN] = {0}; + sprintf(rmCmd, "rm -f %s", filename); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-result" + system(rmCmd); +#pragma GCC diagnostic pop + } +} + +bool simExecuteSystemContentCmd(SScript *script, char *option) { + char buf[4096] = {0}; + char buf1[4096 + 512] = {0}; + char filename[400] = {0}; + sprintf(filename, "%s" TD_DIRSEP "%s.tmp", simScriptDir, script->fileName); + +#ifdef WINDOWS + sprintf(buf, "cd %s && ", simScriptDir); + simVisuallizeOption(script, option, buf + strlen(buf)); + sprintf(buf1, "%s > %s 2>nul", buf, filename); +#else + sprintf(buf, "cd %s; ", simScriptDir); + simVisuallizeOption(script, option, buf + strlen(buf)); + sprintf(buf1, "%s > %s 2>/dev/null", buf, filename); +#endif + + sprintf(script->system_exit_code, "%d", system(buf1)); + simStoreSystemContentResult(script, filename); + + script->linePos++; + return true; +} + +bool simExecutePrintCmd(SScript *script, char *rest) { + char buf[65536]; + + simVisuallizeOption(script, rest, buf); + rest = buf; + + simInfo("script:%s, %s", script->fileName, rest); + script->linePos++; + return true; +} + +bool simExecuteSleepCmd(SScript *script, char *option) { + int32_t delta; + char buf[1024]; + + simVisuallizeOption(script, option, buf); + option = buf; + + delta = atoi(option); + if (delta <= 0) delta = 5; + + simInfo("script:%s, sleep %dms begin", script->fileName, delta); + taosMsleep(delta); + simInfo("script:%s, sleep %dms finished", script->fileName, delta); + + char sleepStr[32] = {0}; + sprintf(sleepStr, "sleep %d", delta); + simLogSql(sleepStr, true); + + script->linePos++; + return true; +} + +bool simExecuteReturnCmd(SScript *script, char *option) { + char buf[1024]; + + simVisuallizeOption(script, option, buf); + option = buf; + + int32_t ret = 1; + if (option && option[0] != 0) ret = atoi(option); + + if (ret < 0) { + sprintf(script->error, "lineNum:%d. error return %s", script->lines[script->linePos].lineNum, option); + return false; + } else { + simInfo("script:%s, return cmd execute with:%d", script->fileName, ret); + script->linePos = script->numOfLines; + } + + script->linePos++; + return true; +} + +void simVisuallizeOption(SScript *script, char *src, char *dst) { + char *var, *token, *value; + int32_t dstLen, srcLen, tokenLen; + + dst[0] = 0, dstLen = 0; + + while (1) { + var = strchr(src, '$'); + if (var == NULL) break; + if (var && ((var - src - 1) > 0) && *(var - 1) == '\\') { + srcLen = (int32_t)(var - src - 1); + memcpy(dst + dstLen, src, srcLen); + dstLen += srcLen; + src = var; + break; + } + + srcLen = (int32_t)(var - src); + memcpy(dst + dstLen, src, srcLen); + dstLen += srcLen; + + src = paGetToken(var + 1, &token, &tokenLen); + value = simGetVariable(script, token, tokenLen); + + strcpy(dst + dstLen, value); + dstLen += (int32_t)strlen(value); + } + + strcpy(dst + dstLen, src); +} + +void simCloseNativeConnect(SScript *script) { + if (script->taos == NULL) return; + + simDebug("script:%s, taos:%p closed", script->fileName, script->taos); + taos_close(script->taos); + + script->taos = NULL; +} + +void simCloseTaosdConnect(SScript *script) { simCloseNativeConnect(script); } + +bool simCreateNativeConnect(SScript *script, char *user, char *pass) { + simCloseTaosdConnect(script); + void *taos = NULL; + for (int32_t attempt = 0; attempt < 10; ++attempt) { + if (abortExecution) { + script->killed = true; + return false; + } + + taos = taos_connect(NULL, user, pass, NULL, 0); + if (taos == NULL) { + simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), + attempt); + taosMsleep(1000); + } else { + simDebug("script:%s, user:%s connect taosd successed, attempt:%d", script->fileName, user, attempt); + break; + } + } + + if (taos == NULL) { + sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, + taos_errstr(NULL)); + return false; + } + + script->taos = taos; + simDebug("script:%s, connect taosd successed, taos:%p", script->fileName, taos); + + return true; +} + +bool simCreateTaosdConnect(SScript *script, char *rest) { + char *user = TSDB_DEFAULT_USER; + char *token; + int32_t tokenLen; + rest = paGetToken(rest, &token, &tokenLen); + rest = paGetToken(rest, &token, &tokenLen); + if (tokenLen != 0) { + user = token; + } + + return simCreateNativeConnect(script, user, TSDB_DEFAULT_PASS); +} + +bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { + char timeStr[30] = {0}; + time_t tt; + struct tm tp; + SCmdLine *line = &script->lines[script->linePos]; + int32_t ret = -1; + + TAOS_RES *pSql = NULL; + + for (int32_t attempt = 0; attempt < 10; ++attempt) { + if (abortExecution) { + script->killed = true; + return false; + } + + simLogSql(rest, false); + pSql = taos_query(script->taos, rest); + ret = taos_errno(pSql); + + if (ret == TSDB_CODE_MND_STB_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { + simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, + tstrerror(ret)); + ret = 0; + break; + } else if (ret != 0) { + simDebug("script:%s, taos:%p, %s failed, ret:%d:%s, error:%s", script->fileName, script->taos, rest, ret & 0XFFFF, + tstrerror(ret), taos_errstr(pSql)); + + if (line->errorJump == SQL_JUMP_TRUE) { + script->linePos = line->jump; + taos_free_result(pSql); + return true; + } + taosMsleep(1000); + } else { + break; + } + + taos_free_result(pSql); + } + + if (ret) { + sprintf(script->error, "lineNum:%d. sql:%s failed, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, tstrerror(ret)); + return false; + } + + int32_t numOfRows = 0; + int32_t num_fields = taos_field_count(pSql); + if (num_fields != 0) { + if (pSql == NULL) { + simDebug("script:%s, taos:%p, %s failed, result is null", script->fileName, script->taos, rest); + if (line->errorJump == SQL_JUMP_TRUE) { + script->linePos = line->jump; + return true; + } + + sprintf(script->error, "lineNum:%d. result set null, sql:%s", line->lineNum, rest); + return false; + } + + TAOS_ROW row; + + while ((row = taos_fetch_row(pSql))) { + if (numOfRows < MAX_QUERY_ROW_NUM) { + TAOS_FIELD *fields = taos_fetch_fields(pSql); + int32_t *length = taos_fetch_lengths(pSql); + + for (int32_t i = 0; i < num_fields; i++) { + char *value = NULL; + if (i < MAX_QUERY_COL_NUM) { + value = script->data[numOfRows][i]; + } + if (value == NULL) { + continue; + } + + if (row[i] == 0) { + strcpy(value, TSDB_DATA_NULL_STR); + continue; + } + + switch (fields[i].type) { + case TSDB_DATA_TYPE_BOOL: + sprintf(value, "%s", ((((int32_t)(*((char *)row[i]))) == 1) ? "1" : "0")); + break; + case TSDB_DATA_TYPE_TINYINT: + sprintf(value, "%d", *((int8_t *)row[i])); + break; + case TSDB_DATA_TYPE_UTINYINT: + sprintf(value, "%u", *((uint8_t *)row[i])); + break; + case TSDB_DATA_TYPE_SMALLINT: + sprintf(value, "%d", *((int16_t *)row[i])); + break; + case TSDB_DATA_TYPE_USMALLINT: + sprintf(value, "%u", *((uint16_t *)row[i])); + break; + case TSDB_DATA_TYPE_INT: + sprintf(value, "%d", *((int32_t *)row[i])); + break; + case TSDB_DATA_TYPE_UINT: + sprintf(value, "%u", *((uint32_t *)row[i])); + break; + case TSDB_DATA_TYPE_BIGINT: + sprintf(value, "%" PRId64, *((int64_t *)row[i])); + break; + case TSDB_DATA_TYPE_UBIGINT: + sprintf(value, "%" PRIu64, *((uint64_t *)row[i])); + break; + case TSDB_DATA_TYPE_FLOAT: + sprintf(value, "%.5f", GET_FLOAT_VAL(row[i])); + break; + case TSDB_DATA_TYPE_DOUBLE: + sprintf(value, "%.9lf", GET_DOUBLE_VAL(row[i])); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + if (length[i] < 0 || length[i] > 1 << 20) { + fprintf(stderr, "Invalid length(%d) of BINARY or NCHAR\n", length[i]); + exit(-1); + } + + memset(value, 0, MAX_QUERY_VALUE_LEN); + memcpy(value, row[i], length[i]); + value[length[i]] = 0; + // snprintf(value, fields[i].bytes, "%s", (char *)row[i]); + break; + case TSDB_DATA_TYPE_TIMESTAMP: { + int32_t precision = taos_result_precision(pSql); + if (precision == TSDB_TIME_PRECISION_MILLI) { + tt = (*(int64_t *)row[i]) / 1000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (*(int64_t *)row[i]) / 1000000; + } else { + tt = (*(int64_t *)row[i]) / 1000000000; + } + + taosLocalTime(&tt, &tp); + strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", &tp); + if (precision == TSDB_TIME_PRECISION_MILLI) { + sprintf(value, "%s.%03d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000)); + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(value, "%s.%06d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000)); + } else { + sprintf(value, "%s.%09d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000000)); + } + + break; + } + default: + break; + } // end of switch + } // end of for + } // end of if + numOfRows++; + if (isSlow && numOfRows % 100 == 0) { + taosMsleep(200); + } + if (numOfRows > 2000000000) { + simError("script:%s, too many rows return from query", script->fileName); + break; + } + } + + } else { + numOfRows = taos_affected_rows(pSql); + } + + taos_free_result(pSql); + sprintf(script->rows, "%d", numOfRows); + + script->linePos++; + return true; +} + +bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) { + char buf[3000]; + SCmdLine *line = &script->lines[script->linePos]; + + simVisuallizeOption(script, rest, buf); + rest = buf; + + simDebug("script:%s, exec:%s", script->fileName, rest); + strcpy(script->rows, "-1"); + for (int32_t row = 0; row < MAX_QUERY_ROW_NUM; ++row) { + for (int32_t col = 0; col < MAX_QUERY_COL_NUM; ++col) { + strcpy(script->data[row][col], "null"); + } + } + + if (strncmp(rest, "connect", 7) == 0) { + if (!simCreateTaosdConnect(script, rest)) { + return false; + } + script->linePos++; + return true; + } + + if (script->taos == NULL) { + if (!simCreateTaosdConnect(script, "connect root")) { + if (line->errorJump == SQL_JUMP_TRUE) { + script->linePos = line->jump; + return true; + } + return false; + } + } + + if (strncmp(rest, "close", 5) == 0) { + simCloseTaosdConnect(script); + script->linePos++; + return true; + } + + return simExecuteNativeSqlCommand(script, rest, isSlow); +} + +bool simExecuteSqlCmd(SScript *script, char *rest) { + bool isSlow = false; + return simExecuteSqlImpCmd(script, rest, isSlow); +} + +bool simExecuteSqlSlowCmd(SScript *script, char *rest) { + bool isSlow = true; + return simExecuteSqlImpCmd(script, rest, isSlow); +} + +bool simExecuteRestfulCmd(SScript *script, char *rest) { + TdFilePtr pFile = NULL; + char filename[256]; + sprintf(filename, "%s/tmp.sql", simScriptDir); + // fp = fopen(filename, "w"); + pFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + if (pFile == NULL) { + fprintf(stderr, "ERROR: failed to open file: %s\n", filename); + return false; + } + + char db[64] = {0}; + char tb[64] = {0}; + char gzip[32] = {0}; + int32_t ts; + int32_t times; + sscanf(rest, "%s %s %d %d %s", db, tb, &ts, ×, gzip); + + taosFprintfFile(pFile, "insert into %s.%s values ", db, tb); + for (int32_t i = 0; i < times; ++i) { + taosFprintfFile(pFile, "(%d000, %d)", ts + i, ts); + } + taosFprintfFile(pFile, " \n"); + taosFsyncFile(pFile); + taosCloseFile(&pFile); + + char cmd[1024] = {0}; + if (strcmp(gzip, "gzip") == 0) { + sprintf(cmd, + "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header " + "--compressed --data-ascii @%s 127.0.0.1:7111/rest/sql", + filename); + } else { + sprintf(cmd, + "curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' --header " + "'Transfer-Encoding: chunked' --data-ascii @%s 127.0.0.1:7111/rest/sql", + filename); + } + + return simExecuteSystemCmd(script, cmd); +} + +bool simExecuteSqlErrorCmd(SScript *script, char *rest) { + char buf[3000]; + SCmdLine *line = &script->lines[script->linePos]; + + simVisuallizeOption(script, rest, buf); + rest = buf; + + simDebug("script:%s, exec:%s", script->fileName, rest); + strcpy(script->rows, "-1"); + for (int32_t row = 0; row < MAX_QUERY_ROW_NUM; ++row) { + for (int32_t col = 0; col < MAX_QUERY_COL_NUM; ++col) { + strcpy(script->data[row][col], "null"); + } + } + + if (strncmp(rest, "connect", 7) == 0) { + if (!simCreateTaosdConnect(script, rest)) { + return false; + } + script->linePos++; + return true; + } + + if (script->taos == NULL) { + if (!simCreateTaosdConnect(script, "connect root")) { + if (line->errorJump == SQL_JUMP_TRUE) { + script->linePos = line->jump; + return true; + } + return false; + } + } + + if (strncmp(rest, "close", 5) == 0) { + simCloseTaosdConnect(script); + script->linePos++; + return true; + } + + TAOS_RES *pSql = pSql = taos_query(script->taos, rest); + int32_t ret = taos_errno(pSql); + taos_free_result(pSql); + + if (ret != TSDB_CODE_SUCCESS) { + simDebug("script:%s, taos:%p, %s execute, expect failed, so success, ret:%d:%s", script->fileName, script->taos, + rest, ret & 0XFFFF, tstrerror(ret)); + script->linePos++; + return true; + } + + sprintf(script->error, "lineNum:%d. sql:%s expect failed, but success, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, + tstrerror(ret)); + + return false; +} + +bool simExecuteLineInsertCmd(SScript *script, char *rest) { + char buf[TSDB_MAX_BINARY_LEN]; + + simVisuallizeOption(script, rest, buf); + rest = buf; + + SCmdLine *line = &script->lines[script->linePos]; + + simInfo("script:%s, %s", script->fileName, rest); + simLogSql(buf, true); + char *lines[] = {rest}; +#if 0 + int32_t ret = taos_insert_lines(script->taos, lines, 1); +#else + int32_t ret = 0; +#endif + if (ret == TSDB_CODE_SUCCESS) { + simDebug("script:%s, taos:%p, %s executed. success.", script->fileName, script->taos, rest); + script->linePos++; + return true; + } else { + sprintf(script->error, "lineNum: %d. line: %s failed, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, + tstrerror(ret)); + return false; + } +} + +bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) { + char buf[TSDB_MAX_BINARY_LEN]; + + simVisuallizeOption(script, rest, buf); + rest = buf; + + SCmdLine *line = &script->lines[script->linePos]; + + simInfo("script:%s, %s", script->fileName, rest); + simLogSql(buf, true); + char *lines[] = {rest}; +#if 0 + int32_t ret = taos_insert_lines(script->taos, lines, 1); +#else + int32_t ret = 0; +#endif + if (ret == TSDB_CODE_SUCCESS) { + sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName, + script->taos, rest); + script->linePos++; + return false; + } else { + simDebug("lineNum: %d. line: %s failed, ret:%d:%s. Expect failed, so success", line->lineNum, rest, ret & 0XFFFF, + tstrerror(ret)); + return true; + } +} diff --git a/utils/tsim/src/simMain.c b/utils/tsim/src/simMain.c new file mode 100644 index 0000000000000000000000000000000000000000..713e46df58e8957be2593eb5d78698ed46e5a69b --- /dev/null +++ b/utils/tsim/src/simMain.c @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "simInt.h" + +bool simExecSuccess = false; +bool abortExecution = false; +bool useMultiProcess = false; +bool useValgrind = false; + +void simHandleSignal(int32_t signo, void *sigInfo, void *context) { + simSystemCleanUp(); + abortExecution = true; +} + +int32_t main(int32_t argc, char *argv[]) { + char scriptFile[MAX_FILE_NAME_LEN] = "sim_main_test.sim"; + + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0 && i < argc - 1) { + tstrncpy(configDir, argv[++i], 128); + } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) { + strcpy(scriptFile, argv[++i]); + } else if (strcmp(argv[i], "-m") == 0) { + useMultiProcess = true; + } else if (strcmp(argv[i], "-v") == 0) { + useValgrind = true; + } else { + printf("usage: %s [options] \n", argv[0]); + printf(" [-c config]: config directory, default is: %s\n", configDir); + printf(" [-f script]: script filename\n"); + return 0; + } + } + + if (!simSystemInit()) { + simError("failed to initialize the system"); + simSystemCleanUp(); + return -1; + } + + simInfo("simulator is running ..."); + taosSetSignal(SIGINT, simHandleSignal); + + SScript *script = simParseScript(scriptFile); + if (script == NULL) { + simError("parse script file:%s failed", scriptFile); + return -1; + } + + if (abortExecution) { + simError("execute abort"); + return -1; + } + + simScriptList[++simScriptPos] = script; + simExecuteScript(script); + + int32_t ret = simExecSuccess ? 0 : -1; + simInfo("execute result %d", ret); + + return ret; +} diff --git a/utils/tsim/src/simParse.c b/utils/tsim/src/simParse.c new file mode 100644 index 0000000000000000000000000000000000000000..b9f7610be83ce77c6e76d9821e9b77bd0816284d --- /dev/null +++ b/utils/tsim/src/simParse.c @@ -0,0 +1,1097 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/* Thi file is to parse the simScriptPos, function file and default file, + * + * syntax + * + * expression is format like: + * $var [=|<|>|==] var + * or: + * $var = var [+|-|*|/|.] var + * + * if expression + * command + * + * if expression then + * commands + * elif expression + * commands + * elif expression + * commands + * else + * commands + * endi + * + * while expression + * commands + * continue + * break + * endw + * + * switch expression + * case 1 + * commands + * break + * case 2 + * commands + * break + * default + * commands + * ends + * + * label: + * goto label + * + */ + +#define _DEFAULT_SOURCE +#include "simParse.h" + +static SCommand *cmdHashList[MAX_NUM_CMD]; +static SCmdLine cmdLine[MAX_CMD_LINES]; +static char parseErr[MAX_ERROR_LEN]; +static char optionBuffer[MAX_OPTION_BUFFER]; +static int32_t numOfLines, optionOffset; +static SLabel label, dest; +static SBlock block; + +int32_t simHashCmd(char *token, int32_t tokenLen) { + int32_t i; + int32_t hash = 0; + + for (i = 0; i < tokenLen; ++i) hash += token[i]; + + hash = hash % MAX_NUM_CMD; + + return hash; +} + +SCommand *simCheckCmd(char *token, int32_t tokenLen) { + int32_t hash; + SCommand *node; + + hash = simHashCmd(token, tokenLen); + + node = cmdHashList[hash]; + + while (node) { + if (node->nlen == tokenLen && strncmp(node->name, token, tokenLen) == 0) { + return node; + } else { + node = node->next; + } + } + + return NULL; +} + +void simAddCmdIntoHash(SCommand *pCmd) { + int32_t hash; + SCommand *node; + + hash = simHashCmd(pCmd->name, (int32_t)strlen(pCmd->name)); + node = cmdHashList[hash]; + pCmd->next = node; + cmdHashList[hash] = pCmd; +} + +void simResetParser() { + optionOffset = 4; + numOfLines = 0; + memset(cmdLine, 0, sizeof(cmdLine)); + memset(optionBuffer, 0, sizeof(optionBuffer)); + memset(&label, 0, sizeof(label)); + memset(&block, 0, sizeof(block)); + memset(&dest, 0, sizeof(dest)); +} + +SScript *simBuildScriptObj(char *fileName) { + int32_t i, destPos; + + /* process labels */ + + cmdLine[numOfLines].cmdno = SIM_CMD_RETURN; + numOfLines++; + + for (i = 0; i < numOfLines; ++i) { + cmdLine[i].errorJump = SQL_JUMP_FALSE; + } + + for (--dest.top; dest.top >= 0; --dest.top) { + for (i = 0; i < label.top; ++i) { + if (strcmp(label.label[i], dest.label[(uint8_t)dest.top]) == 0) break; + } + + if (i == label.top) { + sprintf(parseErr, "label:%s not defined", dest.label[(uint8_t)dest.top]); + return NULL; + } + + destPos = dest.pos[(uint8_t)dest.top]; + cmdLine[destPos].jump = label.pos[i]; + if (cmdLine[destPos].cmdno == SIM_CMD_SQL) { + cmdLine[destPos].errorJump = SQL_JUMP_TRUE; + } + } + + if (block.top != 0) { + sprintf(parseErr, "mismatched block"); + return NULL; + } + + for (i = 0; i < numOfLines; ++i) { + if (cmdLine[i].jump == 0) cmdLine[i].jump = numOfLines; + } + + SScript *script = taosMemoryMalloc(sizeof(SScript)); + memset(script, 0, sizeof(SScript)); + + script->type = SIM_SCRIPT_TYPE_MAIN; + script->numOfLines = numOfLines; + tstrncpy(script->fileName, fileName, sizeof(script->fileName)); + + script->optionBuffer = taosMemoryMalloc(optionOffset); + memcpy(script->optionBuffer, optionBuffer, optionOffset); + + script->lines = taosMemoryMalloc(sizeof(SCmdLine) * numOfLines); + memcpy(script->lines, cmdLine, sizeof(SCmdLine) * numOfLines); + + return script; +} + +SScript *simParseScript(char *fileName) { + TdFilePtr pFile; + int32_t tokenLen, lineNum = 0; + char buffer[10*1024], name[128], *token, *rest; + SCommand *pCmd; + SScript *script; + + if ((fileName[0] == '.') || (fileName[0] == '/')) { + strcpy(name, fileName); + } else { + sprintf(name, "%s" TD_DIRSEP "%s", simScriptDir, fileName); + taosRealPath(name, NULL, sizeof(name)); + } + + // if ((fd = fopen(name, "r")) == NULL) { + if ((pFile = taosOpenFile(name, TD_FILE_READ | TD_FILE_STREAM)) == NULL) { + simError("failed to open file:%s", name); + return NULL; + } + + simResetParser(); + + while (!taosEOFFile(pFile)) { + if (taosGetsFile(pFile, sizeof(buffer) - 1, buffer) == -1) continue; + + lineNum++; + int32_t cmdlen = (int32_t)strlen(buffer); + if (buffer[cmdlen - 1] == '\r' || buffer[cmdlen - 1] == '\n') { + buffer[cmdlen - 1] = 0; + } + rest = buffer; + + for (int32_t i = 0; i < cmdlen; ++i) { + if (buffer[i] == '\r' || buffer[i] == '\n') { + buffer[i] = '\0'; + } + } + + again: + rest = paGetToken(rest, &token, &tokenLen); + if (tokenLen == 0) continue; + + if (token[tokenLen - 1] == ':') { + strncpy(label.label[(uint8_t)label.top], token, tokenLen - 1); + label.pos[(uint8_t)label.top] = numOfLines; + label.top++; + goto again; + } + + if (token[0] == '$') { + if (!simParseExpression(token, lineNum)) { + simError("script:%s line:%d %s", fileName, lineNum, parseErr); + return NULL; + } + continue; + } + + if ((pCmd = simCheckCmd(token, tokenLen)) == NULL) { + token[tokenLen] = 0; + simError("script:%s line:%d invalid cmd:%s", fileName, lineNum, token); + return NULL; + } + + if (!pCmd->parseCmd(rest, pCmd, lineNum)) { + simError("script:%s line:%d %s", fileName, lineNum, parseErr); + return NULL; + } + } + taosCloseFile(&pFile); + + script = simBuildScriptObj(fileName); + if (script == NULL) simError("script:%s %s", fileName, parseErr); + + return script; +} + +int32_t simCheckExpression(char *exp) { + char *op1, *op2, *op, *rest; + int32_t op1Len, op2Len, opLen; + + rest = paGetToken(exp, &op1, &op1Len); + if (op1Len == 0) { + sprintf(parseErr, "expression is required"); + return -1; + } + + rest = paGetToken(rest, &op, &opLen); + if (opLen == 0) { + sprintf(parseErr, "operator is missed"); + return -1; + } + + rest = paGetToken(rest, &op2, &op2Len); + if (op2Len == 0) { + sprintf(parseErr, "operand is missed"); + return -1; + } + + if (opLen == 1) { + if (op[0] != '=' && op[0] != '<' && op[0] != '>') { + sprintf(parseErr, "invalid operator:%s", op); + return -1; + } + + if (op[0] == '=' && op1[0] != '$') { + sprintf(parseErr, "left side of assignment must be variable"); + return -1; + } + } else if (opLen == 2) { + if (op[1] != '=' || (op[0] != '=' && op[0] != '<' && op[0] != '>' && op[0] != '!')) { + sprintf(parseErr, "left side of assignment must be variable"); + return -1; + } + } else { + sprintf(parseErr, "invalid operator:%s", op); + return -1; + } + + rest = paGetToken(rest, &op, &opLen); + + if (opLen == 0) return (int32_t)(rest - exp); + + /* if it is key word "then" */ + if (strncmp(op, "then", 4) == 0) return (int32_t)(op - exp); + + rest = paGetToken(rest, &op2, &op2Len); + if (op2Len == 0) { + sprintf(parseErr, "operand is missed"); + return -1; + } + + if (opLen > 1) { + sprintf(parseErr, "invalid operator:%s", op); + return -1; + } + + if (op[0] == '+' || op[0] == '-' || op[0] == '*' || op[0] == '/' || op[0] == '.') { + return (int32_t)(rest - exp); + } + + return -1; +} + +bool simParseExpression(char *token, int32_t lineNum) { + int32_t expLen; + + expLen = simCheckExpression(token); + if (expLen <= 0) return -1; + + cmdLine[numOfLines].cmdno = SIM_CMD_EXP; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, token, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseIfCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char *ret; + int32_t expLen; + + expLen = simCheckExpression(rest); + + if (expLen <= 0) return -1; + + ret = rest + expLen; + + if (strncmp(ret, "then", 4) == 0) { + block.type[(uint8_t)block.top] = BLOCK_IF; + block.pos[(uint8_t)block.top] = &cmdLine[numOfLines].jump; + block.top++; + } else { + cmdLine[numOfLines].jump = numOfLines + 2; + } + + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + cmdLine[numOfLines].cmdno = SIM_CMD_TEST; + cmdLine[numOfLines].lineNum = lineNum; + + numOfLines++; + return true; +} + +bool simParseElifCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + expLen = simCheckExpression(rest); + + if (expLen <= 0) return -1; + + if (block.top < 1) { + sprintf(parseErr, "no matching if"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_IF) { + sprintf(parseErr, "no matched if block"); + return false; + } + + cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; + block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); + block.numJump[block.top - 1]++; + + numOfLines++; + + *(block.pos[block.top - 1]) = numOfLines; + block.pos[block.top - 1] = &cmdLine[numOfLines].jump; + + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + cmdLine[numOfLines].cmdno = SIM_CMD_TEST; + cmdLine[numOfLines].lineNum = lineNum; + + numOfLines++; + return true; +} + +bool simParseElseCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + if (block.top < 1) { + sprintf(parseErr, "no matching if"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_IF) { + sprintf(parseErr, "no matched if block"); + return false; + } + + cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; + block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); + block.numJump[block.top - 1]++; + + numOfLines++; + + *(block.pos[block.top - 1]) = numOfLines; + block.pos[block.top - 1] = NULL; + + return true; +} + +bool simParseEndiCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t i; + + if (block.top < 1) { + sprintf(parseErr, "no matching if"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_IF) { + sprintf(parseErr, "no matched if block"); + return false; + } + + if (block.pos[block.top - 1]) *(block.pos[block.top - 1]) = numOfLines; + + for (i = 0; i < block.numJump[block.top - 1]; ++i) { + *(block.jump[block.top - 1][i]) = numOfLines; + } + + block.numJump[block.top - 1] = 0; + block.top--; + + return true; +} + +bool simParseWhileCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + expLen = simCheckExpression(rest); + + if (expLen <= 0) return false; + + block.type[(uint8_t)block.top] = BLOCK_WHILE; + block.pos[(uint8_t)block.top] = &(cmdLine[numOfLines].jump); + block.back[(uint8_t)block.top] = numOfLines; + block.top++; + + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + cmdLine[numOfLines].cmdno = SIM_CMD_TEST; + cmdLine[numOfLines].lineNum = lineNum; + + numOfLines++; + return true; +} + +bool simParseEndwCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t i; + + if (block.top < 1) { + sprintf(parseErr, "no matching while"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_WHILE) { + sprintf(parseErr, "no matched while block"); + return false; + } + + cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; + cmdLine[numOfLines].jump = block.back[block.top - 1]; + cmdLine[numOfLines].lineNum = lineNum; + numOfLines++; + + *(block.pos[block.top - 1]) = numOfLines; + + for (i = 0; i < block.numJump[block.top - 1]; ++i) { + *(block.jump[block.top - 1][i]) = numOfLines; + } + + block.top--; + + return true; +} + +bool simParseSwitchCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char *token; + int32_t tokenLen; + + rest = paGetToken(rest, &token, &tokenLen); + if (tokenLen == 0) { + sprintf(parseErr, "switch should be followed by variable"); + return false; + } + + if (token[0] != '$') { + sprintf(parseErr, "switch must be followed by variable"); + return false; + } + + memcpy(block.sexp[(uint8_t)block.top], token, tokenLen); + block.sexpLen[(uint8_t)block.top] = tokenLen; + block.type[(uint8_t)block.top] = BLOCK_SWITCH; + block.top++; + + return true; +} + +bool simParseCaseCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char *token; + int32_t tokenLen; + + rest = paGetToken(rest, &token, &tokenLen); + if (tokenLen == 0) { + sprintf(parseErr, "case should be followed by value"); + return false; + } + + if (block.top < 1) { + sprintf(parseErr, "no matching switch"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_SWITCH) { + sprintf(parseErr, "case not matched"); + return false; + } + + if (block.pos[block.top - 1] != NULL) { + *(block.pos[block.top - 1]) = numOfLines; + } + + block.pos[block.top - 1] = &(cmdLine[numOfLines].jump); + + cmdLine[numOfLines].cmdno = SIM_CMD_TEST; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, block.sexp[block.top - 1], block.sexpLen[block.top - 1]); + optionOffset += block.sexpLen[block.top - 1]; + *(optionBuffer + optionOffset++) = ' '; + *(optionBuffer + optionOffset++) = '='; + *(optionBuffer + optionOffset++) = '='; + *(optionBuffer + optionOffset++) = ' '; + memcpy(optionBuffer + optionOffset, token, tokenLen); + optionOffset += tokenLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseBreakCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + if (block.top < 1) { + sprintf(parseErr, "no blcok exists"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_SWITCH && block.type[block.top - 1] != BLOCK_WHILE) { + sprintf(parseErr, "not in switch or while block"); + return false; + } + + block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); + block.numJump[block.top - 1]++; + + cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; + cmdLine[numOfLines].lineNum = lineNum; + + numOfLines++; + return true; +} + +bool simParseDefaultCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + if (block.top < 1) { + sprintf(parseErr, "no matching switch"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_SWITCH) { + sprintf(parseErr, "default should be matched with switch"); + return false; + } + + if (block.pos[block.top - 1] != NULL) { + *(block.pos[block.top - 1]) = numOfLines; + } + + return true; +} + +bool simParseEndsCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t i; + + if (block.top < 1) { + sprintf(parseErr, "no matching switch"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_SWITCH) { + sprintf(parseErr, "ends should be matched with switch"); + return false; + } + + for (i = 0; i < block.numJump[block.top - 1]; ++i) { + *(block.jump[block.top - 1][i]) = numOfLines; + } + + block.numJump[block.top - 1] = 0; + block.top--; + + return true; +} + +bool simParseContinueCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + if (block.top < 1) { + sprintf(parseErr, "no matching while"); + return false; + } + + if (block.type[block.top - 1] != BLOCK_WHILE) { + sprintf(parseErr, "continue should be matched with while cmd"); + return false; + } + + cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].jump = block.back[block.top - 1]; + + numOfLines++; + return true; +} + +bool simParsePrintCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + rest++; + cmdLine[numOfLines].cmdno = SIM_CMD_PRINT; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + expLen = (int32_t)strlen(rest); + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +void simCheckSqlOption(char *rest) { + int32_t valueLen; + char *value, *xpos; + + xpos = strstr(rest, " -x"); // need a blank + if (xpos) { + paGetToken(xpos + 3, &value, &valueLen); + if (valueLen != 0) { + memcpy(dest.label[(uint8_t)dest.top], value, valueLen); + dest.label[(uint8_t)dest.top][valueLen] = 0; + dest.pos[(uint8_t)dest.top] = numOfLines; + dest.top++; + + *xpos = 0; + } + } +} + +bool simParseSqlCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + rest++; + simCheckSqlOption(rest); + cmdLine[numOfLines].cmdno = SIM_CMD_SQL; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + expLen = (int32_t)strlen(rest); + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + rest++; + cmdLine[numOfLines].cmdno = SIM_CMD_SQL_ERROR; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + expLen = (int32_t)strlen(rest); + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + simParseSqlCmd(rest, pCmd, lineNum); + cmdLine[numOfLines - 1].cmdno = SIM_CMD_SQL_SLOW; + return true; +} + +bool simParseRestfulCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + simParseSqlCmd(rest, pCmd, lineNum); + cmdLine[numOfLines - 1].cmdno = SIM_CMD_RESTFUL; + return true; +} + +bool simParseSystemCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + rest++; + cmdLine[numOfLines].cmdno = SIM_CMD_SYSTEM; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + expLen = (int32_t)strlen(rest); + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseSystemContentCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + simParseSystemCmd(rest, pCmd, lineNum); + cmdLine[numOfLines - 1].cmdno = SIM_CMD_SYSTEM_CONTENT; + return true; +} + +bool simParseSleepCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char *token; + int32_t tokenLen; + + cmdLine[numOfLines].cmdno = SIM_CMD_SLEEP; + cmdLine[numOfLines].lineNum = lineNum; + + paGetToken(rest, &token, &tokenLen); + if (tokenLen > 0) { + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, token, tokenLen); + optionOffset += tokenLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + } + + numOfLines++; + return true; +} + +bool simParseReturnCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char *token; + int32_t tokenLen; + + cmdLine[numOfLines].cmdno = SIM_CMD_RETURN; + cmdLine[numOfLines].lineNum = lineNum; + + paGetToken(rest, &token, &tokenLen); + if (tokenLen > 0) { + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, token, tokenLen); + optionOffset += tokenLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + } + + numOfLines++; + return true; +} + +bool simParseGotoCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char *token; + int32_t tokenLen; + + rest = paGetToken(rest, &token, &tokenLen); + + if (tokenLen == 0) { + sprintf(parseErr, "label should be followed by goto cmd"); + return false; + } + + memcpy(dest.label[(uint8_t)dest.top], token, tokenLen); + dest.label[(uint8_t)dest.top][tokenLen] = 0; + dest.pos[(uint8_t)dest.top] = numOfLines; + dest.top++; + + cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; + cmdLine[numOfLines].lineNum = lineNum; + + numOfLines++; + return true; +} + +bool simParseRunCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char *token; + int32_t tokenLen; + + rest = paGetToken(rest, &token, &tokenLen); + + if (tokenLen == 0) { + sprintf(parseErr, "file name should be followed by run cmd"); + return false; + } + + cmdLine[numOfLines].cmdno = SIM_CMD_RUN; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + memcpy(optionBuffer + optionOffset, token, tokenLen); + optionOffset += tokenLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseRunBackCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + simParseRunCmd(rest, pCmd, lineNum); + cmdLine[numOfLines - 1].cmdno = SIM_CMD_RUN_BACK; + return true; +} + +bool simParseLineInsertCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + rest++; + cmdLine[numOfLines].cmdno = SIM_CMD_LINE_INSERT; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + expLen = (int32_t)strlen(rest); + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseLineInsertErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; + + rest++; + cmdLine[numOfLines].cmdno = SIM_CMD_LINE_INSERT; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + expLen = (int32_t)strlen(rest); + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +void simInitsimCmdList() { + int32_t cmdno; + memset(simCmdList, 0, SIM_CMD_END * sizeof(SCommand)); + + /* internal command */ + cmdno = SIM_CMD_EXP; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "exp"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = NULL; + simCmdList[cmdno].executeCmd = simExecuteExpCmd; + + cmdno = SIM_CMD_IF; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "if"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseIfCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_ELIF; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "elif"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseElifCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_ELSE; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "else"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseElseCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_ENDI; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "endi"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseEndiCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_WHILE; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "while"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseWhileCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_ENDW; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "endw"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseEndwCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_SWITCH; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "switch"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseSwitchCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_CASE; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "case"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseCaseCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_DEFAULT; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "default"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseDefaultCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_BREAK; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "break"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseBreakCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_CONTINUE; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "continue"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseContinueCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_ENDS; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "ends"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseEndsCmd; + simCmdList[cmdno].executeCmd = NULL; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_SLEEP; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "sleep"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseSleepCmd; + simCmdList[cmdno].executeCmd = simExecuteSleepCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_GOTO; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "goto"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseGotoCmd; + simCmdList[cmdno].executeCmd = simExecuteGotoCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_RUN; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "run"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseRunCmd; + simCmdList[cmdno].executeCmd = simExecuteRunCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_RUN_BACK; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "run_back"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseRunBackCmd; + simCmdList[cmdno].executeCmd = simExecuteRunBackCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_SYSTEM; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "system"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseSystemCmd; + simCmdList[cmdno].executeCmd = simExecuteSystemCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_SYSTEM_CONTENT; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "system_content"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseSystemContentCmd; + simCmdList[cmdno].executeCmd = simExecuteSystemContentCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_PRINT; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "print"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParsePrintCmd; + simCmdList[cmdno].executeCmd = simExecutePrintCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_SQL; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "sql"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseSqlCmd; + simCmdList[cmdno].executeCmd = simExecuteSqlCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_SQL_ERROR; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "sql_error"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseSqlErrorCmd; + simCmdList[cmdno].executeCmd = simExecuteSqlErrorCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_SQL_SLOW; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "sql_slow"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseSqlSlowCmd; + simCmdList[cmdno].executeCmd = simExecuteSqlSlowCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_RESTFUL; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "restful"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseRestfulCmd; + simCmdList[cmdno].executeCmd = simExecuteRestfulCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + /* test is only an internal command */ + cmdno = SIM_CMD_TEST; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "test"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = NULL; + simCmdList[cmdno].executeCmd = simExecuteTestCmd; + + cmdno = SIM_CMD_RETURN; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "return"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseReturnCmd; + simCmdList[cmdno].executeCmd = simExecuteReturnCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_LINE_INSERT; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "line_insert"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseLineInsertCmd; + simCmdList[cmdno].executeCmd = simExecuteLineInsertCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_LINE_INSERT_ERROR; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "line_insert_error"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseLineInsertErrorCmd; + simCmdList[cmdno].executeCmd = simExecuteLineInsertErrorCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); +} diff --git a/utils/tsim/src/simSystem.c b/utils/tsim/src/simSystem.c new file mode 100644 index 0000000000000000000000000000000000000000..f2fefb903d926d38d16544253a1c6cae0fc1b5c9 --- /dev/null +++ b/utils/tsim/src/simSystem.c @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "simInt.h" +#include "tconfig.h" + +SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; +SCommand simCmdList[SIM_CMD_END]; +int32_t simScriptPos = -1; +int32_t simScriptSucced = 0; +int32_t simDebugFlag = 143; +void simCloseTaosdConnect(SScript *script); +char simScriptDir[PATH_MAX] = {0}; + +extern bool simExecSuccess; + +int32_t simInitCfg() { + taosCreateLog("simlog", 1, configDir, NULL, NULL, NULL, NULL, 1); + taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1); + + SConfig *pCfg = taosGetCfg(); + simDebugFlag = cfgGetItem(pCfg, "simDebugFlag")->i32; + tstrncpy(simScriptDir, cfgGetItem(pCfg, "scriptDir")->str, PATH_MAX); + return 0; +} + +bool simSystemInit() { + simInitCfg(); + simInitsimCmdList(); + memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM); + return true; +} + +void simSystemCleanUp() {} + +void simFreeScript(SScript *script) { + if (script->type == SIM_SCRIPT_TYPE_MAIN) { + simInfo("script:%s, background script num:%d, stop them", script->fileName, script->bgScriptLen); + + for (int32_t i = 0; i < script->bgScriptLen; ++i) { + SScript *bgScript = script->bgScripts[i]; + simDebug("script:%s, is background script, set stop flag", bgScript->fileName); + bgScript->killed = true; + if (taosCheckPthreadValid(bgScript->bgPid)) { + taosThreadJoin(bgScript->bgPid, NULL); + taosThreadClear(&bgScript->bgPid); + } + + simDebug("script:%s, background thread joined", bgScript->fileName); + taos_close(bgScript->taos); + taosMemoryFreeClear(bgScript->lines); + taosMemoryFreeClear(bgScript->optionBuffer); + taosMemoryFreeClear(bgScript); + } + + simDebug("script:%s, is cleaned", script->fileName); + taos_close(script->taos); + taosMemoryFreeClear(script->lines); + taosMemoryFreeClear(script->optionBuffer); + taosMemoryFreeClear(script); + } +} + +SScript *simProcessCallOver(SScript *script) { + if (script->type == SIM_SCRIPT_TYPE_MAIN) { + simDebug("script:%s, is main script, set stop flag", script->fileName); + if (script->killed) { + simExecSuccess = false; + simInfo("script:" FAILED_PREFIX "%s" FAILED_POSTFIX ", " FAILED_PREFIX "failed" FAILED_POSTFIX ", error:%s", + script->fileName, script->error); + } else { + simExecSuccess = true; + simInfo("script:" SUCCESS_PREFIX "%s" SUCCESS_POSTFIX ", " SUCCESS_PREFIX "success" SUCCESS_POSTFIX, + script->fileName); + } + + simCloseTaosdConnect(script); + simScriptSucced++; + simScriptPos--; + simFreeScript(script); + + if (simScriptPos == -1 && simExecSuccess) { + simInfo("----------------------------------------------------------------------"); + simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced); + return NULL; + } + + if (simScriptPos == -1) return NULL; + if (!simExecSuccess) return NULL; + + return simScriptList[simScriptPos]; + } else { + simDebug("script:%s, is stopped", script->fileName); + simFreeScript(script); + return NULL; + } +} + +void *simExecuteScript(void *inputScript) { + SScript *script = (SScript *)inputScript; + + while (1) { + if (script->type == SIM_SCRIPT_TYPE_MAIN) { + script = simScriptList[simScriptPos]; + } + + if (abortExecution) { + script->killed = true; + } + + if (script->killed || script->linePos >= script->numOfLines) { + script = simProcessCallOver(script); + if (script == NULL) { + simDebug("sim test abort now!"); + break; + } + } else { + SCmdLine *line = &script->lines[script->linePos]; + char *option = script->optionBuffer + line->optionOffset; + simDebug("script:%s, line:%d with option \"%s\"", script->fileName, line->lineNum, option); + + SCommand *cmd = &simCmdList[line->cmdno]; + int32_t ret = (*(cmd->executeCmd))(script, option); + if (!ret) { + script->killed = true; + } + } + } + + simInfo("thread is stopped"); + return NULL; +}