diff --git a/README.md b/README.md index d5b6f1fa85b962253fe504fadff78e953d4da598..2f45d9618ecdb08a0d360dfcadb90c02bb8290fa 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,19 @@ taos If TDengine shell connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. +## Install TDengine by apt-get + +If you use Debian or Ubuntu system, you can use 'apt-get' command to intall TDengine from official repository. Please use following commands to setup: + +``` +wget -qO - http://repos.taosdata.com/tdengine.key | sudo apt-key add - +echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-stable stable main" | sudo tee /etc/apt/sources.list.d/tdengine-stable.list +[Optional] echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-beta beta main" | sudo tee /etc/apt/sources.list.d/tdengine-beta.list +sudo apt-get update +apt-get policy tdengine +sudo apt-get install tdengine +``` + ## Quick Run If you don't want to run TDengine as a service, you can run it in current shell. For example, to quickly start a TDengine server after building, run the command below in terminal: (We take Linux as an example, command on Windows will be `taosd.exe`) diff --git a/cmake/define.inc b/cmake/define.inc index 7894e6dab5d4ddd44e69f77702004183f431d3a6..a5fbd8abe959fb8530dd8cc26c79eefd47ec6b9f 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -124,6 +124,10 @@ IF (TD_APLHINE) MESSAGE(STATUS "aplhine is defined") ENDIF () +IF (TD_BUILD_HTTP) + ADD_DEFINITIONS(-DHTTP_EMBEDDED) +ENDIF () + IF (TD_LINUX) ADD_DEFINITIONS(-DLINUX) ADD_DEFINITIONS(-D_LINUX) diff --git a/cmake/input.inc b/cmake/input.inc index d746cf52f6eb016795d6fa6d01f408925159c710..ed8788922c8026cbc4c6b90ef477dcbd21cf8ba0 100755 --- a/cmake/input.inc +++ b/cmake/input.inc @@ -90,6 +90,12 @@ IF (${BUILD_JDBC} MATCHES "false") SET(TD_BUILD_JDBC FALSE) ENDIF () +SET(TD_BUILD_HTTP TRUE) + +IF (${BUILD_HTTP} MATCHES "false") + SET(TD_BUILD_HTTP FALSE) +ENDIF () + SET(TD_MEMORY_SANITIZER FALSE) IF (${MEMORY_SANITIZER} MATCHES "true") SET(TD_MEMORY_SANITIZER TRUE) diff --git a/documentation20/cn/02.getting-started/docs.md b/documentation20/cn/02.getting-started/docs.md index a37afa9212911f4e48efe5e923607f3f2e05422a..83915f4973957a68b51c6f155a857f11f2039e72 100644 --- a/documentation20/cn/02.getting-started/docs.md +++ b/documentation20/cn/02.getting-started/docs.md @@ -22,6 +22,18 @@ TDengine 的安装非常简单，从下载到安装成功仅仅只要几秒钟 具体的安装过程，请参见 [TDengine 多种安装包的安装和卸载](https://www.taosdata.com/blog/2019/08/09/566.html) 以及 [视频教程](https://www.taosdata.com/blog/2020/11/11/1941.html)。 +### 使用 apt-get 安装 + +如果使用 Debian 或 Ubuntu 系统，也可以使用 apt-get 从官方仓库安装，设置方法为： +``` +wget -qO - http://repos.taosdata.com/tdengine.key | sudo apt-key add - +echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-stable stable main" | sudo tee /etc/apt/sources.list.d/tdengine-stable.list +[ beta 版安装包仓库为可选安装项 ] echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-beta beta main" | sudo tee /etc/apt/sources.list.d/tdengine-beta.list +sudo apt-get update +apt-get policy tdengine +sudo apt-get install tdengine +``` + ## 轻松启动 diff --git a/documentation20/en/02.getting-started/docs.md b/documentation20/en/02.getting-started/docs.md index 50a8c2fabb8c93a847a79a4de47c218de7ccd60a..fa77fee0abd931cf34290cd75b5a8a95090040c4 100644 --- a/documentation20/en/02.getting-started/docs.md +++ b/documentation20/en/02.getting-started/docs.md @@ -20,6 +20,19 @@ Three different packages for TDengine server are provided, please pick up the on Click [here](https://www.taosdata.com/en/getting-started/#Install-from-Package) to download the install package. +### Install TDengine by apt-get + +If you use Debian or Ubuntu system you can use 'apt-get' command to install TDengine from official repository. Please use following commands to setup: + +``` +wget -qO - http://repos.taosdata.com/tdengine.key | sudo apt-key add - +echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-stable stable main" | sudo tee /etc/apt/sources.list.d/tdengine-stable.list +[Optional] echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-beta beta main" | sudo tee /etc/apt/sources.list.d/tdengine-beta.list +sudo apt-get update +apt-get policy tdengine +sudo apt-get install tdengine +``` + ## Quick Launch After installation, you can start the TDengine service by the `systemctl` command. @@ -218,4 +231,4 @@ Comparison matrix as following: Note: ● has been verified by official tests; ○ has been verified by unofficial tests. -Please visit Connectors section for more detailed information. \ No newline at end of file +Please visit Connectors section for more detailed information. diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index cf53977103c3a9760286e70447d826f7026d7e53..4eb0d2b2bfdd1d42ec93a85083c2f0fc0382af5a 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -5,7 +5,11 @@ INCLUDE_DIRECTORIES(inc) INCLUDE_DIRECTORIES(jni) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/plugins/http/inc) + +IF (TD_BUILD_HTTP) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/plugins/http/inc) +ENDIF () + AUX_SOURCE_DIRECTORY(src SRC) IF (TD_LINUX) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 60a6e241ccdfd48c7eb5a68f2dd7a251f76097a5..04d1472bb0101e903aa697ccc443d60e386c4ba2 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -29,7 +29,10 @@ #include "tsclient.h" #include "ttimer.h" #include "ttokendef.h" + +#ifdef HTTP_EMBEDDED #include "httpInt.h" +#endif static void freeQueryInfoImpl(SQueryInfo* pQueryInfo); @@ -5118,11 +5121,14 @@ void tscRemoveCachedTableMeta(STableMetaInfo* pTableMetaInfo, uint64_t id) { char* cloneCurrentDBName(SSqlObj* pSql) { char *p = NULL; +#ifdef HTTP_EMBEDDED HttpContext *pCtx = NULL; +#endif pthread_mutex_lock(&pSql->pTscObj->mutex); STscObj *pTscObj = pSql->pTscObj; switch (pTscObj->from) { +#ifdef HTTP_EMBEDDED case TAOS_REQ_FROM_HTTP: pCtx = pSql->param; if (pCtx && pCtx->db[0] != '\0') { @@ -5133,6 +5139,7 @@ char* cloneCurrentDBName(SSqlObj* pSql) { p = strdup(db); } break; +#endif default: break; } diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 47186130ead0d1ee3f4593b7ef346f8cc47f7cba..a009448c822ca651fd91583277ed92b183f155d2 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -18,7 +18,12 @@ ELSE () ENDIF () ADD_EXECUTABLE(taosd ${SRC}) + +IF (TD_BUILD_HTTP) TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lua lz4 balance sync ${LINK_JEMALLOC}) +ELSE () +TARGET_LINK_LIBRARIES(taosd mnode monitor tsdb twal vnode cJson lua lz4 balance sync ${LINK_JEMALLOC}) +ENDIF () IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(taosd taos_static) diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c index a661585b3b39df986ac7866a255472e47e789fe6..39f9e352793ffcab885438309980c0c530e048ad 100644 --- a/src/dnode/src/dnodeModule.c +++ b/src/dnode/src/dnodeModule.c @@ -49,6 +49,7 @@ static void dnodeAllocModules() { tsModule[TSDB_MOD_MNODE].startFp = mnodeStartSystem; tsModule[TSDB_MOD_MNODE].stopFp = mnodeStopSystem; +#ifdef HTTP_EMBEDDED tsModule[TSDB_MOD_HTTP].enable = (tsEnableHttpModule == 1); tsModule[TSDB_MOD_HTTP].name = "http"; tsModule[TSDB_MOD_HTTP].initFp = httpInitSystem; @@ -58,6 +59,7 @@ static void dnodeAllocModules() { if (tsEnableHttpModule) { dnodeSetModuleStatus(TSDB_MOD_HTTP); } +#endif #ifdef _MQTT tsModule[TSDB_MOD_MQTT].enable = (tsEnableMqttModule == 1); diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index 5606681f0f931070e9cbf21d6b98b0d2eb51bdfa..98bbbf8f73b26535030c5096f128a7f84c2b9f61 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -240,7 +240,9 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) { SStatisInfo dnodeGetStatisInfo() { SStatisInfo info = {0}; if (dnodeGetRunStatus() == TSDB_RUN_STATUS_RUNING) { +#ifdef HTTP_EMBEDDED info.httpReqNum = httpGetReqCount(); +#endif info.queryReqNum = atomic_exchange_32(&tsQueryReqNum, 0); info.submitReqNum = atomic_exchange_32(&tsSubmitReqNum, 0); } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index b8201f04b78234b554532d2b3a3427bc888e89c1..fa062c3f3ebf8e98f27dc27b5787b3790f8f2aae 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -986,36 +986,55 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->performance_print = true; } else if ((0 == strncmp(argv[i], "-P", strlen("-P"))) || (0 == strncmp(argv[i], "--port", strlen("--port")))) { + uint64_t port; + char strPort[BIGINT_BUFF_LEN]; + if (2 == strlen(argv[i])) { if (argc == i+1) { errorPrintReqArg(argv[0], "P"); exit(EXIT_FAILURE); - } else if (!isStringNumber(argv[i+1])) { + } else if (isStringNumber(argv[i+1])) { + tstrncpy(strPort, argv[++i], BIGINT_BUFF_LEN); + } else { errorPrintReqArg2(argv[0], "P"); exit(EXIT_FAILURE); } - arguments->port = atoi(argv[++i]); } else if (0 == strncmp(argv[i], "--port=", strlen("--port="))) { if (isStringNumber((char *)(argv[i] + strlen("--port=")))) { - arguments->port = atoi((char *)(argv[i]+strlen("--port="))); + tstrncpy(strPort, (char *)(argv[i]+strlen("--port=")), BIGINT_BUFF_LEN); + } else { + errorPrintReqArg2(argv[0], "--port"); + exit(EXIT_FAILURE); } } else if (0 == strncmp(argv[i], "-P", strlen("-P"))) { if (isStringNumber((char *)(argv[i] + strlen("-P")))) { - arguments->port = atoi((char *)(argv[i]+strlen("-P"))); + tstrncpy(strPort, (char *)(argv[i]+strlen("-P")), BIGINT_BUFF_LEN); + } else { + errorPrintReqArg2(argv[0], "--port"); + exit(EXIT_FAILURE); } } else if (strlen("--port") == strlen(argv[i])) { if (argc == i+1) { errorPrintReqArg3(argv[0], "--port"); exit(EXIT_FAILURE); - } else if (!isStringNumber(argv[i+1])) { + } else if (isStringNumber(argv[i+1])) { + tstrncpy(strPort, argv[++i], BIGINT_BUFF_LEN); + } else { errorPrintReqArg2(argv[0], "--port"); exit(EXIT_FAILURE); } - arguments->port = atoi(argv[++i]); } else { errorUnrecognized(argv[0], argv[i]); exit(EXIT_FAILURE); } + + port = atoi(strPort); + if (port > 65535) { + errorWrongValue("taosdump", "-P or --port", strPort); + exit(EXIT_FAILURE); + } + arguments->port = (uint16_t)port; + } else if ((0 == strncmp(argv[i], "-I", strlen("-I"))) || (0 == strncmp(argv[i], "--interface", strlen("--interface")))) { if (2 == strlen(argv[i])) { @@ -1581,7 +1600,9 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { && strcasecmp(dataType, "BIGINT") && strcasecmp(dataType, "DOUBLE") && strcasecmp(dataType, "TIMESTAMP") - && !regexMatch(dataType, "^(NCHAR|BINARY)(\\([1-9][0-9]*\\))?$", REG_ICASE | REG_EXTENDED) + && !regexMatch(dataType, + "^(NCHAR|BINARY)(\\([1-9][0-9]*\\))?$", + REG_ICASE | REG_EXTENDED) && strcasecmp(dataType, "UTINYINT") && strcasecmp(dataType, "USMALLINT") && strcasecmp(dataType, "UINT") @@ -1603,11 +1624,13 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->data_type[0] = TSDB_DATA_TYPE_FLOAT; } else if (0 == strcasecmp(dataType, "DOUBLE")) { arguments->data_type[0] = TSDB_DATA_TYPE_DOUBLE; - } else if (1 == regexMatch(dataType, "^BINARY(\\([1-9][0-9]*\\))?$", REG_ICASE | - REG_EXTENDED)) { + } else if (1 == regexMatch(dataType, + "^BINARY(\\([1-9][0-9]*\\))?$", + REG_ICASE | REG_EXTENDED)) { arguments->data_type[0] = TSDB_DATA_TYPE_BINARY; - } else if (1 == regexMatch(dataType, "^NCHAR(\\([1-9][0-9]*\\))?$", REG_ICASE | - REG_EXTENDED)) { + } else if (1 == regexMatch(dataType, + "^NCHAR(\\([1-9][0-9]*\\))?$", + REG_ICASE | REG_EXTENDED)) { arguments->data_type[0] = TSDB_DATA_TYPE_NCHAR; } else if (0 == strcasecmp(dataType, "BOOL")) { arguments->data_type[0] = TSDB_DATA_TYPE_BOOL; @@ -2663,117 +2686,127 @@ static int printfInsertMeta() { } } - printf(" super table count: \033[33m%"PRIu64"\033[0m\n", - g_Dbs.db[i].superTblCount); - for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { - printf(" super table[\033[33m%"PRIu64"\033[0m]:\n", j); - - printf(" stbName: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].stbName); - if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { - printf(" autoCreateTable: \033[33m%s\033[0m\n", "no"); - } else if (AUTO_CREATE_SUBTBL == - g_Dbs.db[i].superTbls[j].autoCreateTable) { - printf(" autoCreateTable: \033[33m%s\033[0m\n", "yes"); - } else { - printf(" autoCreateTable: \033[33m%s\033[0m\n", "error"); - } - - if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { - printf(" childTblExists: \033[33m%s\033[0m\n", "no"); - } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { - printf(" childTblExists: \033[33m%s\033[0m\n", "yes"); - } else { - printf(" childTblExists: \033[33m%s\033[0m\n", "error"); - } - - printf(" childTblCount: \033[33m%"PRId64"\033[0m\n", - g_Dbs.db[i].superTbls[j].childTblCount); - printf(" childTblPrefix: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].childTblPrefix); - printf(" dataSource: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].dataSource); - printf(" iface: \033[33m%s\033[0m\n", - (g_Dbs.db[i].superTbls[j].iface==TAOSC_IFACE)?"taosc": - (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest":"stmt"); - if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) { - printf(" childTblLimit: \033[33m%"PRId64"\033[0m\n", - g_Dbs.db[i].superTbls[j].childTblLimit); - } - if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) { - printf(" childTblOffset: \033[33m%"PRIu64"\033[0m\n", - g_Dbs.db[i].superTbls[j].childTblOffset); - } - printf(" insertRows: \033[33m%"PRId64"\033[0m\n", - g_Dbs.db[i].superTbls[j].insertRows); - /* - if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { - printf(" multiThreadWriteOneTbl: \033[33m no\033[0m\n"); - }else { - printf(" multiThreadWriteOneTbl: \033[33m yes\033[0m\n"); - } - */ - printf(" interlaceRows: \033[33m%u\033[0m\n", - g_Dbs.db[i].superTbls[j].interlaceRows); + if (g_args.use_metric) { + printf(" super table count: \033[33m%"PRIu64"\033[0m\n", + g_Dbs.db[i].superTblCount); + for (uint64_t j = 0; j < g_Dbs.db[i].superTblCount; j++) { + printf(" super table[\033[33m%"PRIu64"\033[0m]:\n", j); - if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) { - printf(" stable insert interval: \033[33m%"PRIu64"\033[0m\n", - g_Dbs.db[i].superTbls[j].insertInterval); - } + printf(" stbName: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].stbName); - printf(" disorderRange: \033[33m%d\033[0m\n", - g_Dbs.db[i].superTbls[j].disorderRange); - printf(" disorderRatio: \033[33m%d\033[0m\n", - g_Dbs.db[i].superTbls[j].disorderRatio); - printf(" maxSqlLen: \033[33m%"PRIu64"\033[0m\n", - g_Dbs.db[i].superTbls[j].maxSqlLen); - printf(" timeStampStep: \033[33m%"PRId64"\033[0m\n", - g_Dbs.db[i].superTbls[j].timeStampStep); - printf(" startTimestamp: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].startTimestamp); - printf(" sampleFormat: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].sampleFormat); - printf(" sampleFile: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].sampleFile); - printf(" tagsFile: \033[33m%s\033[0m\n", - g_Dbs.db[i].superTbls[j].tagsFile); - printf(" columnCount: \033[33m%d\033[0m\n ", - g_Dbs.db[i].superTbls[j].columnCount); - for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { - //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); - if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, - "binary", 6)) - || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, - "nchar", 5))) { - printf("column[%d]:\033[33m%s(%d)\033[0m ", k, - g_Dbs.db[i].superTbls[j].columns[k].dataType, - g_Dbs.db[i].superTbls[j].columns[k].dataLen); + if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "no"); + } else if (AUTO_CREATE_SUBTBL == + g_Dbs.db[i].superTbls[j].autoCreateTable) { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "yes"); } else { - printf("column[%d]:\033[33m%s\033[0m ", k, - g_Dbs.db[i].superTbls[j].columns[k].dataType); + printf(" autoCreateTable: \033[33m%s\033[0m\n", "error"); } - } - printf("\n"); - printf(" tagCount: \033[33m%d\033[0m\n ", - g_Dbs.db[i].superTbls[j].tagCount); - for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { - //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); - if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, - "binary", strlen("binary"))) - || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, - "nchar", strlen("nchar")))) { - printf("tag[%d]:\033[33m%s(%d)\033[0m ", k, - g_Dbs.db[i].superTbls[j].tags[k].dataType, - g_Dbs.db[i].superTbls[j].tags[k].dataLen); + if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + printf(" childTblExists: \033[33m%s\033[0m\n", "no"); + } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + printf(" childTblExists: \033[33m%s\033[0m\n", "yes"); } else { - printf("tag[%d]:\033[33m%s\033[0m ", k, - g_Dbs.db[i].superTbls[j].tags[k].dataType); + printf(" childTblExists: \033[33m%s\033[0m\n", "error"); + } + + printf(" childTblCount: \033[33m%"PRId64"\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblCount); + printf(" childTblPrefix: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblPrefix); + printf(" dataSource: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].dataSource); + printf(" iface: \033[33m%s\033[0m\n", + (g_Dbs.db[i].superTbls[j].iface==TAOSC_IFACE)?"taosc": + (g_Dbs.db[i].superTbls[j].iface==REST_IFACE)?"rest":"stmt"); + if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) { + printf(" childTblLimit: \033[33m%"PRId64"\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblLimit); + } + if (g_Dbs.db[i].superTbls[j].childTblOffset > 0) { + printf(" childTblOffset: \033[33m%"PRIu64"\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblOffset); + } + printf(" insertRows: \033[33m%"PRId64"\033[0m\n", + g_Dbs.db[i].superTbls[j].insertRows); + /* + if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { + printf(" multiThreadWriteOneTbl: \033[33m no\033[0m\n"); + }else { + printf(" multiThreadWriteOneTbl: \033[33m yes\033[0m\n"); + } + */ + printf(" interlaceRows: \033[33m%u\033[0m\n", + g_Dbs.db[i].superTbls[j].interlaceRows); + + if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) { + printf(" stable insert interval: \033[33m%"PRIu64"\033[0m\n", + g_Dbs.db[i].superTbls[j].insertInterval); + } + + printf(" disorderRange: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].disorderRange); + printf(" disorderRatio: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].disorderRatio); + printf(" maxSqlLen: \033[33m%"PRIu64"\033[0m\n", + g_Dbs.db[i].superTbls[j].maxSqlLen); + printf(" timeStampStep: \033[33m%"PRId64"\033[0m\n", + g_Dbs.db[i].superTbls[j].timeStampStep); + printf(" startTimestamp: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].startTimestamp); + printf(" sampleFormat: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].sampleFormat); + printf(" sampleFile: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].sampleFile); + printf(" tagsFile: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].tagsFile); + printf(" columnCount: \033[33m%d\033[0m\n ", + g_Dbs.db[i].superTbls[j].columnCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, + "binary", 6)) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, + "nchar", 5))) { + printf("column[%d]:\033[33m%s(%d)\033[0m ", k, + g_Dbs.db[i].superTbls[j].columns[k].dataType, + g_Dbs.db[i].superTbls[j].columns[k].dataLen); + } else { + printf("column[%d]:\033[33m%s\033[0m ", k, + g_Dbs.db[i].superTbls[j].columns[k].dataType); + } + } + printf("\n"); + + printf(" tagCount: \033[33m%d\033[0m\n ", + g_Dbs.db[i].superTbls[j].tagCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, + "binary", strlen("binary"))) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, + "nchar", strlen("nchar")))) { + printf("tag[%d]:\033[33m%s(%d)\033[0m ", k, + g_Dbs.db[i].superTbls[j].tags[k].dataType, + g_Dbs.db[i].superTbls[j].tags[k].dataLen); + } else { + printf("tag[%d]:\033[33m%s\033[0m ", k, + g_Dbs.db[i].superTbls[j].tags[k].dataType); + } } + printf("\n"); } - printf("\n"); + } else { + printf(" childTblCount: \033[33m%"PRId64"\033[0m\n", + g_args.ntables); + printf(" insertRows: \033[33m%"PRId64"\033[0m\n", + g_args.insertRows); } + + printf("\n"); } @@ -10270,7 +10303,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, &stbInfo->childTblName, &childTblCount, limit, offset); - ntables = childTblCount; // CBD + ntables = childTblCount; } else { ntables = g_args.ntables; tableFrom = 0; @@ -10289,8 +10322,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, b = ntables % threads; } - if ((stbInfo) - && (stbInfo->iface == REST_IFACE)) { + if (g_args.iface == REST_IFACE || ((stbInfo) && (stbInfo->iface == REST_IFACE))) { if (convertHostToServAddr( g_Dbs.host, g_Dbs.port, &(g_Dbs.serv_addr)) != 0) { ERROR_EXIT("convert host to server address"); @@ -11824,7 +11856,7 @@ static void setParaFromArg() { g_Dbs.db[0].superTbls[0].columns[i].data_type = data_type[i]; tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, dataType[i], min(DATATYPE_BUFF_LEN, strlen(dataType[i]) + 1)); - if (1 == regexMatch(dataType[i], "^(NCHAR|BINARY)(\\([1-9][0-9]*\\))$", REG_ICASE | + if (1 == regexMatch(dataType[i], "^(NCHAR|BINARY)(\\([1-9][0-9]*\\))$", REG_ICASE | REG_EXTENDED)) { sscanf(dataType[i], "%[^(](%[^)]", type, length); g_Dbs.db[0].superTbls[0].columns[i].dataLen = atoi(length); @@ -11833,7 +11865,7 @@ static void setParaFromArg() { } else { g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.binwidth; tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, - dataType[i], min(DATATYPE_BUFF_LEN, strlen(dataType[i]) + 1)); + dataType[i], min(DATATYPE_BUFF_LEN, strlen(dataType[i]) + 1)); } g_Dbs.db[0].superTbls[0].columnCount++; } diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 3bb62af6f63cf4c896b3bb2ffb375a15f74f85b6..b760c642e5f57baf872148d21dcf29ca85061927 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#include +#include #include #include #include @@ -26,6 +28,12 @@ #include "tsdb.h" #include "tutil.h" +#define AVRO_SUPPORT 0 + +#if AVRO_SUPPORT == 1 +#include +#endif + #define TSDB_SUPPORT_NANOSECOND 1 #define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255 @@ -38,8 +46,8 @@ static int converStringToReadable(char *str, int size, char *buf, int bufsize); static int convertNCharToReadable(char *str, int size, char *buf, int bufsize); -static void taosDumpCharset(FILE *fp); -static void taosLoadFileCharset(FILE *fp, char *fcharset); +static void dumpCharset(FILE *fp); +static void loadFileCharset(FILE *fp, char *fcharset); typedef struct { short bytes; @@ -128,19 +136,23 @@ enum _describe_table_index { TSDB_MAX_DESCRIBE_METRIC }; -#define COL_NOTE_LEN 128 +#define COL_NOTE_LEN 4 +#define COL_TYPEBUF_LEN 16 +#define COL_VALUEBUF_LEN 32 typedef struct { - char field[TSDB_COL_NAME_LEN + 1]; - char type[16]; + char field[TSDB_COL_NAME_LEN]; + char type[COL_TYPEBUF_LEN]; int length; char note[COL_NOTE_LEN]; -} SColDes; + char value[COL_VALUEBUF_LEN]; + char *var_value; +} ColDes; typedef struct { char name[TSDB_TABLE_NAME_LEN]; - SColDes cols[]; -} STableDef; + ColDes cols[]; +} TableDef; extern char version[]; @@ -159,7 +171,8 @@ typedef struct { } TableRecord; typedef struct { - bool isStable; + bool isStb; + bool belongStb; int64_t dumpNtbCount; TableRecord **dumpNtbInfos; TableRecord tableRecord; @@ -329,24 +342,21 @@ static resultStatistics g_resultStatistics = {0}; static FILE *g_fpOfResult = NULL; static int g_numOfCores = 1; -static int taosDumpOut(); -static int taosDumpIn(); -static void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, +static int dumpOut(); +static int dumpIn(); +static void dumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp); -//static int taosDumpDb(SDbInfo *dbInfo, FILE *fp, TAOS *taos); -static int dumpStable(char *table, FILE *fp, SDbInfo *dbInfo); -static int dumpCreateTableClause(STableDef *tableDes, int numOfCols, +static int dumpCreateTableClause(TableDef *tableDes, int numOfCols, FILE *fp, char* dbName); -static void taosDumpCreateMTableClause(STableDef *tableDes, char *stable, - int numOfCols, FILE *fp, char* dbName); -static int64_t taosDumpTable(char *tbName, char *stable, - FILE *fp, char* dbName, int precision); +static int getTableDes( + char* dbName, char *table, + TableDef *stableDes, bool isSuperTable); static int64_t dumpTableData(FILE *fp, char *tbName, char* dbName, int precision, char *jsonAvroSchema); static int checkParam(); -static void taosFreeDbInfos(); +static void freeDbInfos(); struct arguments g_args = { // connection option @@ -420,16 +430,16 @@ static void printVersion() { } } -UNUSED_FUNC void errorWrongValue(char *program, char *wrong_arg, char *wrong_value) +void errorWrongValue(char *program, char *wrong_arg, char *wrong_value) { fprintf(stderr, "%s %s: %s is an invalid value\n", program, wrong_arg, wrong_value); - fprintf(stderr, "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); + fprintf(stderr, "Try `taosdump --help' or `taosdump --usage' for more information.\n"); } static void errorUnrecognized(char *program, char *wrong_arg) { fprintf(stderr, "%s: unrecognized options '%s'\n", program, wrong_arg); - fprintf(stderr, "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); + fprintf(stderr, "Try `taosdump --help' or `taosdump --usage' for more information.\n"); } static void errorPrintReqArg(char *program, char *wrong_arg) @@ -438,7 +448,7 @@ static void errorPrintReqArg(char *program, char *wrong_arg) "%s: option requires an argument -- '%s'\n", program, wrong_arg); fprintf(stderr, - "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); + "Try `taosdump --help' or `taosdump --usage' for more information.\n"); } static void errorPrintReqArg2(char *program, char *wrong_arg) @@ -447,7 +457,7 @@ static void errorPrintReqArg2(char *program, char *wrong_arg) "%s: option requires a number argument '-%s'\n", program, wrong_arg); fprintf(stderr, - "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); + "Try `taosdump --help' or `taosdump --usage' for more information.\n"); } static void errorPrintReqArg3(char *program, char *wrong_arg) @@ -456,7 +466,7 @@ static void errorPrintReqArg3(char *program, char *wrong_arg) "%s: option '%s' requires an argument\n", program, wrong_arg); fprintf(stderr, - "Try `taosdemo --help' or `taosdemo --usage' for more information.\n"); + "Try `taosdump --help' or `taosdump --usage' for more information.\n"); } /* Parse a single option. */ @@ -483,7 +493,14 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { errorPrintReqArg2("taosdump", "P"); exit(EXIT_FAILURE); } - g_args.port = atoi(arg); + + uint64_t port = atoi(arg); + if (port > 65535) { + errorWrongValue("taosdump", "-P or --port", arg); + exit(EXIT_FAILURE); + } + g_args.port = (uint16_t)port; + break; case 'q': g_args.mysqlFlag = atoi(arg); @@ -606,6 +623,17 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { return 0; } +static void freeTbDes(TableDef *tableDes) +{ + for (int i = 0; i < TSDB_MAX_COLUMNS; i ++) { + if (tableDes->cols[i].var_value) { + free(tableDes->cols[i].var_value); + } + } + + free(tableDes); +} + static int queryDbImpl(TAOS *taos, char *command) { TAOS_RES *res = NULL; int32_t code = -1; @@ -685,7 +713,7 @@ static void parse_args( exit(EXIT_FAILURE); } g_args.databases = true; - } else if (0 == strncmp(argv[i], "--version", strlen("--version")) || + } else if (0 == strncmp(argv[i], "--version", strlen("--version")) || 0 == strncmp(argv[i], "-V", strlen("-V"))) { printVersion(); exit(EXIT_SUCCESS); @@ -764,7 +792,7 @@ static int getPrecisionByString(char *precision) return -1; } -static void taosFreeDbInfos() { +static void freeDbInfos() { if (g_dbInfos == NULL) return; for (int i = 0; i < g_args.dumpDbCount; i++) tfree(g_dbInfos[i]); @@ -815,15 +843,20 @@ static int getTableRecordInfo( while ((row = taos_fetch_row(result)) != NULL) { isSet = true; - pTableRecordInfo->isStable = false; + pTableRecordInfo->isStb = false; tstrncpy(pTableRecordInfo->tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], min(TSDB_TABLE_NAME_LEN, fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes + 1)); - tstrncpy(pTableRecordInfo->tableRecord.stable, - (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], - min(TSDB_TABLE_NAME_LEN, - fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes + 1)); + if (strlen((char *)row[TSDB_SHOW_TABLES_METRIC_INDEX]) > 0) { + pTableRecordInfo->belongStb = true; + tstrncpy(pTableRecordInfo->tableRecord.stable, + (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], + min(TSDB_TABLE_NAME_LEN, + fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes + 1)); + } else { + pTableRecordInfo->belongStb = false; + } break; } @@ -848,7 +881,7 @@ static int getTableRecordInfo( while ((row = taos_fetch_row(result)) != NULL) { isSet = true; - pTableRecordInfo->isStable = true; + pTableRecordInfo->isStb = true; tstrncpy(pTableRecordInfo->tableRecord.stable, table, TSDB_TABLE_NAME_LEN); break; @@ -885,7 +918,6 @@ static int inDatabasesSeq( dbname = strsep(&running, ","); } - } return -1; @@ -951,7 +983,213 @@ static int getDumpDbCount() return count; } -static int64_t dumpNormalTableWithoutStb(SDbInfo *dbInfo, char *ntbName) +static void dumpCreateMTableClause( + char* dbName, + char *stable, + TableDef *tableDes, + int numOfCols, + FILE *fp + ) { + int counter = 0; + int count_temp = 0; + + char* tmpBuf = (char *)malloc(COMMAND_SIZE); + if (tmpBuf == NULL) { + errorPrint("%s() LN%d, failed to allocate %d memory\n", + __func__, __LINE__, COMMAND_SIZE); + return; + } + + char *pstr = NULL; + pstr = tmpBuf; + + pstr += sprintf(tmpBuf, + "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", + dbName, tableDes->name, dbName, stable); + + for (; counter < numOfCols; counter++) { + if (tableDes->cols[counter].note[0] != '\0') break; + } + + assert(counter < numOfCols); + count_temp = counter; + + for (; counter < numOfCols; counter++) { + if (counter != count_temp) { + if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || + strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { + //pstr += sprintf(pstr, ", \'%s\'", tableDes->cols[counter].note); + if (tableDes->cols[counter].var_value) { + pstr += sprintf(pstr, ", %s", + tableDes->cols[counter].var_value); + } else { + pstr += sprintf(pstr, ", %s", tableDes->cols[counter].value); + } + } else { + pstr += sprintf(pstr, ", %s", tableDes->cols[counter].value); + } + } else { + if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || + strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { + //pstr += sprintf(pstr, "\'%s\'", tableDes->cols[counter].note); + if (tableDes->cols[counter].var_value) { + pstr += sprintf(pstr, "%s", tableDes->cols[counter].var_value); + } else { + pstr += sprintf(pstr, "%s", tableDes->cols[counter].value); + } + } else { + pstr += sprintf(pstr, "%s", tableDes->cols[counter].value); + } + /* pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); */ + } + + /* if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || strcasecmp(tableDes->cols[counter].type, "nchar") + * == 0) { */ + /* pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length); */ + /* } */ + } + + pstr += sprintf(pstr, ");"); + + fprintf(fp, "%s\n", tmpBuf); + free(tmpBuf); +} + +static int convertTbDesToAvroSchema( + char *dbName, char *tbName, TableDef *tableDes, int colCount, + char **avroSchema) +{ + errorPrint("%s() LN%d TODO: covert table schema to avro schema\n", + __func__, __LINE__); + // { + // "namesapce": "database name", + // "type": "record", + // "name": "table name", + // "fields": [ + // { + // "name": "col0 name", + // "type": "long" + // }, + // { + // "name": "col1 name", + // "type": ["int", "null"] + // }, + // { + // "name": "col2 name", + // "type": ["float", "null"] + // }, + // ... + // { + // "name": "coln name", + // "type": ["string", "null"] + // } + // ] + // } + *avroSchema = (char *)calloc(1, + 17 + TSDB_DB_NAME_LEN /* dbname section */ + + 17 /* type: record */ + + 11 + TSDB_TABLE_NAME_LEN /* tbname section */ + + 10 /* fields section */ + + (TSDB_COL_NAME_LEN + 11 + 16) * colCount + 4); /* fields section */ + if (*avroSchema == NULL) { + errorPrint("%s() LN%d, memory allocation failed!\n", __func__, __LINE__); + return -1; + } + + char *pstr = *avroSchema; + pstr += sprintf(pstr, + "{\"namespace\": \"%s\", \"type\": \"record\", \"name\": \"%s\", \"fields\": [", + dbName, tbName); + for (int i = 0; i < colCount; i ++) { + if (0 == i) { + pstr += sprintf(pstr, + "{\"name\": \"%s\", \"type\": \"%s\"", + tableDes->cols[i].field, "long"); + } else { + if (strcasecmp(tableDes->cols[i].type, "binary") == 0 || + strcasecmp(tableDes->cols[i].type, "nchar") == 0) { + pstr += sprintf(pstr, + "{\"name\": \"%s\", \"type\": [\"%s\", \"null\"]", + tableDes->cols[i].field, "string"); + } else { + pstr += sprintf(pstr, + "{\"name\": \"%s\", \"type\": [\"%s\", \"null\"]", + tableDes->cols[i].field, tableDes->cols[i].type); + } + } + if ((i != (colCount -1)) + && (strcmp(tableDes->cols[i + 1].note, "TAG") != 0)) { + pstr += sprintf(pstr, "},"); + } else { + pstr += sprintf(pstr, "}"); + break; + } + } + + pstr += sprintf(pstr, "]}"); + + debugPrint("%s() LN%d, avroSchema: %s\n", __func__, __LINE__, *avroSchema); + + return 0; +} + +static int64_t dumpNormalTable( + char *dbName, + char *stable, + char *tbName, + int precision, + FILE *fp + ) { + int colCount = 0; + + TableDef *tableDes = (TableDef *)calloc(1, sizeof(TableDef) + + sizeof(ColDes) * TSDB_MAX_COLUMNS); + + if (stable != NULL && stable[0] != '\0') { // dump table schema which is created by using super table + colCount = getTableDes(dbName, tbName, tableDes, false); + + if (colCount < 0) { + free(tableDes); + return -1; + } + + // create child-table using super-table + dumpCreateMTableClause(dbName, stable, tableDes, colCount, fp); + + } else { // dump table definition + colCount = getTableDes(dbName, tbName, tableDes, false); + + if (colCount < 0) { + free(tableDes); + return -1; + } + + // create normal-table or super-table + dumpCreateTableClause(tableDes, colCount, fp, dbName); + } + + char *jsonAvroSchema = NULL; + if (g_args.avro) { + if (0 != convertTbDesToAvroSchema( + dbName, tbName, tableDes, colCount, &jsonAvroSchema)) { + freeTbDes(tableDes); + return -1; + } + } + + free(tableDes); + + int64_t ret = 0; + if (!g_args.schemaonly) { + ret = dumpTableData(fp, tbName, dbName, precision, + jsonAvroSchema); + } + + return ret; +} + +static int64_t dumpNormalTableBelongStb( + SDbInfo *dbInfo, char *stbName, char *ntbName) { int64_t count = 0; @@ -973,21 +1211,47 @@ static int64_t dumpNormalTableWithoutStb(SDbInfo *dbInfo, char *ntbName) return -1; } - count = taosDumpTable(ntbName, NULL, - fp, dbInfo->name, getPrecisionByString(dbInfo->precision)); + count = dumpNormalTable( + dbInfo->name, + stbName, + ntbName, + getPrecisionByString(dbInfo->precision), + fp); fclose(fp); return count; } -static int64_t dumpNormalTable(FILE *fp, TAOS *taos, char *dbName, char *tbName, - char *stbName, - int precision) +static int64_t dumpNormalTableWithoutStb(SDbInfo *dbInfo, char *ntbName) { int64_t count = 0; - count = taosDumpTable(tbName, stbName, - fp, dbName, precision); + char tmpBuf[4096] = {0}; + FILE *fp = NULL; + + if (g_args.outpath[0] != 0) { + sprintf(tmpBuf, "%s/%s.%s.sql", + g_args.outpath, dbInfo->name, ntbName); + } else { + sprintf(tmpBuf, "%s.%s.sql", + dbInfo->name, ntbName); + } + + fp = fopen(tmpBuf, "w"); + if (fp == NULL) { + errorPrint("%s() LN%d, failed to open file %s\n", + __func__, __LINE__, tmpBuf); + return -1; + } + + count = dumpNormalTable( + dbInfo->name, + NULL, + ntbName, + getPrecisionByString(dbInfo->precision), + fp); + + fclose(fp); return count; } @@ -1021,12 +1285,12 @@ static void *dumpNtbOfDb(void *arg) { debugPrint("[%d] No.\t%"PRId64" table name: %s\n", pThreadInfo->threadIndex, i, ((TableInfo *)(g_tablesList + pThreadInfo->tableFrom+i))->name); - dumpNormalTable(fp, - pThreadInfo->taos, + dumpNormalTable( pThreadInfo->dbName, - ((TableInfo *)(g_tablesList + pThreadInfo->tableFrom+i))->name, ((TableInfo *)(g_tablesList + pThreadInfo->tableFrom+i))->stable, - pThreadInfo->precision); + ((TableInfo *)(g_tablesList + pThreadInfo->tableFrom+i))->name, + pThreadInfo->precision, + fp); } fclose(fp); @@ -1080,12 +1344,12 @@ static void *dumpNormalTablesOfStb(void *arg) { debugPrint("[%d] sub table %"PRId64": name: %s\n", pThreadInfo->threadIndex, i++, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX]); - dumpNormalTable(fp, - pThreadInfo->taos, + dumpNormalTable( pThreadInfo->dbName, - (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], pThreadInfo->stbName, - pThreadInfo->precision); + (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], + pThreadInfo->precision, + fp); } fclose(fp); @@ -1271,6 +1535,34 @@ static int64_t dumpNtbOfStbByThreads( return records; } +static int dumpStableClasuse(SDbInfo *dbInfo, char *stbName, FILE *fp) +{ + uint64_t sizeOfTableDes = + (uint64_t)(sizeof(TableDef) + sizeof(ColDes) * TSDB_MAX_COLUMNS); + + TableDef *tableDes = (TableDef *)calloc(1, sizeOfTableDes); + if (NULL == tableDes) { + errorPrint("%s() LN%d, failed to allocate %"PRIu64" memory\n", + __func__, __LINE__, sizeOfTableDes); + exit(-1); + } + + int colCount = getTableDes(dbInfo->name, + stbName, tableDes, true); + + if (colCount < 0) { + free(tableDes); + errorPrint("%s() LN%d, failed to get stable[%s] schema\n", + __func__, __LINE__, stbName); + exit(-1); + } + + dumpCreateTableClause(tableDes, colCount, fp, dbInfo->name); + free(tableDes); + + return 0; +} + static int64_t dumpCreateSTableClauseOfDb( SDbInfo *dbInfo, FILE *fp) { @@ -1300,7 +1592,7 @@ static int64_t dumpCreateSTableClauseOfDb( int64_t superTblCnt = 0; while ((row = taos_fetch_row(res)) != NULL) { - if (0 == dumpStable(row[TSDB_SHOW_TABLES_NAME_INDEX], fp, dbInfo)) { + if (0 == dumpStableClasuse(dbInfo, row[TSDB_SHOW_TABLES_NAME_INDEX], fp)) { superTblCnt ++; } } @@ -1382,7 +1674,7 @@ static int64_t dumpNTablesOfDb(SDbInfo *dbInfo) static int64_t dumpWholeDatabase(SDbInfo *dbInfo, FILE *fp) { - taosDumpCreateDbClause(dbInfo, g_args.with_property, fp); + dumpCreateDbClause(dbInfo, g_args.with_property, fp); fprintf(g_fpOfResult, "\n#### database: %s\n", dbInfo->name); @@ -1393,14 +1685,13 @@ static int64_t dumpWholeDatabase(SDbInfo *dbInfo, FILE *fp) return dumpNTablesOfDb(dbInfo); } -static int taosDumpOut() { +static int dumpOut() { TAOS *taos = NULL; TAOS_RES *result = NULL; TAOS_ROW row; FILE *fp = NULL; int32_t count = 0; - TableRecordInfo tableRecordInfo; char tmpBuf[4096] = {0}; if (g_args.outpath[0] != 0) { @@ -1446,7 +1737,7 @@ static int taosDumpOut() { /* --------------------------------- Main Code -------------------------------- */ /* if (g_args.databases || g_args.all_databases) { // dump part of databases or all databases */ /* */ - taosDumpCharset(fp); + dumpCharset(fp); sprintf(command, "show databases"); result = taos_query(taos, command); @@ -1568,11 +1859,13 @@ static int taosDumpOut() { g_totalDumpOutRows += records; } } else { - taosDumpCreateDbClause(g_dbInfos[0], g_args.with_property, fp); + dumpCreateDbClause(g_dbInfos[0], g_args.with_property, fp); } int superTblCnt = 0 ; for (int i = 1; g_args.arg_list[i]; i++) { + TableRecordInfo tableRecordInfo; + if (getTableRecordInfo(g_dbInfos[0]->name, g_args.arg_list[i], &tableRecordInfo) < 0) { @@ -1582,14 +1875,24 @@ static int taosDumpOut() { } int64_t records = 0; - if (tableRecordInfo.isStable) { // dump all table of this stable - int ret = dumpStable( + if (tableRecordInfo.isStb) { // dump all table of this stable + int ret = dumpStableClasuse( + g_dbInfos[0], tableRecordInfo.tableRecord.stable, - fp, g_dbInfos[0]); + fp); if (ret >= 0) { superTblCnt++; records = dumpNtbOfStbByThreads(g_dbInfos[0], g_args.arg_list[i]); } + } else if (tableRecordInfo.belongStb){ + dumpStableClasuse( + g_dbInfos[0], + tableRecordInfo.tableRecord.stable, + fp); + records = dumpNormalTableBelongStb( + g_dbInfos[0], + tableRecordInfo.tableRecord.stable, + g_args.arg_list[i]); } else { records = dumpNormalTableWithoutStb(g_dbInfos[0], g_args.arg_list[i]); } @@ -1604,7 +1907,7 @@ static int taosDumpOut() { /* Close the handle and return */ fclose(fp); taos_free_result(result); - taosFreeDbInfos(); + freeDbInfos(); fprintf(stderr, "dump out rows: %" PRId64 "\n", g_totalDumpOutRows); return 0; @@ -1612,14 +1915,14 @@ _exit_failure: fclose(fp); taos_close(taos); taos_free_result(result); - taosFreeDbInfos(); + freeDbInfos(); errorPrint("dump out rows: %" PRId64 "\n", g_totalDumpOutRows); return -1; } static int getTableDes( char* dbName, char *table, - STableDef *stableDes, bool isSuperTable) { + TableDef *tableDes, bool isSuperTable) { TAOS_ROW row = NULL; TAOS_RES* res = NULL; int colCount = 0; @@ -1648,22 +1951,21 @@ static int getTableDes( TAOS_FIELD *fields = taos_fetch_fields(res); - tstrncpy(stableDes->name, table, TSDB_TABLE_NAME_LEN); + tstrncpy(tableDes->name, table, TSDB_TABLE_NAME_LEN); while ((row = taos_fetch_row(res)) != NULL) { - tstrncpy(stableDes->cols[colCount].field, + tstrncpy(tableDes->cols[colCount].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], min(TSDB_COL_NAME_LEN + 1, fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes + 1)); - tstrncpy(stableDes->cols[colCount].type, + tstrncpy(tableDes->cols[colCount].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], min(16, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes + 1)); - stableDes->cols[colCount].length = + tableDes->cols[colCount].length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - tstrncpy(stableDes->cols[colCount].note, + tstrncpy(tableDes->cols[colCount].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], min(COL_NOTE_LEN, fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes + 1)); - colCount++; } @@ -1676,10 +1978,10 @@ static int getTableDes( // if child-table have tag, using select tagName from table to get tagValue for (int i = 0 ; i < colCount; i++) { - if (strcmp(stableDes->cols[i].note, "TAG") != 0) continue; + if (strcmp(tableDes->cols[i].note, "TAG") != 0) continue; sprintf(sqlstr, "select %s from %s.%s", - stableDes->cols[i].field, dbName, table); + tableDes->cols[i].field, dbName, table); res = taos_query(taos, sqlstr); code = taos_errno(res); @@ -1703,7 +2005,7 @@ static int getTableDes( } if (row[TSDB_SHOW_TABLES_NAME_INDEX] == NULL) { - sprintf(stableDes->cols[i].note, "%s", "NULL"); + sprintf(tableDes->cols[i].note, "%s", "NUL"); taos_free_result(res); res = NULL; taos_close(taos); @@ -1715,58 +2017,82 @@ static int getTableDes( //int32_t* length = taos_fetch_lengths(tmpResult); switch (fields[0].type) { case TSDB_DATA_TYPE_BOOL: - sprintf(stableDes->cols[i].note, "%d", + sprintf(tableDes->cols[i].value, "%d", ((((int32_t)(*((char *)row[TSDB_SHOW_TABLES_NAME_INDEX]))) == 1) ? 1 : 0)); break; case TSDB_DATA_TYPE_TINYINT: - sprintf(stableDes->cols[i].note, "%d", + sprintf(tableDes->cols[i].value, "%d", *((int8_t *)row[TSDB_SHOW_TABLES_NAME_INDEX])); break; case TSDB_DATA_TYPE_SMALLINT: - sprintf(stableDes->cols[i].note, "%d", *((int16_t *)row[TSDB_SHOW_TABLES_NAME_INDEX])); + sprintf(tableDes->cols[i].value, "%d", + *((int16_t *)row[TSDB_SHOW_TABLES_NAME_INDEX])); break; case TSDB_DATA_TYPE_INT: - sprintf(stableDes->cols[i].note, "%d", *((int32_t *)row[TSDB_SHOW_TABLES_NAME_INDEX])); + sprintf(tableDes->cols[i].value, "%d", + *((int32_t *)row[TSDB_SHOW_TABLES_NAME_INDEX])); break; case TSDB_DATA_TYPE_BIGINT: - sprintf(stableDes->cols[i].note, "%" PRId64 "", *((int64_t *)row[TSDB_SHOW_TABLES_NAME_INDEX])); + sprintf(tableDes->cols[i].value, "%" PRId64 "", + *((int64_t *)row[TSDB_SHOW_TABLES_NAME_INDEX])); break; case TSDB_DATA_TYPE_FLOAT: - sprintf(stableDes->cols[i].note, "%f", GET_FLOAT_VAL(row[TSDB_SHOW_TABLES_NAME_INDEX])); + sprintf(tableDes->cols[i].value, "%f", + GET_FLOAT_VAL(row[TSDB_SHOW_TABLES_NAME_INDEX])); break; case TSDB_DATA_TYPE_DOUBLE: - sprintf(stableDes->cols[i].note, "%f", GET_DOUBLE_VAL(row[TSDB_SHOW_TABLES_NAME_INDEX])); + sprintf(tableDes->cols[i].value, "%f", + GET_DOUBLE_VAL(row[TSDB_SHOW_TABLES_NAME_INDEX])); break; case TSDB_DATA_TYPE_BINARY: - { - memset(stableDes->cols[i].note, 0, sizeof(stableDes->cols[i].note)); - stableDes->cols[i].note[0] = '\''; - char tbuf[COL_NOTE_LEN]; - converStringToReadable((char *)row[TSDB_SHOW_TABLES_NAME_INDEX], length[0], tbuf, COL_NOTE_LEN); - char* pstr = stpcpy(&(stableDes->cols[i].note[1]), tbuf); - *(pstr++) = '\''; - break; + memset(tableDes->cols[i].value, 0, + sizeof(tableDes->cols[i].value)); + int len = strlen((char *)row[0]); + // FIXME for long value + if (len < (COL_VALUEBUF_LEN - 2)) { + tableDes->cols[i].value[0] = '\''; + converStringToReadable( + (char *)row[0], + length[0], + tableDes->cols[i].value + 1, + len); + tableDes->cols[i].value[len+1] = '\''; + } else { + tableDes->cols[i].var_value = calloc(1, len + 2); + if (tableDes->cols[i].var_value == NULL) { + errorPrint("%s() LN%d, memory alalocation failed!\n", + __func__, __LINE__); + taos_free_result(res); + return -1; + } + tableDes->cols[i].var_value[0] = '\''; + converStringToReadable((char *)row[0], + length[0], + (char *)(tableDes->cols[i].var_value + 1), len); + tableDes->cols[i].var_value[len+1] = '\''; } + break; + case TSDB_DATA_TYPE_NCHAR: { - memset(stableDes->cols[i].note, 0, sizeof(stableDes->cols[i].note)); + memset(tableDes->cols[i].value, 0, sizeof(tableDes->cols[i].note)); char tbuf[COL_NOTE_LEN-2]; // need reserve 2 bytes for ' ' convertNCharToReadable((char *)row[TSDB_SHOW_TABLES_NAME_INDEX], length[0], tbuf, COL_NOTE_LEN); - sprintf(stableDes->cols[i].note, "\'%s\'", tbuf); + sprintf(tableDes->cols[i].value, "\'%s\'", tbuf); break; } case TSDB_DATA_TYPE_TIMESTAMP: - sprintf(stableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[TSDB_SHOW_TABLES_NAME_INDEX]); + sprintf(tableDes->cols[i].value, "%" PRId64 "", *(int64_t *)row[TSDB_SHOW_TABLES_NAME_INDEX]); #if 0 if (!g_args.mysqlFlag) { - sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[TSDB_SHOW_TABLES_NAME_INDEX]); + sprintf(tableDes->cols[i].value, "%" PRId64 "", *(int64_t *)row[TSDB_SHOW_TABLES_NAME_INDEX]); } else { char buf[64] = "\0"; int64_t ts = *((int64_t *)row[TSDB_SHOW_TABLES_NAME_INDEX]); time_t tt = (time_t)(ts / 1000); struct tm *ptm = localtime(&tt); strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm); - sprintf(tableDes->cols[i].note, "\'%s.%03d\'", buf, (int)(ts % 1000)); + sprintf(tableDes->cols[i].value, "\'%s.%03d\'", buf, (int)(ts % 1000)); } #endif break; @@ -1781,74 +2107,7 @@ static int getTableDes( return colCount; } -static int convertSchemaToAvroSchema(STableDef *stableDes, char **avroSchema) -{ - errorPrint("%s() LN%d TODO: covert table schema to avro schema\n", - __func__, __LINE__); - return 0; -} - -static int64_t taosDumpTable( - char *tbName, char *stable, - FILE *fp, char* dbName, int precision) { - int colCount = 0; - - STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) - + sizeof(SColDes) * TSDB_MAX_COLUMNS); - - if (stable != NULL && stable[0] != '\0') { // dump table schema which is created by using super table - /* - colCount = getTableDes(stable, tableDes, taos); - - if (count < 0) { - free(tableDes); - return -1; - } - - dumpCreateTableClause(tableDes, count, fp); - - memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); - */ - - colCount = getTableDes(dbName, tbName, tableDes, false); - - if (colCount < 0) { - free(tableDes); - return -1; - } - - // create child-table using super-table - taosDumpCreateMTableClause(tableDes, stable, colCount, fp, dbName); - - } else { // dump table definition - colCount = getTableDes(dbName, tbName, tableDes, false); - - if (colCount < 0) { - free(tableDes); - return -1; - } - - // create normal-table or super-table - dumpCreateTableClause(tableDes, colCount, fp, dbName); - } - - char *jsonAvroSchema = NULL; - if (g_args.avro) { - convertSchemaToAvroSchema(tableDes, &jsonAvroSchema); - } - - free(tableDes); - - int64_t ret = 0; - if (!g_args.schemaonly) { - ret = dumpTableData(fp, tbName, dbName, precision, - jsonAvroSchema); - } - - return ret; -} - -static void taosDumpCreateDbClause( +static void dumpCreateDbClause( SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { char sqlstr[TSDB_MAX_SQL_LEN] = {0}; @@ -1870,35 +2129,7 @@ static void taosDumpCreateDbClause( fprintf(fp, "%s\n\n", sqlstr); } -static int dumpStable(char *stbName, FILE *fp, SDbInfo *dbInfo) -{ - uint64_t sizeOfTableDes = - (uint64_t)(sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); - - STableDef *stableDes = (STableDef *)calloc(1, sizeOfTableDes); - if (NULL == stableDes) { - errorPrint("%s() LN%d, failed to allocate %"PRIu64" memory\n", - __func__, __LINE__, sizeOfTableDes); - exit(-1); - } - - int colCount = getTableDes(dbInfo->name, - stbName, stableDes, true); - - if (colCount < 0) { - free(stableDes); - errorPrint("%s() LN%d, failed to get stable[%s] schema\n", - __func__, __LINE__, stbName); - exit(-1); - } - - dumpCreateTableClause(stableDes, colCount, fp, dbInfo->name); - free(stableDes); - - return 0; -} - -static int dumpCreateTableClause(STableDef *tableDes, int numOfCols, +static int dumpCreateTableClause(TableDef *tableDes, int numOfCols, FILE *fp, char* dbName) { int counter = 0; int count_temp = 0; @@ -1949,64 +2180,6 @@ static int dumpCreateTableClause(STableDef *tableDes, int numOfCols, return fprintf(fp, "%s\n\n", sqlstr); } -static void taosDumpCreateMTableClause(STableDef *tableDes, char *stable, - int numOfCols, FILE *fp, char* dbName) { - int counter = 0; - int count_temp = 0; - - char* tmpBuf = (char *)malloc(COMMAND_SIZE); - if (tmpBuf == NULL) { - errorPrint("%s() LN%d, failed to allocate %d memory\n", - __func__, __LINE__, COMMAND_SIZE); - return; - } - - char *pstr = NULL; - pstr = tmpBuf; - - pstr += sprintf(tmpBuf, - "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", - dbName, tableDes->name, dbName, stable); - - for (; counter < numOfCols; counter++) { - if (tableDes->cols[counter].note[0] != '\0') break; - } - - assert(counter < numOfCols); - count_temp = counter; - - for (; counter < numOfCols; counter++) { - if (counter != count_temp) { - if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || - strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { - //pstr += sprintf(pstr, ", \'%s\'", tableDes->cols[counter].note); - pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note); - } else { - pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note); - } - } else { - if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || - strcasecmp(tableDes->cols[counter].type, "nchar") == 0) { - //pstr += sprintf(pstr, "\'%s\'", tableDes->cols[counter].note); - pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); - } else { - pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); - } - /* pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); */ - } - - /* if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || strcasecmp(tableDes->cols[counter].type, "nchar") - * == 0) { */ - /* pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length); */ - /* } */ - } - - pstr += sprintf(pstr, ");"); - - fprintf(fp, "%s\n", tmpBuf); - free(tmpBuf); -} - static int writeSchemaToAvro(char *jsonAvroSchema) { errorPrint("%s() LN%d, TODO: implement write schema to avro", @@ -2103,10 +2276,7 @@ static int64_t writeResultToSql(TAOS_RES *res, FILE *fp, char *dbName, char *tbN case TSDB_DATA_TYPE_BINARY: { char tbuf[COMMAND_SIZE] = {0}; - //*(pstr++) = '\''; converStringToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); - //pstr = stpcpy(pstr, tbuf); - //*(pstr++) = '\''; curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); break; } @@ -2362,7 +2532,6 @@ static int converStringToReadable(char *str, int size, char *buf, int bufsize) { static int convertNCharToReadable(char *str, int size, char *buf, int bufsize) { char *pstr = str; char *pbuf = buf; - // TODO wchar_t wc; while (size > 0) { if (*pstr == '\0') break; @@ -2386,7 +2555,7 @@ static int convertNCharToReadable(char *str, int size, char *buf, int bufsize) { return 0; } -static void taosDumpCharset(FILE *fp) { +static void dumpCharset(FILE *fp) { char charsetline[256]; (void)fseek(fp, 0, SEEK_SET); @@ -2394,7 +2563,7 @@ static void taosDumpCharset(FILE *fp) { (void)fwrite(charsetline, strlen(charsetline), 1, fp); } -static void taosLoadFileCharset(FILE *fp, char *fcharset) { +static void loadFileCharset(FILE *fp, char *fcharset) { char * line = NULL; size_t line_size = 0; @@ -2526,7 +2695,7 @@ static void taosMallocDumpFiles() } } -static void taosFreeDumpFiles() +static void freeDumpFiles() { for (int i = 0; i < g_tsSqlFileNum; i++) { tfree(g_tsDumpInSqlFiles[i]); @@ -2594,7 +2763,7 @@ static FILE* taosOpenDumpInFile(char *fptr) { return f; } -static int taosDumpInOneFile(TAOS* taos, FILE* fp, char* fcharset, +static int dumpInOneFile(TAOS* taos, FILE* fp, char* fcharset, char* encode, char* fileName) { int read_len = 0; char * cmd = NULL; @@ -2651,7 +2820,7 @@ static int taosDumpInOneFile(TAOS* taos, FILE* fp, char* fcharset, return 0; } -static void* taosDumpInWorkThreadFp(void *arg) +static void* dumpInWorkThreadFp(void *arg) { threadInfo *pThread = (threadInfo*)arg; setThreadName("dumpInWorkThrd"); @@ -2665,14 +2834,14 @@ static void* taosDumpInWorkThreadFp(void *arg) } fprintf(stderr, ", Success Open input file: %s\n", SQLFileName); - taosDumpInOneFile(pThread->taos, fp, g_tsCharset, g_args.encode, SQLFileName); + dumpInOneFile(pThread->taos, fp, g_tsCharset, g_args.encode, SQLFileName); } } return NULL; } -static void taosStartDumpInWorkThreads() +static void startDumpInWorkThreads() { pthread_attr_t thattr; threadInfo *pThread; @@ -2704,7 +2873,7 @@ static void taosStartDumpInWorkThreads() pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pThread->threadID), &thattr, - taosDumpInWorkThreadFp, (void*)pThread) != 0) { + dumpInWorkThreadFp, (void*)pThread) != 0) { errorPrint("%s() LN%d, thread:%d failed to start\n", __func__, __LINE__, pThread->threadIndex); exit(0); @@ -2721,7 +2890,7 @@ static void taosStartDumpInWorkThreads() free(threadObj); } -static int taosDumpIn() { +static int dumpIn() { assert(g_args.isDumpIn); TAOS *taos = NULL; @@ -2750,19 +2919,19 @@ static int taosDumpIn() { } fprintf(stderr, "Success Open input file: %s\n", g_tsDbSqlFile); - taosLoadFileCharset(fp, g_tsCharset); + loadFileCharset(fp, g_tsCharset); - taosDumpInOneFile(taos, fp, g_tsCharset, g_args.encode, + dumpInOneFile(taos, fp, g_tsCharset, g_args.encode, g_tsDbSqlFile); } taos_close(taos); if (0 != tsSqlFileNumOfTbls) { - taosStartDumpInWorkThreads(); + startDumpInWorkThreads(); } - taosFreeDumpFiles(); + freeDumpFiles(); return 0; } @@ -2883,7 +3052,7 @@ int main(int argc, char *argv[]) { fprintf(g_fpOfResult, "# DumpIn start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - if (taosDumpIn() < 0) { + if (dumpIn() < 0) { ret = -1; } } else { @@ -2891,7 +3060,7 @@ int main(int argc, char *argv[]) { fprintf(g_fpOfResult, "# DumpOut start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - if (taosDumpOut() < 0) { + if (dumpOut() < 0) { ret = -1; } else { fprintf(g_fpOfResult, "\n============================== TOTAL STATISTICS ============================== \n"); diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index 320445f7f784884f8aa009e37182fc57a38bb96f..33f8c81a6d82f3573d15f74b7c24fe801d89a6a3 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -2,7 +2,11 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) PROJECT(TDengine) ADD_SUBDIRECTORY(monitor) -ADD_SUBDIRECTORY(http) + +IF (TD_BUILD_HTTP) + ADD_SUBDIRECTORY(http) +ENDIF () + IF (TD_LINUX AND TD_MQTT) ADD_SUBDIRECTORY(mqtt) -ENDIF () \ No newline at end of file +ENDIF () diff --git a/tests/connectorTest/C#Test/nanosupport/nanotest.cs b/tests/connectorTest/C#Test/nanosupport/nanotest.cs index b9eaefef8c740f8196a715282c8c28ffd79bbdac..666c731e3a88feb727ff9ea0f84111ae36bd211e 100644 --- a/tests/connectorTest/C#Test/nanosupport/nanotest.cs +++ b/tests/connectorTest/C#Test/nanosupport/nanotest.cs @@ -29,21 +29,7 @@ namespace TDengineDriver private string password="taosdata"; private short port = 0; - //sql parameters - private string dbName; - private string tbName; - private string precision; - - private bool isInsertData; - private bool isQueryData; - - private long tableCount; - private long totalRows; - private long batchRows; - private long beginTimestamp = 1551369600000L; - private IntPtr conn = IntPtr.Zero; - private long rowsInserted = 0; static void Main(string[] args) { @@ -73,15 +59,6 @@ namespace TDengineDriver tester.executeQuery("select * from tb;"); - // Console.WriteLine("expected is : {0}", width); - // tdSql.checkData(0,0,"2021-06-10 0:00:00.100000001"); - // tdSql.checkData(1,0,"2021-06-10 0:00:00.150000000"); - // tdSql.checkData(2,0,"2021-06-10 0:00:00.299999999"); - // tdSql.checkData(3,1,3); - // tdSql.checkData(4,1,5); - // tdSql.checkData(5,1,7); - // tdSql.checkRows(6); - tester.executeQuery("select count(*) from tb where ts > 1623254400100000000 and ts < 1623254400100000002;"); Console.WriteLine("expected is : 1 " ); tester.executeQuery("select count(*) from tb where ts > '2021-06-10 0:00:00.100000001' and ts < '2021-06-10 0:00:00.160000000';"); @@ -271,8 +248,8 @@ namespace TDengineDriver // tdSql.checkData(0,0,1623258000123456789); - Console.WriteLine("usdb" ); + tester.execute("drop database if exists usdb;"); tester.execute("create database usdb precision 'us';"); tester.execute("use usdb;"); tester.execute("create stable st (ts timestamp ,speed float ) tags(time timestamp ,id int);"); @@ -289,16 +266,12 @@ namespace TDengineDriver tester.execute("insert into tb1 using st tags('2021-06-10 0:00:00.123' , 1 ) values('2021-06-10T0:00:00.123+07:00' , 1.0);" ); tester.executeQuery("select first(*) from tb1;"); Console.WriteLine("expected is : 1623258000123 " ); - - - + tester.CloseConnection(); tester.cleanup(); - - } - public void InitTDengine() + public void InitTDengine() { TDengine.Options((int)TDengineInitOption.TDDB_OPTION_CONFIGDIR, this.configDir); TDengine.Options((int)TDengineInitOption.TDDB_OPTION_SHELL_ACTIVITY_TIMER, "60"); @@ -307,7 +280,7 @@ namespace TDengineDriver Console.WriteLine("get connection starting..."); } - public void ConnectTDengine() + public void ConnectTDengine() { string db = ""; this.conn = TDengine.Connect(this.host, this.user, this.password, db, this.port); @@ -321,13 +294,13 @@ namespace TDengineDriver Console.WriteLine("[ OK ] Connection established."); } } - //EXECUTE SQL - public void execute(string sql) + + //EXECUTE SQL + public void execute(string sql) { DateTime dt1 = DateTime.Now; IntPtr res = TDengine.Query(this.conn, sql.ToString()); DateTime dt2 = DateTime.Now; - TimeSpan span = dt2 - dt1; if ((res == IntPtr.Zero) || (TDengine.ErrorNo(res) != 0)) { @@ -345,7 +318,7 @@ namespace TDengineDriver TDengine.FreeResult(res); } //EXECUTE QUERY - public void executeQuery(string sql) + public void executeQuery(string sql) { DateTime dt1 = DateTime.Now; @@ -454,7 +427,7 @@ namespace TDengineDriver } - public void CloseConnection() + public void CloseConnection() { if (this.conn != IntPtr.Zero) { @@ -481,22 +454,16 @@ namespace TDengineDriver switch(psc) { case 0: - Console.WriteLine("db：[{0:G}]'s precision is {1:G}",this.dbName,"millisecond"); + Console.WriteLine("db：[{0:G}]'s precision is {1:G} millisecond"); break; case 1: - Console.WriteLine("db：[{0:G}]'s precision is {1:G}",this.dbName,"microsecond"); + Console.WriteLine("db：[{0:G}]'s precision is {1:G} microsecond"); break; case 2: - Console.WriteLine("db：[{0:G}]'s precision is {1:G}",this.dbName,"nanosecond"); + Console.WriteLine("db：[{0:G}]'s precision is {1:G} nanosecond"); break; } - - } - - // public void checkData(int x ,int y , long ts ){ - - // } - + } } } diff --git a/tests/gotest/batchtest.bat b/tests/gotest/batchtest.bat index 2a96ee31eb6211dbc5f300fbb2f3d62c03df3061..f9e6f83d50b1f1fa04cb18972376b3951447cc81 100755 --- a/tests/gotest/batchtest.bat +++ b/tests/gotest/batchtest.bat @@ -9,7 +9,7 @@ if "%severIp%"=="" (set severIp=127.0.0.1) if "%serverPort%"=="" (set serverPort=6030) go env -w GO111MODULE=on -go env -w GOPROXY=https://goproxy.io,direct +go env -w GOPROXY=https://goproxy.cn,direct cd case001 case001.bat %severIp% %serverPort% @@ -25,4 +25,4 @@ rem cd nanosupport rem nanoCase.bat :: cd nanosupport -:: nanoCase.bat \ No newline at end of file +:: nanoCase.bat diff --git a/tests/gotest/batchtest.sh b/tests/gotest/batchtest.sh index 503d77b226885b10e3874a3e0718789bed34b200..046249bcf7e8abab57d43b6b6e268361ccc1a695 100755 --- a/tests/gotest/batchtest.sh +++ b/tests/gotest/batchtest.sh @@ -14,7 +14,7 @@ if [ ! -n "$serverPort" ]; then fi go env -w GO111MODULE=on -go env -w GOPROXY=https://goproxy.io,direct +go env -w GOPROXY=https://goproxy.cn,direct bash ./case001/case001.sh $severIp $serverPort bash ./case002/case002.sh $severIp $serverPort diff --git a/tests/pytest/client/nettest.py b/tests/pytest/client/nettest.py new file mode 100644 index 0000000000000000000000000000000000000000..50bc5cd01489c35eead69537dac64af38ad365cf --- /dev/null +++ b/tests/pytest/client/nettest.py @@ -0,0 +1,57 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import taos +import subprocess + +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + try: + str1 = 'taos -n speed -P 6030 -N 1000 -l 100000 -S tcp' + result1 = subprocess.call(str1) + except Exception as result1: + if result1 == 1: + tdLog.exit("the shell 'taos -n speed -P 6030 -N 1000 -l 100000 -S tcp' is wrong") + + try: + str2 = 'taos -n speed -P 6030 -N 1000 -l 100000 -S udp' + result2 = subprocess.call(str2) + except Exception as result2: + if result2 == 1: + tdLog.exit("the shell 'taos -n speed -P 6030 -N 1000 -l 100000 -S udp' is wrong") + + try: + str3 = 'taos -n fqdn' + result3 = subprocess.call(str3) + except Exception as result3: + if result3 ==1: + tdLog.exit('the shell"taos -n fqdn" is wrong') + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index a7c375489aec74517f9bb92938b4179b6f9e5dc0..32551042a69f63a5fbf9eb84e26a1c4a8c6ce4b7 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -398,7 +398,12 @@ python3 ./test.py -f query/queryWildcardLength.py python3 ./test.py -f query/queryTbnameUpperLower.py python3 ./test.py -f query/query.py python3 ./test.py -f query/queryDiffColsOr.py + + +python3 ./test.py -f client/nettest.py + python3 ./test.py -f query/queryRegex.py + #======================p4-end=============== diff --git a/tests/pytest/insert/schemaChangeTest.py b/tests/pytest/insert/schemaChangeTest.py new file mode 100644 index 0000000000000000000000000000000000000000..a62a15bcc0f05bf0229d12698b01c7917f6b9d95 --- /dev/null +++ b/tests/pytest/insert/schemaChangeTest.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import multiprocessing as mp + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.ts = 1609430400000 + + def alterTableSchema(self): + conn1 = taos.connect(host="127.0.0.1", user="root", password="taosdata", config=tdDnodes.getSimCfgPath()) + c1 = conn1.cursor() + + c1.execute("use db") + c1.execute("alter table st drop column c2") + c1.execute("alter table st add column c2 double") + + tdLog.sleep(1) + c1.execute("select * from st") + for data in c1: + print("Process 1: c2 = %s" % data[2]) + + + def insertData(self): + conn2 = taos.connect(host="127.0.0.1", user="root", password="taosdata", config=tdDnodes.getSimCfgPath()) + c2 = conn2.cursor() + + tdLog.sleep(1) + c2.execute("use db") + c2.execute("insert into t1 values(%d, 2, 2.22)" % (self.ts + 1)) + + c2.execute("select * from st") + for data in c2: + print("Process 2: c2 = %f" % data[2]) + + def run(self): + tdSql.prepare() + tdSql.execute("create table st(ts timestamp, c1 int, c2 float) tags(t1 int)") + tdSql.execute("create table t1 using st tags(1)") + tdSql.execute("insert into t1 values(%d, 1, 1.11)" % self.ts) + p1 = mp.Process(target=self.alterTableSchema, args=()) + p2 = mp.Process(target=self.insertData, args=()) + p1.start() + p2.start() + + p1.join() + p2.join() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file