diff --git a/cmake/install.inc b/cmake/install.inc index 8418612d4c54d7f7643e190da0af81064197b5d1..55b3fa188ff2479b245213ef5dab15cfa2909f02 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.14-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.15-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index a1b6174de0d21f02fdde1e367bcb45901707c6ba..748a9b299626c285e5089af848bed9e0ad26e0a8 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -382,6 +382,7 @@ typedef struct SSqlObj { typedef struct SSqlStream { SSqlObj *pSql; + const char* dstTable; uint32_t streamId; char listed; bool isProject; @@ -408,6 +409,8 @@ typedef struct SSqlStream { struct SSqlStream *prev, *next; } SSqlStream; +void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable); + int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn); void tscInitMsgsFp(); diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index 18fc79c4748f84d4ec073e4bc678e3a067afb025..f813ff85d99e6642827a49defe6b96f29720fc57 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -262,6 +262,11 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { SSqlStream *pStream = pObj->streamList; while (pStream) { tstrncpy(pSdesc->sql, pStream->pSql->sqlstr, sizeof(pSdesc->sql)); + if (pStream->dstTable == NULL) { + pSdesc->dstTable[0] = 0; + } else { + tstrncpy(pSdesc->dstTable, pStream->dstTable, sizeof(pSdesc->dstTable)); + } pSdesc->streamId = htonl(pStream->streamId); pSdesc->num = htobe64(pStream->num); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 91b00e0109b0551b3bde52e326966ff396921019..dd336bcf051ea28f172ad31beb10eee09c0f2549 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3282,7 +3282,12 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0); if (pColFilter->filterstr) { - if (pExpr->nSQLOptr != TK_EQ && pExpr->nSQLOptr != TK_NE && pExpr->nSQLOptr != TK_LIKE) { + if (pExpr->nSQLOptr != TK_EQ + && pExpr->nSQLOptr != TK_NE + && pExpr->nSQLOptr != TK_ISNULL + && pExpr->nSQLOptr != TK_NOTNULL + && pExpr->nSQLOptr != TK_LIKE + ) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } } else { diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 68c3bcae165050863cc4bf9c92a1510581531c3a..74b8e4d95878d8a784d4d82e28028e94bd32207b 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -535,6 +535,10 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) { pStream, pTableMetaInfo->name, pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr); } +void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable) { + pStream->dstTable = dstTable; +} + TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), int64_t stime, void *param, void (*callback)(void *)) { STscObj *pObj = (STscObj *)taos; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index a912cdfd7f531a6d863323056b1d9f78857a9284..4b5392257ffa072c53c4332733518c8f1f68f8bd 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -203,10 +203,10 @@ int32_t tsVersion = 0; // log int32_t tsNumOfLogLines = 10000000; -int32_t mDebugFlag = 135; -int32_t sdbDebugFlag = 135; +int32_t mDebugFlag = 131; +int32_t sdbDebugFlag = 131; int32_t dDebugFlag = 135; -int32_t vDebugFlag = 135; +int32_t vDebugFlag = 131; int32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; int32_t odbcDebugFlag = 131; @@ -220,7 +220,7 @@ int32_t debugFlag = 0; int32_t sDebugFlag = 135; int32_t wDebugFlag = 135; int32_t tsdbDebugFlag = 131; -int32_t cqDebugFlag = 135; +int32_t cqDebugFlag = 131; int32_t (*monStartSystemFp)() = NULL; void (*monStopSystemFp)() = NULL; @@ -416,7 +416,7 @@ static void doInitGlobalConfig(void) { cfg.option = "arbitrator"; cfg.ptr = tsArbitrator; cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; cfg.ptrLength = TSDB_EP_LEN; @@ -901,7 +901,7 @@ static void doInitGlobalConfig(void) { cfg.option = "timezone"; cfg.ptr = tsTimezone; cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; cfg.ptrLength = tListLen(tsTimezone); @@ -911,7 +911,7 @@ static void doInitGlobalConfig(void) { cfg.option = "locale"; cfg.ptr = tsLocale; cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; cfg.ptrLength = tListLen(tsLocale); @@ -921,7 +921,7 @@ static void doInitGlobalConfig(void) { cfg.option = "charset"; cfg.ptr = tsCharset; cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 0; cfg.maxValue = 0; cfg.ptrLength = tListLen(tsCharset); diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index 0eb3eb21cec7d79fc5fec5f7b0400a0a5a39a137..e289f1ae1b4823ad5c6fd580551a9ee3f9800d9e 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.14-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.15-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 4564bde81e450f9c44be52ef69918dc0b3ada26f..1a86bc57dcc86c45ef5deb05fc58b39ee0939059 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.14 + 2.0.15 jar JDBCDriver @@ -36,7 +36,6 @@ - commons-logging diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 7e087ebd9b05709518d45b8096cbd2b051c4a214..25a36e3a4822bfbea0c7f07ff3a4a09032d44542 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.14 + 2.0.15 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index cdd88b825e2a8f6ba81c131bfd1214ffbe46f32c..cd2a768a38c377916004875f3f6f143b555a0294 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -81,7 +81,7 @@ public class TSDBStatement implements Statement { } if (!this.connector.isUpdateQuery(pSql)) { - TSDBResultSet res = new TSDBResultSet(this.connector, resultSetPointer); + TSDBResultSet res = new TSDBResultSet(this.connector, resultSetPointer); res.setBatchFetch(this.connection.getBatchFetch()); return res; } else { @@ -125,7 +125,8 @@ public class TSDBStatement implements Statement { } public int getMaxFieldSize() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + return 0; +// throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } public void setMaxFieldSize(int max) throws SQLException { @@ -218,7 +219,8 @@ public class TSDBStatement implements Statement { } public int getFetchDirection() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + return ResultSet.FETCH_FORWARD; +// throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } /* diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index efb8795962e769dd0b11b8d313177855c994f52d..3968d5b8c9dfbb53ad04179e0451fc9aab76980e 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -57,6 +57,7 @@ typedef struct SCqObj { uint64_t uid; int32_t tid; // table ID int32_t rowSize; // bytes of a row + char * dstTable; char * sqlStr; // SQL string STSchema * pSchema; // pointer to schema array void * pStream; @@ -185,7 +186,7 @@ void cqStop(void *handle) { pthread_mutex_unlock(&pContext->mutex); } -void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) { +void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema) { if (tsEnableStream == 0) { return NULL; } @@ -195,9 +196,11 @@ void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema * if (pObj == NULL) return NULL; pObj->uid = uid; - pObj->tid = tid; - pObj->sqlStr = malloc(strlen(sqlStr)+1); - strcpy(pObj->sqlStr, sqlStr); + pObj->tid = sid; + if (dstTable != NULL) { + pObj->dstTable = strdup(dstTable); + } + pObj->sqlStr = strdup(sqlStr); pObj->pSchema = tdDupSchema(pSchema); pObj->rowSize = schemaTLen(pSchema); @@ -247,6 +250,7 @@ void cqDrop(void *handle) { cInfo("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr); tdFreeSchema(pObj->pSchema); + free(pObj->dstTable); free(pObj->sqlStr); free(pObj); @@ -292,6 +296,7 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { if (pObj->pStream == NULL) { pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL); if (pObj->pStream) { + tscSetStreamDestTable(pObj->pStream, pObj->dstTable); pContext->num++; cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr); } else { diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c index 41380f0d86ed31b2466596993b8ce1a1d435f8cf..f378835f0a87f9dd3539d92ac052c81cabcc165d 100644 --- a/src/cq/test/cqtest.c +++ b/src/cq/test/cqtest.c @@ -70,7 +70,7 @@ int main(int argc, char *argv[]) { tdDestroyTSchemaBuilder(&schemaBuilder); for (int sid =1; sid<10; ++sid) { - cqCreate(pCq, sid, sid, "select avg(speed) from demo.t1 sliding(1s) interval(5s)", pSchema); + cqCreate(pCq, sid, sid, NULL, "select avg(speed) from demo.t1 sliding(1s) interval(5s)", pSchema); } tdFreeSchema(pSchema); diff --git a/src/dnode/inc/dnodeInt.h b/src/dnode/inc/dnodeInt.h index 7595f5fd02fde85e64cf47d283d3e167cfa751d1..1327cd4433fd2e2157becaaf5cb52e2ca0ffe6ef 100644 --- a/src/dnode/inc/dnodeInt.h +++ b/src/dnode/inc/dnodeInt.h @@ -36,6 +36,14 @@ extern int32_t dDebugFlag; #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }} +typedef enum { + TSDB_RUN_STATUS_INITIALIZE, + TSDB_RUN_STATUS_RUNING, + TSDB_RUN_STATUS_STOPPED +} SRunStatus; + +SRunStatus dnodeGetRunStatus(); + #ifdef __cplusplus } #endif diff --git a/src/dnode/inc/dnodeModule.h b/src/dnode/inc/dnodeModule.h index edcefbdd0c9d6ce7e9d5ae84f383f6fa15fc2417..e645784c8fd7aced259185f45402f9839842b9aa 100644 --- a/src/dnode/inc/dnodeModule.h +++ b/src/dnode/inc/dnodeModule.h @@ -22,8 +22,8 @@ extern "C" { #include "dnodeInt.h" int32_t dnodeInitModules(); -void dnodeStartModules(); void dnodeCleanupModules(); +bool dnodeStartMnode(SMInfos *pMinfos); void dnodeProcessModuleStatus(uint32_t moduleStatus); #ifdef __cplusplus diff --git a/src/dnode/inc/dnodeVnodes.h b/src/dnode/inc/dnodeVnodes.h index 1785ed3d0636b3db564041fc8487f73113781e7b..e60dd290ce478f214c26410194f345fd4840dfb8 100644 --- a/src/dnode/inc/dnodeVnodes.h +++ b/src/dnode/inc/dnodeVnodes.h @@ -23,8 +23,8 @@ extern "C" { int32_t dnodeInitVnodes(); void dnodeCleanupVnodes(); -int32_t dnodeInitTimer(); -void dnodeCleanupTimer(); +int32_t dnodeInitStatusTimer(); +void dnodeCleanupStatusTimer(); void dnodeSendStatusMsgToMnode(); #ifdef __cplusplus diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c index 09151533e29ed8c00a90c17c76b2b24fc5f0eb23..103710bf6f061951fd2d50b2cf362a1021b915c1 100644 --- a/src/dnode/src/dnodeEps.c +++ b/src/dnode/src/dnodeEps.c @@ -237,7 +237,7 @@ PRASE_EPS_OVER: dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort); #else if (dnodeCheckEpChanged(dnodeGetDnodeId(), tsLocalEp)) { - dError("dnode:%d, localEp is changed to %s in dnodeEps.json and need reconfigured", dnodeGetDnodeId(), tsLocalEp); + dError("dnode:%d, localEp is different from %s in dnodeEps.json and need reconfigured", dnodeGetDnodeId(), tsLocalEp); return -1; } #endif diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index bc387e21714a61021fb8140a080937c1eed9d75c..414b66653d123b785643cfdc96b429edbe8d58ad 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -121,7 +121,7 @@ void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) { dnodeSendRedirectMsg(pMsg, true); } else { SMnodeMsg *pWrite = mnodeCreateMsg(pMsg); - dDebug("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, + dTrace("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); } @@ -130,7 +130,7 @@ void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) { } static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) { - dDebug("msg:%p, app:%p type:%s is freed from mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, + dTrace("msg:%p, app:%p type:%s is freed from mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); mnodeCleanupMsg(pWrite); diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 79efe5aa0e5124be39a2430763dd7a43fcb075ce..730dcf3681b4c9c6c14056069c90f4dfe2006666 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -17,6 +17,7 @@ #include "os.h" #include "taos.h" #include "tnote.h" +#include "ttimer.h" #include "tconfig.h" #include "tfile.h" #include "twal.h" @@ -39,6 +40,7 @@ #include "dnodeShell.h" #include "dnodeTelemetry.h" +void *tsDnodeTmr = NULL; static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED; static int32_t dnodeInitStorage(); @@ -68,8 +70,8 @@ static SStep tsDnodeSteps[] = { {"dnode-server", dnodeInitServer, dnodeCleanupServer}, {"dnode-vnodes", dnodeInitVnodes, dnodeCleanupVnodes}, {"dnode-modules", dnodeInitModules, dnodeCleanupModules}, - {"dnode-tmr", dnodeInitTimer, dnodeCleanupTimer}, {"dnode-shell", dnodeInitShell, dnodeCleanupShell}, + {"dnode-statustmr", dnodeInitStatusTimer,dnodeCleanupStatusTimer}, {"dnode-telemetry", dnodeInitTelemetry, dnodeCleanupTelemetry}, }; @@ -91,6 +93,23 @@ static int32_t dnodeInitComponents() { return dnodeStepInit(tsDnodeSteps, stepSize); } +static int32_t dnodeInitTmr() { + tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM"); + if (tsDnodeTmr == NULL) { + dError("failed to init dnode timer"); + return -1; + } + + return 0; +} + +static void dnodeCleanupTmr() { + if (tsDnodeTmr != NULL) { + taosTmrCleanUp(tsDnodeTmr); + tsDnodeTmr = NULL; + } +} + int32_t dnodeInitSystem() { dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE); tscEmbedded = 1; @@ -100,6 +119,7 @@ int32_t dnodeInitSystem() { taosReadGlobalLogCfg(); taosSetCoreDump(); taosInitNotes(); + dnodeInitTmr(); signal(SIGPIPE, SIG_IGN); if (dnodeCreateDir(tsLogDir) < 0) { @@ -125,7 +145,6 @@ int32_t dnodeInitSystem() { return -1; } - dnodeStartModules(); dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING); dInfo("TDengine is initialized successfully"); @@ -136,6 +155,7 @@ int32_t dnodeInitSystem() { void dnodeCleanUpSystem() { if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) { dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED); + dnodeCleanupTmr(); dnodeCleanupComponents(); taos_cleanup(); taosCloseLog(); diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c index 9eb52cbf5aee83852e3f97a89ecf82f06f41f09e..62de85445c2e6756094dea1ac6e7e6a5880f7e18 100644 --- a/src/dnode/src/dnodeModule.c +++ b/src/dnode/src/dnodeModule.c @@ -97,6 +97,20 @@ void dnodeCleanupModules() { } } +static int32_t dnodeStartModules() { + for (EModuleType module = 1; module < TSDB_MOD_MAX; ++module) { + if (tsModule[module].enable && tsModule[module].startFp) { + int32_t code = (*tsModule[module].startFp)(); + if (code != 0) { + dError("failed to start module:%s, code:%d", tsModule[module].name, code); + return code; + } + } + } + + return 0; +} + int32_t dnodeInitModules() { dnodeAllocModules(); @@ -110,17 +124,7 @@ int32_t dnodeInitModules() { } dInfo("dnode modules is initialized"); - return 0; -} - -void dnodeStartModules() { - for (EModuleType module = 1; module < TSDB_MOD_MAX; ++module) { - if (tsModule[module].enable && tsModule[module].startFp) { - if ((*tsModule[module].startFp)() != 0) { - dError("failed to start module:%s", tsModule[module].name); - } - } - } + return dnodeStartModules(); } void dnodeProcessModuleStatus(uint32_t moduleStatus) { diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index bf74e14963e048c72cb57fdedfd788c92b90cbce..de0c360c88e8197aea47beee656c7ddb77a26d8e 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -96,7 +96,7 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { rspMsg.code = TSDB_CODE_APP_NOT_READY; rpcSendResponse(&rspMsg); rpcFreeCont(pMsg->pCont); - dDebug("RPC %p, msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); + dTrace("RPC %p, msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); return; } @@ -151,7 +151,7 @@ void dnodeCleanupClient() { static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { if (dnodeGetRunStatus() == TSDB_RUN_STATUS_STOPPED) { if (pMsg == NULL || pMsg->pCont == NULL) return; - dDebug("msg:%p is ignored since dnode is stopping", pMsg); + dTrace("msg:%p is ignored since dnode is stopping", pMsg); rpcFreeCont(pMsg->pCont); return; } diff --git a/src/dnode/src/dnodeVMgmt.c b/src/dnode/src/dnodeVMgmt.c index 87302026ecd4ba5e2192be3e7a50c58b5c906f6b..3c975f5cf9be7c7293ae08e101262b730e4a1817 100644 --- a/src/dnode/src/dnodeVMgmt.c +++ b/src/dnode/src/dnodeVMgmt.c @@ -126,14 +126,14 @@ static void *dnodeProcessMgmtQueue(void *param) { } pMsg = &pMgmt->rpcMsg; - dDebug("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[pMsg->msgType]); + dTrace("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[pMsg->msgType]); if (dnodeProcessMgmtMsgFp[pMsg->msgType]) { rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg); } else { rsp.code = TSDB_CODE_DND_MSG_NOT_PROCESSED; } - dDebug("msg:%p, is processed, code:0x%x", pMgmt, rsp.code); + dTrace("msg:%p, is processed, code:0x%x", pMgmt, rsp.code); if (rsp.code != TSDB_CODE_DND_ACTION_IN_PROGRESS) { rsp.handle = pMsg->handle; rsp.pCont = NULL; diff --git a/src/dnode/src/dnodeVnodes.c b/src/dnode/src/dnodeVnodes.c index ba7f7625fa7f71d4c2db5b5570c6d126899d9deb..85b997d94c885c131aa033a17b486262f17fc334 100644 --- a/src/dnode/src/dnodeVnodes.c +++ b/src/dnode/src/dnodeVnodes.c @@ -30,39 +30,28 @@ typedef struct { int32_t * vnodeList; } SOpenVnodeThread; -void * tsDnodeTmr = NULL; +extern void * tsDnodeTmr; static void * tsStatusTimer = NULL; static uint32_t tsRebootTime = 0; static void dnodeSendStatusMsg(void *handle, void *tmrId); static void dnodeProcessStatusRsp(SRpcMsg *pMsg); -int32_t dnodeInitTimer() { - tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM"); - if (tsDnodeTmr == NULL) { - dError("failed to init dnode timer"); - return -1; - } - +int32_t dnodeInitStatusTimer() { dnodeAddClientRspHandle(TSDB_MSG_TYPE_DM_STATUS_RSP, dnodeProcessStatusRsp); tsRebootTime = taosGetTimestampSec(); taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer); - dInfo("dnode timer is initialized"); + dInfo("dnode status timer is initialized"); return TSDB_CODE_SUCCESS; } -void dnodeCleanupTimer() { +void dnodeCleanupStatusTimer() { if (tsStatusTimer != NULL) { taosTmrStopA(&tsStatusTimer); tsStatusTimer = NULL; } - - if (tsDnodeTmr != NULL) { - taosTmrCleanUp(tsDnodeTmr); - tsDnodeTmr = NULL; - } } static int32_t dnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { diff --git a/src/inc/dnode.h b/src/inc/dnode.h index 9dd95e32d759166490230bc06beac6b652b4fe77..cb7e9f0b0d1267c7879a63ca1cd1a9f15883d996 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -29,13 +29,6 @@ typedef struct { int32_t httpReqNum; } SStatisInfo; -typedef enum { - TSDB_RUN_STATUS_INITIALIZE, - TSDB_RUN_STATUS_RUNING, - TSDB_RUN_STATUS_STOPPED -} SRunStatus; - -SRunStatus dnodeGetRunStatus(); SStatisInfo dnodeGetStatisInfo(); bool dnodeIsFirstDeploy(); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index ea9d608d92a3fb84ec447ce0ea00ca8ce388f8f4..a2d2e2ebbd9009918c2c0758db0daed41d45de61 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -787,6 +787,7 @@ typedef struct { typedef struct { char sql[TSDB_SHOW_SQL_LEN]; + char dstTable[TSDB_TABLE_NAME_LEN]; uint32_t streamId; int64_t num; // number of computing/cycles int64_t useconds; diff --git a/src/inc/tcq.h b/src/inc/tcq.h index afa744a9c4856521075fb9427d0ed1bb412aa7d5..ad123d4080e23d2c7933dcea2981fd2376b99afe 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -42,7 +42,7 @@ void cqStart(void *handle); void cqStop(void *handle); // cqCreate is called by TSDB to start an instance of CQ -void *cqCreate(void *handle, uint64_t uid, int32_t sid, char *sqlStr, STSchema *pSchema); +void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema); // cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate void cqDrop(void *handle); diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 1769bd6566bbc27d75bf6b830add8ef79356f111..04d6c7881556defb93b0b65cefff6f69b9bf7cbc 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -48,7 +48,7 @@ typedef struct { void *cqH; int (*notifyStatus)(void *, int status, int eno); int (*eventCallBack)(void *); - void *(*cqCreateFunc)(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema); + void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema); void (*cqDropFunc)(void *handle); } STsdbAppH; diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index ce986813918249ebe501e92d3af307a67c296907..a92831de25cad262365e2a95163d83aa8fc8355f 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -45,6 +45,10 @@ void printHelp() { printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server."); printf("%s%s\n", indent, "-t"); printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local."); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup."); + printf("%s%s\n", indent, "-l"); + printf("%s%s%s\n", indent, indent, "Packet length used for net test, default is 1000 bytes."); exit(EXIT_SUCCESS); } @@ -137,6 +141,24 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { exit(EXIT_FAILURE); } } + // For time zone + else if (strcmp(argv[i], "-n") == 0) { + if (i < argc - 1) { + arguments->netTestRole = argv[++i]; + } else { + fprintf(stderr, "option -n requires an argument\n"); + exit(EXIT_FAILURE); + } + } + // For time zone + else if (strcmp(argv[i], "-l") == 0) { + if (i < argc - 1) { + arguments->pktLen = atoi(argv[++i]); + } else { + fprintf(stderr, "option -l requires an argument\n"); + exit(EXIT_FAILURE); + } + } // For temperory command TODO else if (strcmp(argv[i], "--help") == 0) { printHelp(); diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index a7258c97241f1b0f629f168952a9d98d60d343ff..bdfea26294201e4ee8a96acc3ec0a8c17c524f13 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -64,7 +64,10 @@ enum _show_tables_index { TSDB_SHOW_TABLES_NAME_INDEX, TSDB_SHOW_TABLES_CREATED_TIME_INDEX, TSDB_SHOW_TABLES_COLUMNS_INDEX, - TSDB_SHOW_TABLES_METRIC_INDEX, + TSDB_SHOW_TABLES_METRIC_INDEX, + TSDB_SHOW_TABLES_UID_INDEX, + TSDB_SHOW_TABLES_TID_INDEX, + TSDB_SHOW_TABLES_VGID_INDEX, TSDB_MAX_SHOW_TABLES }; @@ -92,24 +95,27 @@ typedef struct { extern char version[]; typedef struct { - char name[TSDB_DB_NAME_LEN + 1]; - int32_t tables; + char name[TSDB_DB_NAME_LEN + 1]; + char create_time[32]; + int32_t ntables; int32_t vgroups; - int16_t replications; + int16_t replica; int16_t quorum; - int16_t daysPerFile; - int16_t daysToKeep; - int16_t daysToKeep1; - int16_t daysToKeep2; - int32_t cacheBlockSize; //MB - int32_t totalBlocks; - int32_t minRowsPerFileBlock; - int32_t maxRowsPerFileBlock; - int8_t walLevel; - int32_t fsyncPeriod; - int8_t compression; - int8_t precision; // time resolution + int16_t days; + char keeplist[32]; + //int16_t daysToKeep; + //int16_t daysToKeep1; + //int16_t daysToKeep2; + int32_t cache; //MB + int32_t blocks; + int32_t minrows; + int32_t maxrows; + int8_t wallevel; + int32_t fsync; + int8_t comp; + char precision[8]; // time resolution int8_t update; + char status[16]; } SDbInfo; typedef struct { @@ -128,8 +134,17 @@ typedef struct { int32_t totalThreads; char dbName[TSDB_TABLE_NAME_LEN + 1]; void *taosCon; + int64_t rowsOfDumpOut; + int64_t tablesOfDumpOut; } SThreadParaObj; +typedef struct { + int64_t totalRowsOfDumpOut; + int64_t totalChildTblsOfDumpOut; + int32_t totalSuperTblsOfDumpOut; + int32_t totalDatabasesOfDumpOut; +} resultStatistics; + static int64_t totalDumpOutRows = 0; SDbInfo **dbInfos = NULL; @@ -167,6 +182,7 @@ static struct argp_option options[] = { // input/output file {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, {"inpath", 'i', "INPATH", 0, "Input file path.", 1}, + {"resultFile", 'r', "RESULTFILE", 0, "DumpOut/In Result file path and name.", 1}, #ifdef _TD_POWER_ {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, #else @@ -200,6 +216,8 @@ struct arguments { // output file char outpath[TSDB_FILENAME_LEN+1]; char inpath[TSDB_FILENAME_LEN+1]; + // result file + char *resultFile; char *encode; // dump unit option bool all_databases; @@ -274,6 +292,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { tstrncpy(arguments->inpath, full_path.we_wordv[0], TSDB_FILENAME_LEN); wordfree(&full_path); break; + case 'r': + arguments->resultFile = arg; + break; case 'c': if (wordexp(arg, &full_path, 0) != 0) { fprintf(stderr, "Invalid path %s\n", arg); @@ -343,16 +364,18 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { /* Our argp parser. */ static struct argp argp = {options, parse_opt, args_doc, doc}; +static resultStatistics g_resultStatistics = {0}; +static FILE *g_fpOfResult = NULL; int taosDumpOut(struct arguments *arguments); int taosDumpIn(struct arguments *arguments); void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp); int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon); -int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon); -void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp); -void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp); -int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon); -int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon); +int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, char* dbName); +void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, char* dbName); +void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName); +int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName); +int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName); int taosCheckParam(struct arguments *arguments); void taosFreeDbInfos(); static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfThread, char *dbName); @@ -371,7 +394,8 @@ struct arguments tsArguments = { 0, // outpath and inpath "", - "", + "", + "./dump_result.txt", NULL, // dump unit option false, @@ -392,18 +416,34 @@ struct arguments tsArguments = { 0, false }; - -int queryDB(TAOS *taos, char *command) { - TAOS_RES *pSql = NULL; + +static int queryDbImpl(TAOS *taos, char *command) { + int i; + TAOS_RES *res = NULL; int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != res) { + taos_free_result(res); + res = NULL; + } - pSql = taos_query(taos, command); - code = taos_errno(pSql); - if (code) { - fprintf(stderr, "sql error: %s, reason:%s\n", command, taos_errstr(pSql)); - } - taos_free_result(pSql); - return code; + res = taos_query(taos, command); + code = taos_errno(res); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run <%s>, reason: %s\n", command, taos_errstr(res)); + taos_free_result(res); + //taos_close(taos); + return -1; + } + + taos_free_result(res); + return 0; } int main(int argc, char *argv[]) { @@ -430,6 +470,7 @@ int main(int argc, char *argv[]) { printf("mysqlFlag: %d\n", tsArguments.mysqlFlag); printf("outpath: %s\n", tsArguments.outpath); printf("inpath: %s\n", tsArguments.inpath); + printf("resultFile: %s\n", tsArguments.resultFile); printf("encode: %s\n", tsArguments.encode); printf("all_databases: %d\n", tsArguments.all_databases); printf("databases: %d\n", tsArguments.databases); @@ -459,13 +500,78 @@ int main(int argc, char *argv[]) { if (taosCheckParam(&tsArguments) < 0) { exit(EXIT_FAILURE); } + + g_fpOfResult = fopen(tsArguments.resultFile, "a"); + if (NULL == g_fpOfResult) { + fprintf(stderr, "Failed to open %s for save result\n", tsArguments.resultFile); + return 1; + }; + + fprintf(g_fpOfResult, "#############################################################################\n"); + fprintf(g_fpOfResult, "============================== arguments config =============================\n"); + { + fprintf(g_fpOfResult, "host: %s\n", tsArguments.host); + fprintf(g_fpOfResult, "user: %s\n", tsArguments.user); + fprintf(g_fpOfResult, "password: %s\n", tsArguments.password); + fprintf(g_fpOfResult, "port: %u\n", tsArguments.port); + fprintf(g_fpOfResult, "cversion: %s\n", tsArguments.cversion); + fprintf(g_fpOfResult, "mysqlFlag: %d\n", tsArguments.mysqlFlag); + fprintf(g_fpOfResult, "outpath: %s\n", tsArguments.outpath); + fprintf(g_fpOfResult, "inpath: %s\n", tsArguments.inpath); + fprintf(g_fpOfResult, "resultFile: %s\n", tsArguments.resultFile); + fprintf(g_fpOfResult, "encode: %s\n", tsArguments.encode); + fprintf(g_fpOfResult, "all_databases: %d\n", tsArguments.all_databases); + fprintf(g_fpOfResult, "databases: %d\n", tsArguments.databases); + fprintf(g_fpOfResult, "schemaonly: %d\n", tsArguments.schemaonly); + fprintf(g_fpOfResult, "with_property: %d\n", tsArguments.with_property); + fprintf(g_fpOfResult, "start_time: %" PRId64 "\n", tsArguments.start_time); + fprintf(g_fpOfResult, "end_time: %" PRId64 "\n", tsArguments.end_time); + fprintf(g_fpOfResult, "data_batch: %d\n", tsArguments.data_batch); + fprintf(g_fpOfResult, "max_sql_len: %d\n", tsArguments.max_sql_len); + fprintf(g_fpOfResult, "table_batch: %d\n", tsArguments.table_batch); + fprintf(g_fpOfResult, "thread_num: %d\n", tsArguments.thread_num); + fprintf(g_fpOfResult, "allow_sys: %d\n", tsArguments.allow_sys); + fprintf(g_fpOfResult, "abort: %d\n", tsArguments.abort); + fprintf(g_fpOfResult, "isDumpIn: %d\n", tsArguments.isDumpIn); + fprintf(g_fpOfResult, "arg_list_len: %d\n", tsArguments.arg_list_len); - if (tsArguments.isDumpIn) { - if (taosDumpIn(&tsArguments) < 0) return -1; + for (int32_t i = 0; i < tsArguments.arg_list_len; i++) { + fprintf(g_fpOfResult, "arg_list[%d]: %s\n", i, tsArguments.arg_list[i]); + } + } + + time_t tTime = time(NULL); + struct tm tm = *localtime(&tTime); + + if (tsArguments.isDumpIn) { + fprintf(g_fpOfResult, "============================== DUMP IN ============================== \n"); + fprintf(g_fpOfResult, "# DumpIn start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + if (taosDumpIn(&tsArguments) < 0) { + fprintf(g_fpOfResult, "\n"); + fclose(g_fpOfResult); + return -1; + } } else { - if (taosDumpOut(&tsArguments) < 0) return -1; + fprintf(g_fpOfResult, "============================== DUMP OUT ============================== \n"); + fprintf(g_fpOfResult, "# DumpOut start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + if (taosDumpOut(&tsArguments) < 0) { + fprintf(g_fpOfResult, "\n"); + fclose(g_fpOfResult); + return -1; + } + + fprintf(g_fpOfResult, "\n============================== TOTAL STATISTICS ============================== \n"); + fprintf(g_fpOfResult, "# total database count: %d\n", g_resultStatistics.totalDatabasesOfDumpOut); + fprintf(g_fpOfResult, "# total super table count: %d\n", g_resultStatistics.totalSuperTblsOfDumpOut); + fprintf(g_fpOfResult, "# total child table count: %"PRId64"\n", g_resultStatistics.totalChildTblsOfDumpOut); + fprintf(g_fpOfResult, "# total row count: %"PRId64"\n", g_resultStatistics.totalRowsOfDumpOut); } + fprintf(g_fpOfResult, "\n"); + fclose(g_fpOfResult); + return 0; } @@ -700,7 +806,7 @@ int taosDumpOut(struct arguments *arguments) { int32_t code = taos_errno(result); if (code != 0) { - fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(taos)); + fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(result)); goto _exit_failure; } @@ -736,27 +842,29 @@ int taosDumpOut(struct arguments *arguments) { } strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes); -#if 0 if (arguments->with_property) { - dbInfos[count]->tables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); + dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); - dbInfos[count]->replications = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); + dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); - dbInfos[count]->daysPerFile = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); - dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]); - dbInfos[count]->daysToKeep1; - dbInfos[count]->daysToKeep2; - dbInfos[count]->cacheBlockSize = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]); - dbInfos[count]->totalBlocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]); - dbInfos[count]->minRowsPerFileBlock = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]); - dbInfos[count]->maxRowsPerFileBlock = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]); - dbInfos[count]->walLevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]); - dbInfos[count]->fsyncPeriod = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]); - dbInfos[count]->compression = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); - dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]); + dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); + + strncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); + //dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]); + //dbInfos[count]->daysToKeep1; + //dbInfos[count]->daysToKeep2; + dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]); + dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]); + dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]); + dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]); + dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]); + dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]); + dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); + + strncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); + //dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]); dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); } -#endif count++; if (arguments->databases) { @@ -781,6 +889,8 @@ int taosDumpOut(struct arguments *arguments) { taosDumpDb(dbInfos[0], arguments, fp, taos); } else { // case: taosdump tablex tabley ... taosDumpCreateDbClause(dbInfos[0], arguments->with_property, fp); + fprintf(g_fpOfResult, "\n#### database: %s\n", dbInfos[0]->name); + g_resultStatistics.totalDatabasesOfDumpOut++; sprintf(command, "use %s", dbInfos[0]->name); @@ -796,6 +906,7 @@ int taosDumpOut(struct arguments *arguments) { int32_t totalNumOfThread = 1; // 0: all normal talbe into .tables.tmp.0 int normalTblFd = -1; int32_t retCode; + int superTblCnt = 0 ; for (int i = 1; arguments->arg_list[i]; i++) { if (taosGetTableRecordInfo(arguments->arg_list[i], &tableRecordInfo, taos) < 0) { fprintf(stderr, "input the invalide table %s\n", arguments->arg_list[i]); @@ -803,11 +914,17 @@ int taosDumpOut(struct arguments *arguments) { } if (tableRecordInfo.isMetric) { // dump all table of this metric - (void)taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos); + int ret = taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos, dbInfos[0]->name); + if (0 == ret) { + superTblCnt++; + } retCode = taosSaveTableOfMetricToTempFile(taos, tableRecordInfo.tableRecord.metric, arguments, &totalNumOfThread); } else { if (tableRecordInfo.tableRecord.metric[0] != '\0') { // dump this sub table and it's metric - (void)taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos); + int ret = taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos, dbInfos[0]->name); + if (0 == ret) { + superTblCnt++; + } } retCode = taosSaveAllNormalTableToTempFile(taos, tableRecordInfo.tableRecord.name, tableRecordInfo.tableRecord.metric, &normalTblFd); } @@ -819,6 +936,10 @@ int taosDumpOut(struct arguments *arguments) { goto _clean_tmp_file; } } + + // TODO: save dump super table into result_output.txt + fprintf(g_fpOfResult, "# super table counter: %d\n", superTblCnt); + g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; if (-1 != normalTblFd){ taosClose(normalTblFd); @@ -855,41 +976,27 @@ _exit_failure: return -1; } -int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSuperTable) { +int taosGetTableDes(char* dbName, char *table, STableDef *tableDes, TAOS* taosCon, bool isSuperTable) { TAOS_ROW row = NULL; - TAOS_RES *tmpResult = NULL; + TAOS_RES* res = NULL; int count = 0; - char* tempCommand = (char *)malloc(COMMAND_SIZE); - if (tempCommand == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return -1; - } - - char* tbuf = (char *)malloc(COMMAND_SIZE); - if (tbuf == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - free(tempCommand); - return -1; - } - - sprintf(tempCommand, "describe %s", table); + char sqlstr[COMMAND_SIZE]; + sprintf(sqlstr, "describe %s.%s;", dbName, table); - tmpResult = taos_query(taosCon, tempCommand); - int32_t code = taos_errno(tmpResult); + res = taos_query(taosCon, sqlstr); + int32_t code = taos_errno(res); if (code != 0) { - fprintf(stderr, "failed to run command %s\n", tempCommand); - free(tempCommand); - free(tbuf); - taos_free_result(tmpResult); + fprintf(stderr, "failed to run command <%s>, reason:%s\n", sqlstr, taos_errstr(res)); + taos_free_result(res); return -1; } - TAOS_FIELD *fields = taos_fetch_fields(tmpResult); + TAOS_FIELD *fields = taos_fetch_fields(res); tstrncpy(tableDes->name, table, TSDB_COL_NAME_LEN); - while ((row = taos_fetch_row(tmpResult)) != NULL) { + while ((row = taos_fetch_row(res)) != NULL) { strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); strncpy(tableDes->cols[count].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], @@ -901,12 +1008,10 @@ int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSupe count++; } - taos_free_result(tmpResult); - tmpResult = NULL; + taos_free_result(res); + res = NULL; if (isSuperTable) { - free(tempCommand); - free(tbuf); return count; } @@ -915,37 +1020,33 @@ int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSupe if (strcmp(tableDes->cols[i].note, "TAG") != 0) continue; - sprintf(tempCommand, "select %s from %s", tableDes->cols[i].field, table); + sprintf(sqlstr, "select %s from %s.%s", tableDes->cols[i].field, dbName, table); - tmpResult = taos_query(taosCon, tempCommand); - code = taos_errno(tmpResult); + res = taos_query(taosCon, sqlstr); + code = taos_errno(res); if (code != 0) { - fprintf(stderr, "failed to run command %s\n", tempCommand); - free(tempCommand); - free(tbuf); - taos_free_result(tmpResult); + fprintf(stderr, "failed to run command <%s>, reason:%s\n", sqlstr, taos_errstr(res)); + taos_free_result(res); return -1; } - fields = taos_fetch_fields(tmpResult); + fields = taos_fetch_fields(res); - row = taos_fetch_row(tmpResult); + row = taos_fetch_row(res); if (NULL == row) { - fprintf(stderr, " fetch failed to run command %s\n", tempCommand); - free(tempCommand); - free(tbuf); - taos_free_result(tmpResult); + fprintf(stderr, " fetch failed to run command <%s>, reason:%s\n", sqlstr, taos_errstr(res)); + taos_free_result(res); return -1; } if (row[0] == NULL) { sprintf(tableDes->cols[i].note, "%s", "NULL"); - taos_free_result(tmpResult); - tmpResult = NULL; + taos_free_result(res); + res = NULL; continue; } - int32_t* length = taos_fetch_lengths(tmpResult); + int32_t* length = taos_fetch_lengths(res); //int32_t* length = taos_fetch_lengths(tmpResult); switch (fields[0].type) { @@ -970,18 +1071,22 @@ int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSupe case TSDB_DATA_TYPE_DOUBLE: sprintf(tableDes->cols[i].note, "%f", GET_DOUBLE_VAL(row[0])); break; - case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_BINARY: { memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note)); tableDes->cols[i].note[0] = '\''; + char tbuf[COMMAND_SIZE]; converStringToReadable((char *)row[0], length[0], tbuf, COMMAND_SIZE); char* pstr = stpcpy(&(tableDes->cols[i].note[1]), tbuf); *(pstr++) = '\''; break; - case TSDB_DATA_TYPE_NCHAR: + } + case TSDB_DATA_TYPE_NCHAR: { memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note)); + char tbuf[COMMAND_SIZE]; convertNCharToReadable((char *)row[0], length[0], tbuf, COMMAND_SIZE); sprintf(tableDes->cols[i].note, "\'%s\'", tbuf); break; + } case TSDB_DATA_TYPE_TIMESTAMP: sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]); #if 0 @@ -1001,17 +1106,14 @@ int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSupe break; } - taos_free_result(tmpResult); - tmpResult = NULL; + taos_free_result(res); + res = NULL; } - free(tempCommand); - free(tbuf); - return count; } -int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon) { +int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName) { int count = 0; STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); @@ -1030,7 +1132,7 @@ int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FI memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); */ - count = taosGetTableDes(table, tableDes, taosCon, false); + count = taosGetTableDes(dbName, table, tableDes, taosCon, false); if (count < 0) { free(tableDes); @@ -1038,10 +1140,10 @@ int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FI } // create child-table using super-table - taosDumpCreateMTableClause(tableDes, metric, count, fp); + taosDumpCreateMTableClause(tableDes, metric, count, fp, dbName); } else { // dump table definition - count = taosGetTableDes(table, tableDes, taosCon, false); + count = taosGetTableDes(dbName, table, tableDes, taosCon, false); if (count < 0) { free(tableDes); @@ -1049,39 +1151,28 @@ int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FI } // create normal-table or super-table - taosDumpCreateTableClause(tableDes, count, fp); + taosDumpCreateTableClause(tableDes, count, fp, dbName); } free(tableDes); - return taosDumpTableData(fp, table, arguments, taosCon); + return taosDumpTableData(fp, table, arguments, taosCon, dbName); } void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { + char sqlstr[TSDB_MAX_SQL_LEN] = {0}; - char* tmpCommand = (char *)malloc(COMMAND_SIZE); - if (tmpCommand == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return; - } - - char *pstr = tmpCommand; - - pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s", dbInfo->name); + char *pstr = sqlstr; + pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s ", dbInfo->name); if (isDumpProperty) { - #if 0 pstr += sprintf(pstr, - "TABLES %d vgroups %d REPLICA %d quorum %d DAYS %d KEEP %d CACHE %d BLOCKS %d MINROWS %d MAXROWS %d WALLEVEL %d FYNC %d COMP %d PRECISION %s UPDATE %d", - dbInfo->tables, dbInfo->vgroups, dbInfo->replications, dbInfo->quorum, dbInfo->daysPerFile, dbInfo->daysToKeep, dbInfo->cacheBlockSize, - dbInfo->totalBlocks, dbInfo->minRowsPerFileBlock, dbInfo->maxRowsPerFileBlock, dbInfo->walLevel, dbInfo->fsyncPeriod, dbInfo->compression, - dbInfo->precision, dbInfo->update); - #endif + "TABLES %d VGROUPS %d REPLICA %d QUORUM %d DAYS %d KEEP %s CACHE %d BLOCKS %d MINROWS %d MAXROWS %d WALLEVEL %d FYNC %d COMP %d PRECISION '%s' UPDATE %d", + dbInfo->ntables, dbInfo->vgroups, dbInfo->replica, dbInfo->quorum, dbInfo->days, dbInfo->keeplist, dbInfo->cache, + dbInfo->blocks, dbInfo->minrows, dbInfo->maxrows, dbInfo->wallevel, dbInfo->fsync, dbInfo->comp, dbInfo->precision, dbInfo->update); } pstr += sprintf(pstr, ";"); - - fprintf(fp, "%s\n\n", tmpCommand); - free(tmpCommand); + fprintf(fp, "%s\n\n", sqlstr); } void* taosDumpOutWorkThreadFp(void *arg) @@ -1131,7 +1222,13 @@ void* taosDumpOutWorkThreadFp(void *arg) while (1) { ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); if (readLen <= 0) break; - taosDumpTable(tableRecord.name, tableRecord.metric, &tsArguments, fp, pThread->taosCon); + + int ret = taosDumpTable(tableRecord.name, tableRecord.metric, &tsArguments, fp, pThread->taosCon, pThread->dbName); + if (ret >= 0) { + // TODO: sum table count and table rows by self + pThread->tablesOfDumpOut++; + pThread->rowsOfDumpOut += ret; + } } taos_free_result(tmpResult); @@ -1147,13 +1244,15 @@ static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfTh SThreadParaObj *threadObj = (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); for (int t = 0; t < numOfThread; ++t) { SThreadParaObj *pThread = threadObj + t; + pThread->rowsOfDumpOut = 0; + pThread->tablesOfDumpOut = 0; pThread->threadIndex = t; pThread->totalThreads = numOfThread; tstrncpy(pThread->dbName, dbName, TSDB_TABLE_NAME_LEN); pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port); if (pThread->taosCon == NULL) { - fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, taos_errstr(pThread->taosCon)); + fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, reason:%s\n", pThread->threadIndex, taos_errstr(NULL)); exit(0); } @@ -1170,15 +1269,25 @@ static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t numOfTh pthread_join(threadObj[t].threadID, NULL); } + // TODO: sum all thread dump table count and rows of per table, then save into result_output.txt + int64_t totalRowsOfDumpOut = 0; + int64_t totalChildTblsOfDumpOut = 0; for (int32_t t = 0; t < numOfThread; ++t) { taos_close(threadObj[t].taosCon); + totalChildTblsOfDumpOut += threadObj[t].tablesOfDumpOut; + totalRowsOfDumpOut += threadObj[t].rowsOfDumpOut; } + + fprintf(g_fpOfResult, "# child table counter: %"PRId64"\n", totalChildTblsOfDumpOut); + fprintf(g_fpOfResult, "# row counter: %"PRId64"\n", totalRowsOfDumpOut); + g_resultStatistics.totalChildTblsOfDumpOut += totalChildTblsOfDumpOut; + g_resultStatistics.totalRowsOfDumpOut += totalRowsOfDumpOut; free(threadObj); } -int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon) { +int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, char* dbName) { int count = 0; STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); @@ -1187,15 +1296,15 @@ int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon) { exit(-1); } - count = taosGetTableDes(table, tableDes, taosCon, true); + count = taosGetTableDes(dbName, table, tableDes, taosCon, true); if (count < 0) { free(tableDes); - fprintf(stderr, "failed to get stable schema\n"); + fprintf(stderr, "failed to get stable[%s] schema\n", table); exit(-1); } - taosDumpCreateTableClause(tableDes, count, fp); + taosDumpCreateTableClause(tableDes, count, fp, dbName); free(tableDes); return 0; @@ -1207,38 +1316,19 @@ int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp) TAOS_ROW row; int fd = -1; STableRecord tableRecord; + char sqlstr[TSDB_MAX_SQL_LEN] = {0}; - char* tmpCommand = (char *)malloc(COMMAND_SIZE); - if (tmpCommand == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - exit(-1); - } - - sprintf(tmpCommand, "use %s", dbName); + sprintf(sqlstr, "show %s.stables", dbName); - TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand); - int32_t code = taos_errno(tmpResult); + TAOS_RES* res = taos_query(taosCon, sqlstr); + int32_t code = taos_errno(res); if (code != 0) { - fprintf(stderr, "invalid database %s, error: %s\n", dbName, taos_errstr(taosCon)); - free(tmpCommand); - taos_free_result(tmpResult); + fprintf(stderr, "failed to run command <%s>, reason: %s\n", sqlstr, taos_errstr(res)); + taos_free_result(res); exit(-1); } - - taos_free_result(tmpResult); - sprintf(tmpCommand, "show stables"); - - tmpResult = taos_query(taosCon, tmpCommand); - code = taos_errno(tmpResult); - if (code != 0) { - fprintf(stderr, "failed to run command %s, error: %s\n", tmpCommand, taos_errstr(taosCon)); - free(tmpCommand); - taos_free_result(tmpResult); - exit(-1); - } - - TAOS_FIELD *fields = taos_fetch_fields(tmpResult); + TAOS_FIELD *fields = taos_fetch_fields(res); char tmpFileName[TSDB_FILENAME_LEN + 1]; memset(tmpFileName, 0, TSDB_FILENAME_LEN); @@ -1246,32 +1336,38 @@ int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp) fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); if (fd == -1) { fprintf(stderr, "failed to open temp file: %s\n", tmpFileName); - taos_free_result(tmpResult); - free(tmpCommand); + taos_free_result(res); (void)remove(".stables.tmp"); exit(-1); } - while ((row = taos_fetch_row(tmpResult)) != NULL) { + while ((row = taos_fetch_row(res)) != NULL) { memset(&tableRecord, 0, sizeof(STableRecord)); strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); taosWrite(fd, &tableRecord, sizeof(STableRecord)); } - taos_free_result(tmpResult); + taos_free_result(res); (void)lseek(fd, 0, SEEK_SET); + int superTblCnt = 0; while (1) { ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); if (readLen <= 0) break; - (void)taosDumpStable(tableRecord.name, fp, taosCon); + int ret = taosDumpStable(tableRecord.name, fp, taosCon, dbName); + if (0 == ret) { + superTblCnt++; + } } + // TODO: save dump super table into result_output.txt + fprintf(g_fpOfResult, "# super table counter: %d\n", superTblCnt); + g_resultStatistics.totalSuperTblsOfDumpOut += superTblCnt; + close(fd); (void)remove(".stables.tmp"); - free(tmpCommand); return 0; } @@ -1282,58 +1378,43 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao STableRecord tableRecord; taosDumpCreateDbClause(dbInfo, arguments->with_property, fp); - - char* tmpCommand = (char *)malloc(COMMAND_SIZE); - if (tmpCommand == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return -1; - } - - sprintf(tmpCommand, "use %s", dbInfo->name); - TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand); - int32_t code = taos_errno(tmpResult); - if (code != 0) { - fprintf(stderr, "invalid database %s\n", dbInfo->name); - free(tmpCommand); - taos_free_result(tmpResult); - return -1; - } - taos_free_result(tmpResult); + fprintf(g_fpOfResult, "\n#### database: %s\n", dbInfo->name); + g_resultStatistics.totalDatabasesOfDumpOut++; + + char sqlstr[TSDB_MAX_SQL_LEN] = {0}; fprintf(fp, "USE %s;\n\n", dbInfo->name); (void)taosDumpCreateSuperTableClause(taosCon, dbInfo->name, fp); - sprintf(tmpCommand, "show tables"); + sprintf(sqlstr, "show %s.tables", dbInfo->name); - tmpResult = taos_query(taosCon, tmpCommand); - code = taos_errno(tmpResult); + TAOS_RES* res = taos_query(taosCon, sqlstr); + int code = taos_errno(res); if (code != 0) { - fprintf(stderr, "failed to run command %s\n", tmpCommand); - free(tmpCommand); - taos_free_result(tmpResult); + fprintf(stderr, "failed to run command <%s>, reason:%s\n", sqlstr, taos_errstr(res)); + taos_free_result(res); return -1; } - TAOS_FIELD *fields = taos_fetch_fields(tmpResult); + TAOS_FIELD *fields = taos_fetch_fields(res); int32_t numOfTable = 0; int32_t numOfThread = 0; char tmpFileName[TSDB_FILENAME_LEN + 1]; - while ((row = taos_fetch_row(tmpResult)) != NULL) { + while ((row = taos_fetch_row(res)) != NULL) { if (0 == numOfTable) { memset(tmpFileName, 0, TSDB_FILENAME_LEN); sprintf(tmpFileName, ".tables.tmp.%d", numOfThread); fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH); if (fd == -1) { fprintf(stderr, "failed to open temp file: %s\n", tmpFileName); - taos_free_result(tmpResult); + taos_free_result(res); for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) { sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); (void)remove(tmpFileName); } - free(tmpCommand); return -1; } @@ -1360,33 +1441,26 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao fd = -1; } - taos_free_result(tmpResult); + taos_free_result(res); // start multi threads to dumpout taosStartDumpOutWorkThreads(arguments, numOfThread, dbInfo->name); for (int loopCnt = 0; loopCnt < numOfThread; loopCnt++) { sprintf(tmpFileName, ".tables.tmp.%d", loopCnt); (void)remove(tmpFileName); - } - - free(tmpCommand); + } return 0; } -void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp) { +void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, char* dbName) { int counter = 0; int count_temp = 0; + char sqlstr[COMMAND_SIZE]; - char* tmpBuf = (char *)malloc(COMMAND_SIZE); - if (tmpBuf == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return; - } - - char* pstr = tmpBuf; + char* pstr = sqlstr; - pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s", tableDes->name); + pstr += sprintf(sqlstr, "CREATE TABLE IF NOT EXISTS %s.%s", dbName, tableDes->name); for (; counter < numOfCols; counter++) { if (tableDes->cols[counter].note[0] != '\0') break; @@ -1420,12 +1494,10 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp) { pstr += sprintf(pstr, ");"); - fprintf(fp, "%s\n", tmpBuf); - - free(tmpBuf); + fprintf(fp, "%s\n\n", sqlstr); } -void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp) { +void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName) { int counter = 0; int count_temp = 0; @@ -1438,7 +1510,7 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols char *pstr = NULL; pstr = tmpBuf; - pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s USING %s TAGS (", tableDes->name, metric); + pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", dbName, tableDes->name, dbName, metric); for (; counter < numOfCols; counter++) { if (tableDes->cols[counter].note[0] != '\0') break; @@ -1479,48 +1551,36 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols free(tmpBuf); } -int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon) { +int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName) { /* char temp[MAX_COMMAND_SIZE] = "\0"; */ int64_t totalRows = 0; int count = 0; char *pstr = NULL; TAOS_ROW row = NULL; int numFields = 0; - char *tbuf = NULL; - - char* tmpCommand = (char *)calloc(1, COMMAND_SIZE); - if (tmpCommand == NULL) { - fprintf(stderr, "failed to allocate memory\n"); - return -1; + + if (arguments->schemaonly) { + return 0; } int32_t sql_buf_len = arguments->max_sql_len; char* tmpBuffer = (char *)calloc(1, sql_buf_len + 128); if (tmpBuffer == NULL) { fprintf(stderr, "failed to allocate memory\n"); - free(tmpCommand); return -1; } pstr = tmpBuffer; - if (arguments->schemaonly) { - free(tmpCommand); - free(tmpBuffer); - return 0; - } + char sqlstr[1024] = {0}; + sprintf(sqlstr, + "select * from %s.%s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc;", + dbName, tbname, arguments->start_time, arguments->end_time); - sprintf(tmpCommand, - "select * from %s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc", - tbname, - arguments->start_time, - arguments->end_time); - - TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand); + TAOS_RES* tmpResult = taos_query(taosCon, sqlstr); int32_t code = taos_errno(tmpResult); if (code != 0) { - fprintf(stderr, "failed to run command %s, reason: %s\n", tmpCommand, taos_errstr(taosCon)); - free(tmpCommand); + fprintf(stderr, "failed to run command %s, reason: %s\n", sqlstr, taos_errstr(tmpResult)); free(tmpBuffer); taos_free_result(tmpResult); return -1; @@ -1529,14 +1589,6 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* numFields = taos_field_count(tmpResult); assert(numFields > 0); TAOS_FIELD *fields = taos_fetch_fields(tmpResult); - tbuf = (char *)malloc(COMMAND_SIZE); - if (tbuf == NULL) { - fprintf(stderr, "No enough memory\n"); - free(tmpCommand); - free(tmpBuffer); - taos_free_result(tmpResult); - return -1; - } int rowFlag = 0; int32_t curr_sqlstr_len = 0; @@ -1550,7 +1602,7 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* if (count == 0) { total_sqlstr_len = 0; - curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "INSERT INTO %s VALUES (", tbname); + curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "INSERT INTO %s.%s VALUES (", dbName, tbname); } else { if (arguments->mysqlFlag) { if (0 == rowFlag) { @@ -1594,17 +1646,21 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* case TSDB_DATA_TYPE_DOUBLE: curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%f", GET_DOUBLE_VAL(row[col])); break; - case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_BINARY: { + char tbuf[COMMAND_SIZE] = {0}; //*(pstr++) = '\''; converStringToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); //pstr = stpcpy(pstr, tbuf); //*(pstr++) = '\''; pstr += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); break; - case TSDB_DATA_TYPE_NCHAR: + } + case TSDB_DATA_TYPE_NCHAR: { + char tbuf[COMMAND_SIZE] = {0}; convertNCharToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE); pstr += sprintf(pstr + curr_sqlstr_len, "\'%s\'", tbuf); break; + } case TSDB_DATA_TYPE_TIMESTAMP: if (!arguments->mysqlFlag) { curr_sqlstr_len += sprintf(pstr + curr_sqlstr_len, "%" PRId64 "", *(int64_t *)row[col]); @@ -1638,19 +1694,12 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* //} } + fprintf(fp, "\n"); atomic_add_fetch_64(&totalDumpOutRows, totalRows); - - fprintf(fp, "\n"); - - if (tbuf) { - free(tbuf); - } taos_free_result(tmpResult); - tmpResult = NULL; - free(tmpCommand); free(tmpBuffer); - return 0; + return totalRows; } int taosCheckParam(struct arguments *arguments) { @@ -1986,159 +2035,6 @@ static FILE* taosOpenDumpInFile(char *fptr) { return f; } -int taosDumpInOneFile_old(TAOS * taos, FILE* fp, char* fcharset, char* encode) { - char *command = NULL; - char *lcommand = NULL; - int tsize = 0; - char *line = NULL; - _Bool isRun = true; - size_t line_size = 0; - char *pstr = NULL; - char *lstr = NULL; - size_t inbytesleft = 0; - size_t outbytesleft = COMMAND_SIZE; - char *tcommand = NULL; - char *charsetOfFile = NULL; - iconv_t cd = (iconv_t)(-1); - - command = (char *)malloc(COMMAND_SIZE); - lcommand = (char *)malloc(COMMAND_SIZE); - if (command == NULL || lcommand == NULL) { - fprintf(stderr, "failed to connect to allocate memory\n"); - goto _dumpin_exit_failure; - } - - // Resolve locale - if (*fcharset != '\0') { - charsetOfFile = fcharset; - } else { - charsetOfFile = encode; - } - - if (charsetOfFile != NULL && strcasecmp(tsCharset, charsetOfFile) != 0) { - cd = iconv_open(tsCharset, charsetOfFile); - if (cd == ((iconv_t)(-1))) { - fprintf(stderr, "Failed to open iconv handle\n"); - goto _dumpin_exit_failure; - } - } - - pstr = command; - int64_t linenu = 0; - while (1) { - ssize_t size = getline(&line, &line_size, fp); - linenu++; - if (size <= 0) break; - if (size == 1) { - if (pstr != command) { - inbytesleft = pstr - command; - memset(lcommand, 0, COMMAND_SIZE); - pstr = command; - lstr = lcommand; - outbytesleft = COMMAND_SIZE; - if (cd != ((iconv_t)(-1))) { - iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); - tcommand = lcommand; - } else { - tcommand = command; - } - - taosReplaceCtrlChar(tcommand); - - if (queryDB(taos, tcommand) != 0) { - fprintf(stderr, "error sql: linenu: %" PRId64 " failed\n", linenu); - exit(0); - } - - pstr = command; - pstr[0] = '\0'; - tsize = 0; - isRun = true; - } - - continue; - } - - /* if (line[0] == '-' && line[1] == '-') continue; */ - - line[size - 1] = 0; - - if (tsize + size - 1 > COMMAND_SIZE) { - fprintf(stderr, "command is too long\n"); - goto _dumpin_exit_failure; - } - - if (line[size - 2] == '\\') { - line[size - 2] = ' '; - isRun = false; - } else { - isRun = true; - } - - memcpy(pstr, line, size - 1); - pstr += (size - 1); - *pstr = '\0'; - - if (!isRun) continue; - - if (command != pstr && !isEmptyCommand(command)) { - inbytesleft = pstr - command; - memset(lcommand, 0, COMMAND_SIZE); - pstr = command; - lstr = lcommand; - outbytesleft = COMMAND_SIZE; - if (cd != ((iconv_t)(-1))) { - iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); - tcommand = lcommand; - } else { - tcommand = command; - } - taosReplaceCtrlChar(tcommand); - if (queryDB(taos, tcommand) != 0) { - fprintf(stderr, "error sql: linenu:%" PRId64 " failed\n", linenu); - exit(0); - } - } - - pstr = command; - *pstr = '\0'; - tsize = 0; - } - - if (pstr != command) { - inbytesleft = pstr - command; - memset(lcommand, 0, COMMAND_SIZE); - pstr = command; - lstr = lcommand; - outbytesleft = COMMAND_SIZE; - if (cd != ((iconv_t)(-1))) { - iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft); - tcommand = lcommand; - } else { - tcommand = command; - } - taosReplaceCtrlChar(lcommand); - if (queryDB(taos, tcommand) != 0) - fprintf(stderr, "error sql: linenu:%" PRId64 " failed \n", linenu); - } - - if (cd != ((iconv_t)(-1))) iconv_close(cd); - tfree(line); - tfree(command); - tfree(lcommand); - taos_close(taos); - fclose(fp); - return 0; - -_dumpin_exit_failure: - if (cd != ((iconv_t)(-1))) iconv_close(cd); - tfree(command); - tfree(lcommand); - taos_close(taos); - fclose(fp); - return -1; -} - int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, char* fileName) { int read_len = 0; char * cmd = NULL; @@ -2172,8 +2068,9 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c memcpy(cmd + cmd_len, line, read_len); cmd[read_len + cmd_len]= '\0'; - if (queryDB(taos, cmd)) { + if (queryDbImpl(taos, cmd)) { fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName); + fprintf(g_fpOfResult, "error sql: linenu:%d, file:%s\n", lineNo, fileName); } memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); @@ -2221,7 +2118,7 @@ static void taosStartDumpInWorkThreads(struct arguments *args) pThread->totalThreads = totalThreads; pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port); if (pThread->taosCon == NULL) { - fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, taos_errstr(pThread->taosCon)); + fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, reason:%s\n", pThread->threadIndex, taos_errstr(NULL)); exit(0); } diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 36b6ff7a59fc1f1c73e64c40ff3a5a1cef65ece6..3256d5cd59699139cf67cebcc41070ca99cffad6 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -450,6 +450,12 @@ static int32_t mnodeGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "dest table"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ip:port"); @@ -524,6 +530,10 @@ static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, v STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]); cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->dstTable, pShow->bytes[cols]); + cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port); STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index ff81c37de78b99af19a70b8cdb6882957b029972..6297bb21d0ba45ba0363c6d7de2865e62c535014 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -396,14 +396,15 @@ static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable) { atomic_add_fetch_32(&pStable->numOfTables, 1); if (pStable->vgHash == NULL) { - pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + mDebug("table:%s, create hash:%p", pStable->info.tableId, pStable->vgHash); } if (pStable->vgHash != NULL) { if (taosHashGet(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId)) == NULL) { taosHashPut(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId), &pCtable->vgId, sizeof(pCtable->vgId)); - mDebug("table:%s, vgId:%d is put into stable vgList, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId, - (int32_t)taosHashGetSize(pStable->vgHash)); + mDebug("table:%s, vgId:%d is put into stable hash:%p, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId, + pStable->vgHash, taosHashGetSize(pStable->vgHash)); } } } @@ -416,13 +417,14 @@ static void mnodeRemoveTableFromStable(SSTableObj *pStable, SCTableObj *pCtable) SVgObj *pVgroup = mnodeGetVgroup(pCtable->vgId); if (pVgroup == NULL) { taosHashRemove(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId)); - mDebug("table:%s, vgId:%d is remove from stable vgList, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId, - (int32_t)taosHashGetSize(pStable->vgHash)); + mDebug("table:%s, vgId:%d is remove from stable hash:%p sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId, + pStable->vgHash, taosHashGetSize(pStable->vgHash)); } mnodeDecVgroupRef(pVgroup); } static void mnodeDestroySuperTable(SSTableObj *pStable) { + mDebug("table:%s, is destroyed, stable hash:%p", pStable->info.tableId, pStable->vgHash); if (pStable->vgHash != NULL) { taosHashCleanup(pStable->vgHash); pStable->vgHash = NULL; @@ -464,6 +466,9 @@ static int32_t mnodeSuperTableActionUpdate(SSdbRow *pRow) { SSTableObj *pNew = pRow->pObj; SSTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId); if (pTable != NULL && pTable != pNew) { + mDebug("table:%s, will be updated, hash:%p sizeOfVgList:%d, new hash:%p sizeOfVgList:%d", pTable->info.tableId, + pTable->vgHash, taosHashGetSize(pTable->vgHash), pNew->vgHash, taosHashGetSize(pNew->vgHash)); + void *oldTableId = pTable->info.tableId; void *oldSchema = pTable->schema; void *oldVgHash = pTable->vgHash; @@ -479,6 +484,9 @@ static int32_t mnodeSuperTableActionUpdate(SSdbRow *pRow) { free(pNew); free(oldTableId); free(oldSchema); + + mDebug("table:%s, update finished, hash:%p sizeOfVgList:%d", pTable->info.tableId, pTable->vgHash, + taosHashGetSize(pTable->vgHash)); } mnodeDecTableRef(pTable); @@ -783,8 +791,8 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { if (pMsg->pTable->type == TSDB_SUPER_TABLE) { SSTableObj *pSTable = (SSTableObj *)pMsg->pTable; - mInfo("msg:%p, app:%p table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d", - pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId, pSTable->uid, pSTable->numOfTables, (int32_t)taosHashGetSize(pSTable->vgHash)); + mInfo("msg:%p, app:%p table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d", pMsg, + pMsg->rpcMsg.ahandle, pDrop->tableId, pSTable->uid, pSTable->numOfTables, taosHashGetSize(pSTable->vgHash)); return mnodeProcessDropSuperTableMsg(pMsg); } else { SCTableObj *pCTable = (SCTableObj *)pMsg->pTable; @@ -925,7 +933,10 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; SSTableObj *pStable = (SSTableObj *)pMsg->pTable; - if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) { + mInfo("msg:%p, app:%p stable:%s will be dropped, hash:%p sizeOfVgList:%d", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, pStable->vgHash, taosHashGetSize(pStable->vgHash)); + + if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) { int32_t *pVgId = taosHashIterate(pStable->vgHash, NULL); while (pVgId) { SVgObj *pVgroup = mnodeGetVgroup(*pVgId); @@ -938,8 +949,9 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { pDrop->uid = htobe64(pStable->uid); mnodeExtractTableName(pStable->info.tableId, pDrop->tableId); - mInfo("msg:%p, app:%p stable:%s, send drop stable msg to vgId:%d", pMsg, pMsg->rpcMsg.ahandle, - pStable->info.tableId, pVgroup->vgId); + mInfo("msg:%p, app:%p stable:%s, send drop stable msg to vgId:%d, hash:%p sizeOfVgList:%d", pMsg, + pMsg->rpcMsg.ahandle, pStable->info.tableId, pVgroup->vgId, pStable->vgHash, + taosHashGetSize(pStable->vgHash)); SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pVgroup); SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; dnodeSendMsgToDnode(&epSet, &rpcMsg); @@ -1482,8 +1494,8 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { pMsg->rpcRsp.rsp = pMeta; - mDebug("msg:%p, app:%p stable:%s, uid:%" PRIu64 " table meta is retrieved", pMsg, pMsg->rpcMsg.ahandle, - pTable->info.tableId, pTable->uid); + mDebug("msg:%p, app:%p stable:%s, uid:%" PRIu64 " table meta is retrieved, sizeOfVgList:%d numOfTables:%d", pMsg, + pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->uid, taosHashGetSize(pTable->vgHash), pTable->numOfTables); return TSDB_CODE_SUCCESS; } @@ -1512,7 +1524,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { char *msg = (char *)pRsp + sizeof(SSTableVgroupRspMsg); for (int32_t i = 0; i < numOfTable; ++i) { - char * stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; + char *stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; SSTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable == NULL) { mError("msg:%p, app:%p stable:%s, not exist while get stable vgroup info", pMsg, pMsg->rpcMsg.ahandle, stableName); @@ -1533,6 +1545,8 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { msg += sizeof(SVgroupsMsg); } else { SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg; + mDebug("msg:%p, app:%p stable:%s, hash:%p sizeOfVgList:%d will be returned", pMsg, pMsg->rpcMsg.ahandle, + pTable->info.tableId, pTable->vgHash, taosHashGetSize(pTable->vgHash)); int32_t *pVgId = taosHashIterate(pTable->vgHash, NULL); int32_t vgSize = 0; diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 22f464924e2c21cdeccb0c152457fad11a10294a..266228e7ac7df7641b9166e07ddba9bad7968461 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -79,7 +79,7 @@ bool httpInitContexts() { void httpCleanupContexts() { if (tsHttpServer.contextCache != NULL) { SCacheObj *cache = tsHttpServer.contextCache; - httpInfo("context cache is cleanuping, size:%" PRIzu "", taosHashGetSize(cache->pHashTable)); + httpInfo("context cache is cleanuping, size:%d", taosHashGetSize(cache->pHashTable)); taosCacheCleanup(tsHttpServer.contextCache); tsHttpServer.contextCache = NULL; } diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c index a96e4433b065885a18b8c0b4f35e2a1925125fc3..35ce0160b20b8cec0126f5f2415a878ccb072d0e 100644 --- a/src/plugins/http/src/httpSession.c +++ b/src/plugins/http/src/httpSession.c @@ -107,7 +107,7 @@ static void httpDestroySession(void *data) { void httpCleanUpSessions() { if (tsHttpServer.sessionCache != NULL) { SCacheObj *cache = tsHttpServer.sessionCache; - httpInfo("session cache is cleanuping, size:%" PRIzu "", taosHashGetSize(cache->pHashTable)); + httpInfo("session cache is cleanuping, size:%d", taosHashGetSize(cache->pHashTable)); taosCacheCleanup(tsHttpServer.sessionCache); tsHttpServer.sessionCache = NULL; } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 3990c0c516d1768135934b098efbcd7746ab424c..9d65325001a33dedbf04cf7d065d1d803eeb4e1c 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -872,7 +872,7 @@ static void tsdbStartStream(STsdbRepo *pRepo) { for (int i = 0; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable && pTable->type == TSDB_STREAM_TABLE) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql, + pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, tsdbGetTableSchemaImpl(pTable, false, false, -1)); } } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 25c815b74e3bd5593bef9157cbd42ba869298cab..9dfa147c8f35c20cea9b85e38e38a3db60f3572b 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -828,7 +828,7 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, boo if (lock && tsdbUnlockRepoMeta(pRepo) < 0) return -1; if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE && addIdx) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql, + pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, tsdbGetTableSchemaImpl(pTable, false, false, -1)); } diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index b0319d3e139f72bdaff7993e2e46d61b82c4d0a5..5bada93d1c77462fea0d16708fef2546976a8d99 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -82,7 +82,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp * @param pHashObj * @return */ -size_t taosHashGetSize(const SHashObj *pHashObj); +int32_t taosHashGetSize(const SHashObj *pHashObj); /** * put element into hash table, if the element with the same key exists, update it diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 0e3e0d3e2437f4595179c880a37642e047eb4331..7a835e87e753d98ec19809e4f3eec32bbe37e55a 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -189,7 +189,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp return pHashObj; } -size_t taosHashGetSize(const SHashObj *pHashObj) { return (pHashObj == NULL) ? 0 : pHashObj->size; } +int32_t taosHashGetSize(const SHashObj *pHashObj) { return (int32_t)((pHashObj == NULL) ? 0 : pHashObj->size); } int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 173de294cf333e20b9649a5634905421786a1b30..7c805072c1c94133cde863f03e4932e792d41c84 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -240,9 +240,6 @@ void taosReadGlobalLogCfg() { int olen, vlen; char fileName[PATH_MAX] = {0}; - mDebugFlag = 135; - sdbDebugFlag = 135; - wordexp_t full_path; if ( 0 != wordexp(configDir, &full_path, 0)) { printf("\nconfig file: %s wordexp fail! reason:%s\n", configDir, strerror(errno)); diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index c269d9a1ffe07c162123c44c397aba0da9593068..89601147a5923ac413800bdb6d134b266f0663b9 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -43,12 +43,13 @@ static void *taosNetBindUdpPort(void *sarg) { char buffer[BUFFER_SIZE]; int32_t iDataNum; socklen_t sin_size; + int32_t bufSize = 1024000; struct sockaddr_in server_addr; struct sockaddr_in clientAddr; if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { - uError("failed to create udp socket since %s", strerror(errno)); + uError("failed to create UDP socket since %s", strerror(errno)); return NULL; } @@ -58,11 +59,23 @@ static void *taosNetBindUdpPort(void *sarg) { server_addr.sin_addr.s_addr = htonl(INADDR_ANY); if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { - uError("failed to bind udp port:%d since %s", port, strerror(errno)); + uError("failed to bind UDP port:%d since %s", port, strerror(errno)); return NULL; } - uInfo("udp server at port:%d is listening", port); + if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + uError("failed to set the send buffer size for UDP socket\n"); + taosCloseSocket(serverSocket); + return NULL; + } + + if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + uError("failed to set the receive buffer size for UDP socket\n"); + taosCloseSocket(serverSocket); + return NULL; + } + + uInfo("UDP server at port:%d is listening", port); while (1) { memset(buffer, 0, BUFFER_SIZE); @@ -74,10 +87,13 @@ static void *taosNetBindUdpPort(void *sarg) { continue; } + uInfo("UDP: recv:%d bytes from %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); + if (iDataNum > 0) { - uInfo("UDP: recv:%d bytes from %s:%d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); - sendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size); + iDataNum = taosSendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size); } + + uInfo("UDP: send:%d bytes to %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); } taosCloseSocket(serverSocket); @@ -94,10 +110,9 @@ static void *taosNetBindTcpPort(void *sarg) { int32_t addr_len = sizeof(clientAddr); SOCKET client; char buffer[BUFFER_SIZE]; - int32_t iDataNum = 0; if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { - uError("failed to create tcp socket since %s", strerror(errno)); + uError("failed to create TCP socket since %s", strerror(errno)); return NULL; } @@ -106,130 +121,103 @@ static void *taosNetBindTcpPort(void *sarg) { server_addr.sin_port = htons(port); server_addr.sin_addr.s_addr = htonl(INADDR_ANY); + int32_t reuse = 1; + if (taosSetSockOpt(serverSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(serverSocket); + return NULL; + } + if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { - uError("failed to bind tcp port:%d since %s", port, strerror(errno)); + uError("failed to bind TCP port:%d since %s", port, strerror(errno)); return NULL; } - if (listen(serverSocket, 5) < 0) { - uError("failed to listen tcp port:%d since %s", port, strerror(errno)); + + if (taosKeepTcpAlive(serverSocket) < 0) { + uError("failed to set tcp server keep-alive option since %s", strerror(errno)); + taosCloseSocket(serverSocket); return NULL; } - uInfo("tcp server at port:%d is listening", port); + if (listen(serverSocket, 10) < 0) { + uError("failed to listen TCP port:%d since %s", port, strerror(errno)); + return NULL; + } + + uInfo("TCP server at port:%d is listening", port); while (1) { client = accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len); if (client < 0) { - uDebug("failed to accept from tcp port:%d since %s", port, strerror(errno)); + uDebug("TCP: failed to accept at port:%d since %s", port, strerror(errno)); continue; } - iDataNum = 0; - memset(buffer, 0, BUFFER_SIZE); - int32_t nleft, nread; - char * ptr = buffer; - nleft = pinfo->pktLen; - - while (nleft > 0) { - nread = recv(client, ptr, BUFFER_SIZE, 0); - - if (nread == 0) { - break; - } else if (nread < 0) { - if (errno == EINTR) { - continue; - } else { - uError("failed to perform recv func at %d since %s", port, strerror(errno)); - taosCloseSocket(serverSocket); - return NULL; - } - } else { - nleft -= nread; - ptr += nread; - iDataNum += nread; - } + int32_t ret = taosReadMsg(client, buffer, pinfo->pktLen); + if (ret < 0 || ret != pinfo->pktLen) { + uError("TCP: failed to read %d bytes at port:%d since %s", pinfo->pktLen, port, strerror(errno)); + taosCloseSocket(serverSocket); + return NULL; } - if (iDataNum > 0) { - uInfo("TCP: recv:%d bytes from %s:%d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); - send(client, buffer, iDataNum, 0); + uInfo("TCP: read:%d bytes from %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port); + + ret = taosWriteMsg(client, buffer, pinfo->pktLen); + if (ret < 0) { + uError("TCP: failed to write %d bytes at %d since %s", pinfo->pktLen, port, strerror(errno)); + taosCloseSocket(serverSocket); + return NULL; } + + uInfo("TCP: write:%d bytes to %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port); } - + taosCloseSocket(serverSocket); return NULL; } static int32_t taosNetCheckTcpPort(STestInfo *info) { - SOCKET clientSocket; - char sendbuf[BUFFER_SIZE]; - char recvbuf[BUFFER_SIZE]; - int32_t iDataNum = 0; + SOCKET clientSocket; + char buffer[BUFFER_SIZE] = {0}; - struct sockaddr_in serverAddr; if ((clientSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { - uError("failed to create tcp client socket since %s", strerror(errno)); + uError("failed to create TCP client socket since %s", strerror(errno)); return -1; } - // set send and recv overtime - struct timeval timeout; - timeout.tv_sec = 2; // s - timeout.tv_usec = 0; // us - if (setsockopt(clientSocket, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { - uError("failed to setsockopt send timer since %s", strerror(errno)); - } - if (setsockopt(clientSocket, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { - uError("failed to setsockopt recv timer since %s", strerror(errno)); + int32_t reuse = 1; + if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(clientSocket); + return -1; } + struct sockaddr_in serverAddr; + memset((char *)&serverAddr, 0, sizeof(serverAddr)); serverAddr.sin_family = AF_INET; - serverAddr.sin_port = htons(info->port); + serverAddr.sin_port = (uint16_t)htons((uint16_t)info->port); serverAddr.sin_addr.s_addr = info->hostIp; if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { - uError("failed to connect port:%d since %s", info->port, strerror(errno)); + uError("TCP: failed to connect port %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); return -1; } - memset(sendbuf, 0, BUFFER_SIZE); - memset(recvbuf, 0, BUFFER_SIZE); - - struct in_addr ipStr; - memcpy(&ipStr, &info->hostIp, 4); - sprintf(sendbuf, "client send tcp pkg to %s:%d, content: 1122334455", taosInetNtoa(ipStr), info->port); - sprintf(sendbuf + info->pktLen - 16, "1122334455667788"); - - send(clientSocket, sendbuf, info->pktLen, 0); + taosKeepTcpAlive(clientSocket); - memset(recvbuf, 0, BUFFER_SIZE); - int32_t nleft, nread; - char * ptr = recvbuf; - nleft = info->pktLen; + sprintf(buffer, "client send TCP pkg to %s:%d, content: 1122334455", taosIpStr(info->hostIp), info->port); + sprintf(buffer + info->pktLen - 16, "1122334455667788"); - while (nleft > 0) { - nread = recv(clientSocket, ptr, BUFFER_SIZE, 0);; - - if (nread == 0) { - break; - } else if (nread < 0) { - if (errno == EINTR) { - continue; - } else { - uError("faild to recv pkg from TCP port:%d since %s", info->port, strerror(errno)); - taosCloseSocket(clientSocket); - return -1; - } - } else { - nleft -= nread; - ptr += nread; - iDataNum += nread; - } + int32_t ret = taosWriteMsg(clientSocket, buffer, info->pktLen); + if (ret < 0) { + uError("TCP: failed to write msg to %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); + return -1; } - if (iDataNum < info->pktLen) { - uError("TCP: received ack:%d bytes, less than send:%d bytes from port:%d", iDataNum, info->pktLen, info->port); + ret = taosReadMsg(clientSocket, buffer, info->pktLen); + if (ret < 0) { + uError("TCP: failed to read msg from %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); return -1; } @@ -239,9 +227,9 @@ static int32_t taosNetCheckTcpPort(STestInfo *info) { static int32_t taosNetCheckUdpPort(STestInfo *info) { SOCKET clientSocket; - char sendbuf[BUFFER_SIZE]; - char recvbuf[BUFFER_SIZE]; + char buffer[BUFFER_SIZE] = {0}; int32_t iDataNum = 0; + int32_t bufSize = 1024000; struct sockaddr_in serverAddr; @@ -250,41 +238,39 @@ static int32_t taosNetCheckUdpPort(STestInfo *info) { return -1; } - // set overtime - struct timeval timeout; - timeout.tv_sec = 2; // s - timeout.tv_usec = 0; // us - if (setsockopt(clientSocket, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { - uError("failed to setsockopt send timer since %s", strerror(errno)); + if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + uError("failed to set the send buffer size for UDP socket\n"); + return -1; } - if (setsockopt(clientSocket, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { - uError("failed to setsockopt recv timer since %s", strerror(errno)); + + if (taosSetSockOpt(clientSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { + uError("failed to set the receive buffer size for UDP socket\n"); + return -1; } serverAddr.sin_family = AF_INET; serverAddr.sin_port = htons(info->port); serverAddr.sin_addr.s_addr = info->hostIp; - memset(sendbuf, 0, BUFFER_SIZE); - memset(recvbuf, 0, BUFFER_SIZE); - struct in_addr ipStr; memcpy(&ipStr, &info->hostIp, 4); - sprintf(sendbuf, "client send udp pkg to %s:%d, content: 1122334455", taosInetNtoa(ipStr), info->port); - sprintf(sendbuf + info->pktLen - 16, "1122334455667788"); + sprintf(buffer, "client send udp pkg to %s:%d, content: 1122334455", taosInetNtoa(ipStr), info->port); + sprintf(buffer + info->pktLen - 16, "1122334455667788"); socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr); - int32_t code = sendto(clientSocket, sendbuf, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size); - if (code < 0) { - uError("failed to perform sendto func since %s", strerror(errno)); + iDataNum = taosSendto(clientSocket, buffer, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size); + if (iDataNum < 0 || iDataNum != info->pktLen) { + uError("UDP: failed to perform sendto func since %s", strerror(errno)); return -1; } - iDataNum = recvfrom(clientSocket, recvbuf, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size); + memset(buffer, 0, BUFFER_SIZE); + sin_size = sizeof(*(struct sockaddr *)&serverAddr); + iDataNum = recvfrom(clientSocket, buffer, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size); - if (iDataNum < info->pktLen) { - uError("UDP: received ack:%d bytes, less than send:%d bytes from port:%d", iDataNum, info->pktLen, info->port); + if (iDataNum < 0 || iDataNum != info->pktLen) { + uError("UDP: received ack:%d bytes(expect:%d) from port:%d since %s", iDataNum, info->pktLen, info->port, strerror(errno)); return -1; } @@ -304,19 +290,18 @@ static void taosNetCheckPort(uint32_t hostIp, int32_t startPort, int32_t endPort info.port = port; ret = taosNetCheckTcpPort(&info); if (ret != 0) { - uError("failed to test tcp port:%d", port); + uError("failed to test TCP port:%d", port); } else { - uInfo("successed to test tcp port:%d", port); + uInfo("successed to test TCP port:%d", port); } ret = taosNetCheckUdpPort(&info); if (ret != 0) { - uError("failed to test udp port:%d", port); + uError("failed to test UDP port:%d", port); } else { - uInfo("successed to test udp port:%d", port); + uInfo("successed to test UDP port:%d", port); } } - return; } void *taosNetInitRpc(char *secretEncrypt, char spi) { @@ -440,9 +425,9 @@ static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { int32_t ret = taosNetCheckRpc(host, port, sendpkgLen, spi, NULL); if (ret < 0) { - uError("failed to test tcp port:%d", port); + uError("failed to test TCP port:%d", port); } else { - uInfo("successed to test tcp port:%d", port); + uInfo("successed to test TCP port:%d", port); } if (pkgLen >= tsRpcMaxUdpSize) { @@ -453,9 +438,9 @@ static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { ret = taosNetCheckRpc(host, port, pkgLen, spi, NULL); if (ret < 0) { - uError("failed to test udp port:%d", port); + uError("failed to test UDP port:%d", port); } else { - uInfo("successed to test udp port:%d", port); + uInfo("successed to test UDP port:%d", port); } } } @@ -492,14 +477,15 @@ static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { tcpInfo->pktLen = pkgLen; if (pthread_create(pids + i, NULL, taosNetBindTcpPort, tcpInfo) != 0) { - uInfo("failed to create tcp test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); + uInfo("failed to create TCP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); exit(-1); } STestInfo *udpInfo = uinfos + i; - udpInfo->port = (uint16_t)(port + i); + udpInfo->port = port + i; + tcpInfo->pktLen = pkgLen; if (pthread_create(pids + num + i, NULL, taosNetBindUdpPort, udpInfo) != 0) { - uInfo("failed to create udp test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); + uInfo("failed to create UDP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); exit(-1); } } diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index 82ce2c314d5051492f056ebb539e55d7b37c9ebf..f2db0b331619cac89aae14fe38d9cb0e65f9c034 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -228,7 +228,7 @@ static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxN } void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len) { - if (pNote->fd < 0) return; + if (pNote->fd <= 0) return; taosWrite(pNote->fd, buffer, len); if (pNote->maxLines > 0) { diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index e286a972dc97f94613614fdefea273cae7459f26..36983c1cf0e4b7bc3f86eb11132cc844f2a7e9be 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -441,7 +441,6 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { if (status == TSDB_STATUS_COMMIT_START) { pVnode->isCommiting = 1; - pVnode->fversion = pVnode->version; vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); if (!vnodeInInitStatus(pVnode)) { return walRenew(pVnode->wal); @@ -450,9 +449,10 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { } if (status == TSDB_STATUS_COMMIT_OVER) { - vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); pVnode->isCommiting = 0; pVnode->isFull = 0; + pVnode->fversion = pVnode->version; + vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); if (!vnodeInInitStatus(pVnode)) { walRemoveOneOldFile(pVnode->wal); } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 10e1b4dd614fb881ea8819a16ca2451b20f4393d..9174da3d0397500c3fed11ec8d311ec6b095edc5 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -166,14 +166,14 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) { char walName[WAL_FILE_LEN]; snprintf(walName, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId); - wDebug("vgId:%d, file:%s, will be restored", pWal->vgId, walName); + wInfo("vgId:%d, file:%s, will be restored", pWal->vgId, walName); int32_t code = walRestoreWalFile(pWal, pVnode, writeFp, walName, fileId); if (code != TSDB_CODE_SUCCESS) { wError("vgId:%d, file:%s, failed to restore since %s", pWal->vgId, walName, tstrerror(code)); continue; } - wDebug("vgId:%d, file:%s, restore success", pWal->vgId, walName); + wInfo("vgId:%d, file:%s, restore success", pWal->vgId, walName); count++; } @@ -326,7 +326,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch offset = offset + sizeof(SWalHead) + pHead->len; - wDebug("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId, + wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId, fileId, pHead->version, pWal->version, pHead->len); pWal->version = pHead->version; diff --git a/tests/examples/JDBC/mybatisplus-demo/pom.xml b/tests/examples/JDBC/mybatisplus-demo/pom.xml index 8535f3b797dcf13bd47d968f735ba5e7873fad51..e59b915d2cba0c53bfcea36782ba25f4fa19c37b 100644 --- a/tests/examples/JDBC/mybatisplus-demo/pom.xml +++ b/tests/examples/JDBC/mybatisplus-demo/pom.xml @@ -38,11 +38,16 @@ h2 runtime + + com.alibaba + druid + 1.1.17 + com.taosdata.jdbc taos-jdbcdriver - 2.0.11 + 2.0.14 diff --git a/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml b/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml index 96667f28b8d45d74541609f3d44176534c609f23..71e518602e8f68b9f6691cba60cb6ad0de247bfe 100644 --- a/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml +++ b/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml @@ -1,17 +1,5 @@ spring: datasource: - # driver-class-name: org.h2.Driver - # schema: classpath:db/schema-mysql.sql - # data: classpath:db/data-mysql.sql - # url: jdbc:h2:mem:test - # username: root - # password: test - - # driver-class-name: com.mysql.jdbc.Driver - # url: jdbc:mysql://master:3306/test?useSSL=false - # username: root - # password: 123456 - driver-class-name: com.taosdata.jdbc.TSDBDriver url: jdbc:TAOS://localhost:6030/mp_test user: root @@ -20,6 +8,12 @@ spring: locale: en_US.UTF-8 timezone: UTC-8 + druid: + initial-size: 5 + min-idle: 5 + max-active: 5 + + mybatis-plus: configuration: map-underscore-to-camel-case: false diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index 51bb9b36c3c55802fa904de223c2ae4ea2ea7151..b96daa54649b115961de97245c6cd109d82f0768 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -65,7 +65,18 @@ function runQueryPerfTest { echoInfo "Run Performance Test" cd $WORK_DIR/TDengine/tests/pytest - python3 query/queryPerformance.py 0 | tee -a $PERFORMANCE_TEST_REPORT + python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT + + python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT + + yes | taosdemo -c /etc/taosperf/ -d taosdemo_insert_test -t 1000 -n 1000 > taosdemoperf.txt + + CREATETABLETIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'` + INSERTRECORDSTIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'` + REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $13}'` + + python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND | tee -a $PERFORMANCE_TEST_REPORT + [ -f taosdemoperf.txt ] && rm taosdemoperf.txt } diff --git a/tests/pytest/bug2265.py b/tests/pytest/bug2265.py new file mode 100644 index 0000000000000000000000000000000000000000..e78233928fcd7f2348dc83c64b85cca3cf983477 --- /dev/null +++ b/tests/pytest/bug2265.py @@ -0,0 +1,85 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import taos +if __name__ == "__main__": + + logSql = True + deployPath = "" + testCluster = False + valgrind = 0 + + print("start to execute %s" % __file__) + tdDnodes.init(deployPath) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + + tdDnodes.stopAll() + tdDnodes.addSimExtraCfg("maxSQLLength", "1048576") + tdDnodes.deploy(1) + tdDnodes.start(1) + host = '127.0.0.1' + + tdLog.info("Procedures for tdengine deployed in %s" % (host)) + + tdCases.logSql(logSql) + print('1') + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + + tdSql.init(conn.cursor(), True) + + print("==========step1") + print("create table ") + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute("create table t1 (ts timestamp, c1 int,c2 int ,c3 int)") + + print("==========step2") + print("insert maxSQLLength data ") + data = 'insert into t1 values' + ts = 1604298064000 + i = 0 + while ((len(data)<(1024*1024)) & (i < 32767 - 1) ): + data += '(%s,%d,%d,%d)'%(ts+i,i%1000,i%1000,i%1000) + i+=1 + tdSql.execute(data) + + print("==========step4") + print("insert data batch larger than 32767 ") + i = 0 + while ((len(data)<(1024*1024)) & (i < 32767) ): + data += '(%s,%d,%d,%d)'%(ts+i,i%1000,i%1000,i%1000) + i+=1 + tdSql.error(data) + + print("==========step4") + print("insert data larger than maxSQLLength ") + tdSql.execute("create table t2 (ts timestamp, c1 binary(50))") + data = 'insert into t2 values' + i = 0 + while ((len(data)<(1024*1024)) & (i < 32767 - 1 ) ): + data += '(%s,%s)'%(ts+i,'a'*50) + i+=1 + tdSql.error(data) + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + + diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index dc2c0099b4a1477755543b16312708beb98ade18..983f43729705ca44bcd511ab79b202e010fb93c6 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -20,13 +20,23 @@ python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py python3 ./test.py -f insert/before_1970.py +python3 bug2265.py +#table python3 ./test.py -f table/alter_wal0.py python3 ./test.py -f table/column_name.py python3 ./test.py -f table/column_num.py python3 ./test.py -f table/db_table.py python3 ./test.py -f table/create_sensitive.py #python3 ./test.py -f table/tablename-boundary.py +python3 ./test.py -f table/max_table_length.py +python3 ./test.py -f table/alter_column.py +python3 ./test.py -f table/boundary.py +python3 ./test.py -f table/create-a-lot.py +python3 ./test.py -f table/create.py +python3 ./test.py -f table/del_stable.py +python3 ./test.py -f table/queryWithTaosdKilled.py + # tag python3 ./test.py -f tag_lite/filter.py @@ -135,9 +145,6 @@ python3 ./test.py -f user/pass_len.py # stable python3 ./test.py -f stable/query_after_reset.py -# table -python3 ./test.py -f table/del_stable.py - #query python3 ./test.py -f query/filter.py python3 ./test.py -f query/filterCombo.py @@ -164,7 +171,8 @@ python3 ./test.py -f query/bug2117.py python3 ./test.py -f query/bug2143.py python3 ./test.py -f query/sliding.py python3 ./test.py -f query/unionAllTest.py - +python3 ./test.py -f query/bug2281.py +python3 ./test.py -f query/bug2119.py #stream python3 ./test.py -f stream/metric_1.py python3 ./test.py -f stream/new.py diff --git a/tests/pytest/insert/insertFromCSVPerformance.py b/tests/pytest/insert/insertFromCSVPerformance.py new file mode 100644 index 0000000000000000000000000000000000000000..84fd1d7ccab85d7e4469c779f10f047160ee0565 --- /dev/null +++ b/tests/pytest/insert/insertFromCSVPerformance.py @@ -0,0 +1,131 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import time +import datetime +import csv +import random +import pandas as pd +import argparse +import os.path + +class insertFromCSVPerformace: + def __init__(self, commitID, dbName, stbName, branchName): + self.commitID = commitID + self.dbName = dbName + self.stbName = stbName + self.branchName = branchName + self.ts = 1500074556514 + self.host = "127.0.0.1" + self.user = "root" + self.password = "taosdata" + self.config = "/etc/taosperf" + self.conn = taos.connect( + self.host, + self.user, + self.password, + self.config) + + def writeCSV(self): + with open('test3.csv','w', encoding='utf-8', newline='') as csvFile: + writer = csv.writer(csvFile, dialect='excel') + for i in range(1000000): + newTimestamp = self.ts + random.randint(10000000, 10000000000) + random.randint(1000, 10000000) + random.randint(1, 1000) + d = datetime.datetime.fromtimestamp(newTimestamp / 1000) + dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f")) + writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)]) + + def removCSVHeader(self): + data = pd.read_csv("ordered.csv") + data = data.drop([0]) + data.to_csv("ordered.csv", header = False, index = False) + + def createTables(self): + cursor = self.conn.cursor() + + cursor.execute("create database if not exists %s" % self.dbName) + cursor.execute("use %s" % self.dbName) + cursor.execute("create table if not exists %s(ts timestamp, in_order_time float, out_of_order_time float, commit_id binary(50)) tags(branch binary(50))" % self.stbName) + cursor.execute("create table if not exists %s using %s tags('%s')" % (self.branchName, self.stbName, self.branchName)) + + cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)") + cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)") + + cursor.close() + + def run(self): + cursor = self.conn.cursor() + cursor.execute("use %s" % self.dbName) + print("==================== CSV insert performance ====================") + + totalTime = 0 + for i in range(10): + cursor.execute("create table if not exists t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)") + startTime = time.time() + cursor.execute("insert into t1 file 'outoforder.csv'") + totalTime += time.time() - startTime + cursor.execute("drop table if exists t1") + out_of_order_time = (float) (totalTime / 10) + print("Out of Order - Insert time: %f" % out_of_order_time) + + totalTime = 0 + for i in range(10): + cursor.execute("create table if not exists t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)") + startTime = time.time() + cursor.execute("insert into t2 file 'ordered.csv'") + totalTime += time.time() - startTime + cursor.execute("drop table if exists t2") + + in_order_time = (float) (totalTime / 10) + print("In order - Insert time: %f" % in_order_time) + cursor.execute("insert into %s values(now, %f, %f, '%s')" % (self.branchName, in_order_time, out_of_order_time, self.commitID)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '-c', + '--commit-id', + action='store', + default='null', + type=str, + help='git commit id (default: null)') + parser.add_argument( + '-d', + '--database-name', + action='store', + default='perf', + type=str, + help='Database name to be created (default: perf)') + parser.add_argument( + '-t', + '--stable-name', + action='store', + default='csv_insert', + type=str, + help='Database name to be created (default: csv_insert)') + parser.add_argument( + '-b', + '--branch-name', + action='store', + default='develop', + type=str, + help='branch name (default: develop)') + + args = parser.parse_args() + perftest = insertFromCSVPerformace(args.commit_id, args.database_name, args.stable_name, args.branch_name) + + perftest.createTables() + perftest.run() \ No newline at end of file diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh index 52f5a30f4e83089bc2e6f230c099e4304a94efd4..6e0c8ef8b22b04cff87fd82db710813db273fdaa 100755 --- a/tests/pytest/pytest_1.sh +++ b/tests/pytest/pytest_1.sh @@ -20,12 +20,20 @@ python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py +#table python3 ./test.py -f table/alter_wal0.py python3 ./test.py -f table/column_name.py python3 ./test.py -f table/column_num.py python3 ./test.py -f table/db_table.py python3 ./test.py -f table/create_sensitive.py #python3 ./test.py -f table/tablename-boundary.py +python3 ./test.py -f table/max_table_length.py +python3 ./test.py -f table/alter_column.py +python3 ./test.py -f table/boundary.py +python3 ./test.py -f table/create-a-lot.py +python3 ./test.py -f table/create.py +python3 ./test.py -f table/del_stable.py +python3 ./test.py -f table/queryWithTaosdKilled.py # tag python3 ./test.py -f tag_lite/filter.py @@ -134,9 +142,6 @@ python3 ./test.py -f user/pass_len.py # stable python3 ./test.py -f stable/query_after_reset.py -# table -python3 ./test.py -f table/del_stable.py - #query python3 ./test.py -f query/filter.py python3 ./test.py -f query/filterCombo.py @@ -159,7 +164,9 @@ python3 ./test.py -f query/bug1874.py python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py python3 ./test.py -f query/bug2218.py - +python3 ./test.py -f query/bug2281.py +python3 ./test.py -f query/bug2119.py +python3 bug2265.py #stream python3 ./test.py -f stream/metric_1.py python3 ./test.py -f stream/new.py diff --git a/tests/pytest/query/bug2119.py b/tests/pytest/query/bug2119.py new file mode 100644 index 0000000000000000000000000000000000000000..4224e55596c201f27fbdacd582a90ef5bfd45b85 --- /dev/null +++ b/tests/pytest/query/bug2119.py @@ -0,0 +1,41 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table t1 (ts timestamp, c1 int, c2 float)") + + + print("==========step2") + print("query percentile from blank table") + tdSql.query('select percentile(c1,1) from t1') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/bug2281.py b/tests/pytest/query/bug2281.py new file mode 100644 index 0000000000000000000000000000000000000000..b8eb17f5cd8594389f0a218a95a7024e18735231 --- /dev/null +++ b/tests/pytest/query/bug2281.py @@ -0,0 +1,47 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table t1 (ts timestamp, c1 int, c2 float)") + insertRows = 10 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(insertRows): + ret = tdSql.execute( + "insert into t1 values (%d , %d,%d)" % + (t0+i,i%100,i/2.0)) + + print("==========step2") + print("query diff && top") + tdSql.error('select diff(c1),top(c2) from t1') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/queryPerformance.py b/tests/pytest/query/queryPerformance.py index 72af38450c862f7558df313dd53316599a23dedd..720ae745cb9b3780f1ca7ffaf96d76eda5f307b1 100644 --- a/tests/pytest/query/queryPerformance.py +++ b/tests/pytest/query/queryPerformance.py @@ -16,10 +16,16 @@ import sys import os import taos import time +import argparse class taosdemoQueryPerformace: - def initConnection(self): + def __init__(self, clearCache, commitID, dbName, stbName, tbPerfix): + self.clearCache = clearCache + self.commitID = commitID + self.dbName = dbName + self.stbName = stbName + self.tbPerfix = tbPerfix self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" @@ -30,92 +36,109 @@ class taosdemoQueryPerformace: self.password, self.config) + def createPerfTables(self): + cursor = self.conn.cursor() + cursor.execute("create database if not exists %s" % self.dbName) + cursor.execute("use %s" % self.dbName) + cursor.execute("create table if not exists %s(ts timestamp, query_time float, commit_id binary(50)) tags(query_id int, query_sql binary(300))" % self.stbName) + + sql = "select count(*) from test.meters" + tableid = 1 + cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select avg(f1), max(f2), min(f3) from test.meters" + tableid = 2 + cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select count(*) from test.meters where loc='beijing'" + tableid = 3 + cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select avg(f1), max(f2), min(f3) from test.meters where areaid=10" + tableid = 4 + cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select avg(f1), max(f2), min(f3) from test.t10 interval(10s)" + tableid = 5 + cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select last_row(*) from meters" + tableid = 6 + cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select * from meters" + tableid = 7 + cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + sql = "select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'" + tableid = 8 + cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql)) + + cursor.close() def query(self): - cursor = self.conn.cursor() - cursor.execute("use test") - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select count(*) from test.meters") - totalTime += time.time() - startTime - print("query time for: select count(*) from test.meters %f seconds" % (totalTime / 100)) - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select avg(f1), max(f2), min(f3) from test.meters") - totalTime += time.time() - startTime - print("query time for: select avg(f1), max(f2), min(f3) from test.meters %f seconds" % (totalTime / 100)) - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select count(*) from test.meters where loc='beijing'") - totalTime += time.time() - startTime - print("query time for: select count(*) from test.meters where loc='beijing' %f seconds" % (totalTime / 100)) - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select avg(f1), max(f2), min(f3) from test.meters where areaid=10") - totalTime += time.time() - startTime - print("query time for: select avg(f1), max(f2), min(f3) from test.meters where areaid=10 %f seconds" % (totalTime / 100)) - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select avg(f1), max(f2), min(f3) from test.t10 interval(10s)") - totalTime += time.time() - startTime - print("query time for: select avg(f1), max(f2), min(f3) from test.t10 interval(10s) %f seconds" % (totalTime / 100)) - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select last_row(*) from meters") - totalTime += time.time() - startTime - print("query time for: select last_row(*) from meters %f seconds" % (totalTime / 100)) - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select * from meters") - totalTime += time.time() - startTime - print("query time for: select * from meters %f seconds" % (totalTime / 100)) - - totalTime = 0 - for i in range(100): - if(sys.argv[1] == '1'): - # root permission is required - os.system("echo 3 > /proc/sys/vm/drop_caches") - startTime = time.time() - cursor.execute("select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'") - totalTime += time.time() - startTime - print("query time for: select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000' %f seconds" % (totalTime / 100)) + cursor = self.conn.cursor() + print("==================== query performance ====================") + + cursor.execute("use %s" % self.dbName) + cursor.execute("select tbname, query_id, query_sql from %s" % self.stbName) + + for data in cursor: + table_name = data[0] + query_id = data[1] + sql = data[2] + + totalTime = 0 + cursor2 = self.conn.cursor() + cursor2.execute("use test") + for i in range(100): + if(self.clearCache == True): + # root permission is required + os.system("echo 3 > /proc/sys/vm/drop_caches") + + startTime = time.time() + cursor2.execute(sql) + totalTime += time.time() - startTime + cursor2.close() + print("query time for: %s %f seconds" % (sql, totalTime / 100)) + + cursor3 = self.conn.cursor() + cursor3.execute("insert into %s.%s values(now, %f, '%s')" % (self.dbName, table_name, totalTime / 100, self.commitID)) + + cursor3.close() + cursor.close() if __name__ == '__main__': - perftest = taosdemoQueryPerformace() - perftest.initConnection() - perftest.query() \ No newline at end of file + parser = argparse.ArgumentParser() + parser.add_argument( + '-r', + '--remove-cache', + action='store_true', + default=False, + help='clear cache before query (default: False)') + parser.add_argument( + '-c', + '--commit-id', + action='store', + default='null', + type=str, + help='git commit id (default: null)') + parser.add_argument( + '-d', + '--database-name', + action='store', + default='perf', + type=str, + help='Database name to be created (default: perf)') + parser.add_argument( + '-t', + '--stable-name', + action='store', + default='query_tb', + type=str, + help='table name to be created (default: query_tb)') + parser.add_argument( + '-p', + '--table-perfix', + action='store', + default='q', + type=str, + help='table name perfix (default: q)') + + args = parser.parse_args() + perftest = taosdemoQueryPerformace(args.remove_cache, args.commit_id, args.database_name, args.stable_name, args.table_perfix) + perftest.createPerfTables() + perftest.query() diff --git a/tests/pytest/query/unionAllTest.py b/tests/pytest/query/unionAllTest.py index bb4fb95de676679d448a35b137ee8264cf802ba2..1b69c8ac4d015a2ad8db72947022b7bd27e92756 100644 --- a/tests/pytest/query/unionAllTest.py +++ b/tests/pytest/query/unionAllTest.py @@ -52,7 +52,7 @@ class TDTestCase: tdSql.checkRows(5) sql = ''' select * from st where loc = 'nchar0' limit 1 union all select * from st where loc = 'nchar1' limit 1 union all select * from st where loc = 'nchar2' limit 1 - union all select * from st where loc = 'nchar3' limit 1 union all select * from st where loc = 'nchar4' limit 1 union all select * from st where loc = 'nchar5'''' + union all select * from st where loc = 'nchar3' limit 1 union all select * from st where loc = 'nchar4' limit 1 union all select * from st where loc = 'nchar5' limit 1''' tdSql.query(sql) tdSql.checkRows(6) diff --git a/tests/pytest/table/createTableFromAnotherDb.py b/tests/pytest/table/createTableFromAnotherDb.py new file mode 100644 index 0000000000000000000000000000000000000000..b40e72404cd0eb2e887199eca9964a0765106124 --- /dev/null +++ b/tests/pytest/table/createTableFromAnotherDb.py @@ -0,0 +1,41 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create table db.cars(ts timestamp, c int) tags(id int);") + tdSql.execute("create database db2") + tdSql.error("create table db2.car1 using db.cars tags(1)") + tdSql.error("insert into db2.car1 using db1.cars tags(1) values(now, 1);") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/table/max_table_length.py b/tests/pytest/table/max_table_length.py new file mode 100644 index 0000000000000000000000000000000000000000..ec34f3008fccdb200e645183a5cfdf577fc27503 --- /dev/null +++ b/tests/pytest/table/max_table_length.py @@ -0,0 +1,55 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, db_test.stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("==============step1") + + tdLog.info("check nchar") + tdSql.error("create database anal (ts timestamp ,i nchar(4094))") + tdSql.execute( + "create table anal (ts timestamp ,i nchar(4093))") + + print("==============step2") + tdLog.info("check binary") + tdSql.error("create database anal (ts timestamp ,i binary(16375))") + tdSql.execute( + "create table anal1 (ts timestamp ,i binary(16374))") + + print("==============step3") + tdLog.info("check int & binary") + tdSql.error("create table anal2 (ts timestamp ,i binary(16371),j int)") + tdSql.execute("create table anal2 (ts timestamp ,i binary(16370),j int)") + tdSql.execute("create table anal3 (ts timestamp ,i binary(16366), j int, k int)") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py new file mode 100644 index 0000000000000000000000000000000000000000..6b6296e61a7965facd3a414ad635a4c64af43cc4 --- /dev/null +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -0,0 +1,93 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import time +import datetime +import csv +import random +import pandas as pd +import argparse +import os.path + +class taosdemoPerformace: + def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond): + self.commitID = commitID + self.dbName = dbName + self.createTableTime = createTableTime + self.insertRecordsTime = insertRecordsTime + self.recordsPerSecond = recordsPerSecond + self.host = "127.0.0.1" + self.user = "root" + self.password = "taosdata" + self.config = "/etc/taosperf" + self.conn = taos.connect( + self.host, + self.user, + self.password, + self.config) + + def createTablesAndStoreData(self): + cursor = self.conn.cursor() + + cursor.execute("create database if not exists %s" % self.dbName) + cursor.execute("use %s" % self.dbName) + cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50))") + print("==================== taosdemo performance ====================") + print("create tables time: %f" % self.createTableTime) + print("insert records time: %f" % self.insertRecordsTime) + print("records per second: %f" % self.recordsPerSecond) + cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s')" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID)) + cursor.execute("drop database if exists taosdemo_insert_test") + + cursor.close() + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '-c', + '--commit-id', + action='store', + type=str, + help='git commit id (default: null)') + parser.add_argument( + '-d', + '--database-name', + action='store', + default='perf', + type=str, + help='Database name to be created (default: perf)') + parser.add_argument( + '-t', + '--create-table', + action='store', + type=float, + help='create table time') + parser.add_argument( + '-i', + '--insert-records', + action='store', + type=float, + help='insert records time') + parser.add_argument( + '-r', + '---records-per-second', + action='store', + type=float, + help='records per request') + + args = parser.parse_args() + + perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second) + perftest.createTablesAndStoreData() \ No newline at end of file diff --git a/tests/test/c/hashPerformance.c b/tests/test/c/hashPerformance.c index db3be0e950a81e22f3ad1050cfa9a8e50b967532..111ea25a09823fbbeeda3c990d86af849c3821d8 100644 --- a/tests/test/c/hashPerformance.c +++ b/tests/test/c/hashPerformance.c @@ -24,15 +24,17 @@ #define GREEN "\033[1;32m" #define NC "\033[0m" -int32_t capacity = 100000; -int32_t q1Times = 1; -int32_t q2Times = 1; +int32_t capacity = 128; +int32_t q1Times = 10; +int32_t q2Times = 10; int32_t keyNum = 100000; -int32_t printInterval = 10000; +int32_t printInterval = 1000; +void * hashHandle; +pthread_t thread; typedef struct HashTestRow { - int32_t size; - void * ptr; + int32_t keySize; + char key[100]; } HashTestRow; void shellParseArgument(int argc, char *argv[]); @@ -40,7 +42,7 @@ void shellParseArgument(int argc, char *argv[]); void testHashPerformance() { int64_t initialMs = taosGetTimestampMs(); _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - void * hashHandle = taosHashInit(capacity, hashFp, true); + hashHandle = taosHashInit(128, hashFp, true, HASH_NO_LOCK); int64_t startMs = taosGetTimestampMs(); float seconds = (startMs - initialMs) / 1000.0; @@ -48,17 +50,25 @@ void testHashPerformance() { for (int32_t t = 1; t <= keyNum; ++t) { HashTestRow row = {0}; - char key[100] = {0}; - int32_t keySize = sprintf(key, "0.db.st%d", t); + row.keySize = sprintf(row.key, "0.db.st%d", t); for (int32_t q = 0; q < q1Times; q++) { - taosHashGet(hashHandle, &key, keySize); + taosHashGet(hashHandle, row.key, row.keySize); } - taosHashPut(hashHandle, key, keySize, &row, sizeof(HashTestRow)); + taosHashPut(hashHandle, row.key, row.keySize, &row, sizeof(HashTestRow)); for (int32_t q = 0; q < q2Times; q++) { - taosHashGet(hashHandle, &key, keySize); + taosHashGet(hashHandle, row.key, row.keySize); + } + + // test iterator + { + HashTestRow *row = taosHashIterate(hashHandle, NULL); + while (row) { + taosHashGet(hashHandle, row->key, row->keySize); + row = taosHashIterate(hashHandle, row); + } } if (t % printInterval == 0) { @@ -80,9 +90,35 @@ void testHashPerformance() { taosHashCleanup(hashHandle); } +void *multiThreadFunc(void *param) { + for (int i = 0; i < 100; ++i) { + taosMsleep(1000); + HashTestRow *row = taosHashIterate(hashHandle, NULL); + while (row) { + taosHashGet(hashHandle, row->key, row->keySize); + row = taosHashIterate(hashHandle, row); + } + int64_t hashSize = taosHashGetSize(hashHandle); + pPrint("i:%d hashSize:%ld", i, hashSize); + } + + return NULL; +} + +void multiThreadTest() { + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + + // Start threads to write + pthread_create(&thread, &thattr, multiThreadFunc, NULL); +} + int main(int argc, char *argv[]) { shellParseArgument(argc, argv); + multiThreadTest(); testHashPerformance(); + pthread_join(thread, NULL); } void printHelp() {