diff --git a/Jenkinsfile b/Jenkinsfile index fc2b3562c15a3babf7bc3a5820e65ec8e162e6d7..4b84e1f88e71b3a43bc63df10edffe8a8758052a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -113,7 +113,7 @@ pipeline { ''' sh''' cd ${WKC}/debug - ctest + ctest -VV ''' } } diff --git a/Jenkinsfile2 b/Jenkinsfile2 index cca886d4afda1f8711c06a3dec1afac201c69db7..c9c9c3a7caa64c626fc5bee918ff5d513a593593 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -121,7 +121,7 @@ pipeline { pre_test() sh''' cd ${WKC}/debug - ctest + ctest -VV ''' sh''' export LD_LIBRARY_PATH=${WKC}/debug/build/lib diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 19923a5ad63bc6ec584324116956ea272b3df52b..1ddc765c5c5a93dd4b8a6001ebede7c3bde22a59 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -14,24 +14,9 @@ if(${BUILD_PTHREAD}) cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() -# iconv -if(${BUILD_WITH_ICONV}) - cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif() - -# msvc regex -if(${BUILD_MSVCREGEX}) - cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif() - -# wcwidth -if(${BUILD_WCWIDTH}) - cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif() - -# wingetopt -if(${BUILD_WINGETOPT}) - cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +# gnu regex +if(${BUILD_GNUREGEX}) + cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() # googletest @@ -114,27 +99,8 @@ if(${BUILD_TEST}) target_include_directories( gtest PUBLIC $ + PUBLIC $ ) - if(${TD_WINDOWS}) - target_include_directories( - gtest - PUBLIC $ - ) - endif(${TD_WINDOWS}) - if(${TD_LINUX}) - target_include_directories( - gtest - PUBLIC $ - ) - endif(${TD_LINUX}) - if(${TD_DARWIN}) - target_include_directories( - gtest - PUBLIC $ - ) - endif(${TD_DARWIN}) - - endif(${BUILD_TEST}) # cJson @@ -216,53 +182,6 @@ if(${BUILD_WITH_NURAFT}) add_subdirectory(nuraft) endif(${BUILD_WITH_NURAFT}) -# pthread -if(${BUILD_PTHREAD}) - set(CMAKE_BUILD_TYPE release) - add_definitions(-DPTW32_STATIC_LIB) - add_subdirectory(pthread) - set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread) - add_library(pthread STATIC IMPORTED GLOBAL) - SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib) -endif() - -# iconv -if(${BUILD_WITH_ICONV}) - add_subdirectory(iconv) -endif(${BUILD_WITH_ICONV}) - -# wingetopt -if(${BUILD_WINGETOPT}) - add_subdirectory(wingetopt) -endif(${BUILD_WINGETOPT}) - -# msvcregex -if(${BUILD_MSVCREGEX}) - add_library(msvcregex STATIC "") - target_sources(msvcregex - PRIVATE "msvcregex/regex.c" - ) - target_include_directories(msvcregex - PRIVATE "msvcregex" - ) - target_link_libraries(msvcregex - INTERFACE Shell32 - ) - SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex) -endif(${BUILD_MSVCREGEX}) - -# msvcregex -if(${BUILD_WCWIDTH}) - add_library(wcwidth STATIC "") - target_sources(wcwidth - PRIVATE "wcwidth/wcwidth.c" - ) - target_include_directories(wcwidth - PRIVATE "wcwidth" - ) - SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth) -endif(${BUILD_WCWIDTH}) - # CRAFT if(${BUILD_WITH_CRAFT}) add_library(craft STATIC IMPORTED GLOBAL) @@ -319,12 +238,8 @@ if(${BUILD_WITH_SQLITE}) target_link_libraries(sqlite INTERFACE m INTERFACE pthread + INTERFACE dl ) - if(NOT TD_WINDOWS) - target_link_libraries(sqlite - INTERFACE dl - ) - endif(NOT TD_WINDOWS) endif(${BUILD_WITH_SQLITE}) # pthread diff --git a/contrib/test/craft/CMakeLists.txt b/contrib/test/craft/CMakeLists.txt index ec8b44b67395a481ed2970f19cb5d0e1290f6d6b..e0f6ae64bd3e9c998db15c1006cebf1b15702f5b 100644 --- a/contrib/test/craft/CMakeLists.txt +++ b/contrib/test/craft/CMakeLists.txt @@ -1,9 +1,2 @@ add_executable(simulate_vnode "simulate_vnode.c") -target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) -if(${BUILD_WINGETOPT}) - target_link_libraries(simulate_vnode PUBLIC wingetopt) - target_include_directories( - simulate_vnode - PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" - ) -endif() \ No newline at end of file +target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) \ No newline at end of file diff --git a/contrib/test/tdev/src/main.c b/contrib/test/tdev/src/main.c index e40040ce9762e39ca3b23380f4470352a96c921b..5e1de83e88a2a76d2381726ded0c28fe00ab67a6 100644 --- a/contrib/test/tdev/src/main.c +++ b/contrib/test/tdev/src/main.c @@ -6,39 +6,43 @@ #define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s))) #define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa)) -static inline void tPutA(void **buf, uint64_t val) { - memcpy(buf, &val, sizeof(val)); - *buf = POINTER_SHIFT(buf, sizeof(val)); -} +#define tPutA(buf, val) \ + ({ \ + memcpy(buf, &val, sizeof(val)); \ + POINTER_SHIFT(buf, sizeof(val)); \ + }) -static inline void tPutB(void **buf, uint64_t val) { - ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; - ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; - ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; - ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; - ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; - ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; - ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; - ((uint8_t *)buf)[0] = (val)&0xff; - *buf = POINTER_SHIFT(buf, sizeof(val)); -} +#define tPutB(buf, val) \ + ({ \ + ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \ + ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \ + ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \ + ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \ + ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \ + ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \ + ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; \ + ((uint8_t *)buf)[0] = (val)&0xff; \ + POINTER_SHIFT(buf, sizeof(val)); \ + }) -static inline void tPutC(void **buf, uint64_t val) { - if (buf) { - ((uint64_t *)buf)[0] = (val); - POINTER_SHIFT(buf, sizeof(val)); - } - *buf = NULL; -} +#define tPutC(buf, val) \ + ({ \ + if (buf) { \ + ((uint64_t *)buf)[0] = (val); \ + POINTER_SHIFT(buf, sizeof(val)); \ + } \ + NULL; \ + }) -static inline void tPutD(void **buf, uint64_t val) { - uint64_t tmp = val; - for (size_t i = 0; i < sizeof(val); i++) { - ((uint8_t *)buf)[i] = tmp & 0xff; - tmp >>= 8; - } - *buf = POINTER_SHIFT(buf, sizeof(val)); -} +#define tPutD(buf, val) \ + ({ \ + uint64_t tmp = val; \ + for (size_t i = 0; i < sizeof(val); i++) { \ + ((uint8_t *)buf)[i] = tmp & 0xff; \ + tmp >>= 8; \ + } \ + POINTER_SHIFT(buf, sizeof(val)); \ + }) static inline void tPutE(void **buf, uint64_t val) { if (buf) { @@ -57,7 +61,7 @@ static void func(T t) { switch (t) { case A: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - tPutA(pBuf, val); + pBuf = tPutA(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -65,7 +69,7 @@ static void func(T t) { break; case B: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - tPutB(pBuf, val); + pBuf = tPutB(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -73,7 +77,7 @@ static void func(T t) { break; case C: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - tPutC(pBuf, val); + pBuf = tPutC(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -81,7 +85,7 @@ static void func(T t) { break; case D: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - tPutD(pBuf, val); + pBuf = tPutD(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } diff --git a/examples/c/asyncdemo.c b/examples/c/asyncdemo.c index 9e214e0966f7848e1ea158a50882fa8c4b4739ca..07e61b871ed2bd8e857861d169973f2ff6153a72 100644 --- a/examples/c/asyncdemo.c +++ b/examples/c/asyncdemo.c @@ -45,7 +45,7 @@ typedef struct { void taos_insert_call_back(void *param, TAOS_RES *tres, int code); void taos_select_call_back(void *param, TAOS_RES *tres, int code); -void taos_error(TAOS *taos); +void shellPrintError(TAOS *taos); static void queryDB(TAOS *taos, char *command) { int i; @@ -102,7 +102,7 @@ int main(int argc, char *argv[]) taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); if (taos == NULL) - taos_error(taos); + shellPrintError(taos); printf("success to connect to server\n"); @@ -193,7 +193,7 @@ int main(int argc, char *argv[]) return 0; } -void taos_error(TAOS *con) +void shellPrintError(TAOS *con) { fprintf(stderr, "TDengine error: %s\n", taos_errstr(con)); taos_close(con); diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index fa203e231a938b1a9be04207adec90ed74442154..71e2e4fba36371afa64e07d35a19adcca7938bfb 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -74,8 +74,8 @@ static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, } char *data = colDataGetVarData(pColumnInfoData, row); return (*data == TSDB_DATA_TYPE_NULL); - } - + } + if (!pColumnInfoData->hasNull) { return false; } @@ -238,10 +238,16 @@ static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32 static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { - int32_t* colSizes = (int32_t*)data; + int32_t* actualLen = (int32_t*) data; + data += sizeof(int32_t); + + uint64_t* groupId = (uint64_t*) data; + data += sizeof(uint64_t); + int32_t* colSizes = (int32_t*)data; data += numOfCols * sizeof(int32_t); - *dataLen = (numOfCols * sizeof(int32_t)); + + *dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t)); int32_t numOfRows = pBlock->info.rows; for (int32_t col = 0; col < numOfCols; ++col) { @@ -273,6 +279,9 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da colSizes[col] = htonl(colSizes[col]); } + + *actualLen = *dataLen; + *groupId = pBlock->info.groupId; } #ifdef __cplusplus diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 8a8e3cd2230f15dbd9f04adced498c8cc3437af3..72f67665ba066b46057c8a0f3195f49f89e9a794 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -34,13 +34,9 @@ extern int32_t tsVersion; extern int32_t tsStatusInterval; // common -extern int32_t tsRpcTimer; -extern int32_t tsRpcMaxTime; -extern bool tsRpcForceTcp; // all commands go to tcp protocol if this is enabled extern int32_t tsMaxConnections; extern int32_t tsMaxShellConns; extern int32_t tsShellActivityTimer; -extern int32_t tsMaxTmrCtrl; extern int32_t tsCompressMsgSize; extern int32_t tsCompressColData; extern int32_t tsMaxNumOfDistinctResults; @@ -98,9 +94,6 @@ extern bool tsDeadLockKillQuery; extern int32_t tsQueryPolicy; // client -extern int32_t tsMaxWildCardsLen; -extern int32_t tsMaxRegexStringLen; -extern int32_t tsMaxNumOfOrderedResults; extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; extern int32_t tsMaxStreamComputDelay; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index b37001bde92bff6c344b8610f0d61dedbd81e5e7..691d491eb053962045ee66a40e28818f87a022de 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -85,6 +85,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "dnode-server-status", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "dnode-net-test", NULL, NULL) // Requests handled by MNODE TD_NEW_MSG_SEG(TDMT_MND_MSG) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index b88afcb39df49c518b41968eee74638ca2143ae1..30d1bd0a518415d98b7358337c94834aa70716a2 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -51,7 +51,7 @@ typedef struct SMetaData { SArray *pTableMeta; // STableMeta array SArray *pVgroupInfo; // SVgroupInfo list SArray *pUdfList; // udf info list - SArray *pEpSetList; // qnode epset list, SArray + SArray *pQnodeList; // qnode list, SArray } SMetaData; typedef struct SCatalogCfg { diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 004d834287295f82a1817b3d445b1091d2e0565c..ef32533e5fce3e8e00f133ebbb1a9d7c9d914843 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -37,7 +37,7 @@ typedef struct SFuncExecEnv { typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx); -typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock, int32_t slotId); +typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); typedef struct SScalarFuncExecFuncs { @@ -141,8 +141,7 @@ struct SResultRowEntryInfo; //for selectivity query, the corresponding tag value is assigned if the data is qualified typedef struct SSubsidiaryResInfo { - int16_t bufLen; // keep the tags data for top/bottom query result - int16_t numOfCols; + int16_t num; struct SqlFunctionCtx **pCtx; } SSubsidiaryResInfo; @@ -187,8 +186,8 @@ typedef struct SqlFunctionCtx { uint8_t currentStage; // record current running step, default: 0 bool isAggSet; int64_t startTs; // timestamp range of current query when function is executed on a specific data block, TODO remove it - ///////////////////////////////////////////////////////////////// bool stableQuery; + ///////////////////////////////////////////////////////////////// int16_t functionId; // function id char * pOutput; // final result output buffer, point to sdata->data int32_t numOfParams; @@ -198,11 +197,15 @@ typedef struct SqlFunctionCtx { int32_t offset; SVariant tag; struct SResultRowEntryInfo *resultInfo; - SSubsidiaryResInfo subsidiaryRes; - SPoint1 start; - SPoint1 end; - SFuncExecFuncs fpSet; - SScalarFuncExecFuncs sfp; + SSubsidiaryResInfo subsidiaries; + SPoint1 start; + SPoint1 end; + SFuncExecFuncs fpSet; + SScalarFuncExecFuncs sfp; + SExprInfo *pExpr; + struct SDiskbasedBuf *pBuf; + struct SSDataBlock *pSrcBlock; + int32_t curBufPage; } SqlFunctionCtx; enum { @@ -319,6 +322,20 @@ struct SUdfInfo; void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); +/** + * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf + * @return error code + */ +int32_t udfcOpen(); + +/** + * destroy udfd proxy + * @return error code + */ +int32_t udfcClose(); + +typedef void *UdfcFuncHandle; + #ifdef __cplusplus } #endif diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 33b0a46ee91f706242a1279a3a42567e51621d92..6770be6e461c45c55238e4f9b3f059cfbf81487f 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -31,19 +31,19 @@ extern "C" { typedef struct TdCmd *TdCmdPtr; -TdCmdPtr taosOpenCmd(const char *cmd); -int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf); -int32_t taosEOFCmd(TdCmdPtr pCmd); -int64_t taosCloseCmd(TdCmdPtr *ppCmd); +TdCmdPtr taosOpenCmd(const char* cmd); +int64_t taosGetLineCmd(TdCmdPtr pCmd, char** __restrict ptrBuf); +int32_t taosEOFCmd(TdCmdPtr pCmd); +int64_t taosCloseCmd(TdCmdPtr* ppCmd); void* taosLoadDll(const char* filename); void* taosLoadSym(void* handle, char* name); void taosCloseDll(void* handle); int32_t taosSetConsoleEcho(bool on); -void setTerminalMode(); -int32_t getOldTerminalMode(); -void resetTerminalMode(); +void taosSetTerminalMode(); +int32_t taosGetOldTerminalMode(); +void taosResetTerminalMode(); #ifdef __cplusplus } diff --git a/include/util/tdef.h b/include/util/tdef.h index 31fbe42b6c538a9ce04159d57d7d55e6a3812cd4..aa0d0bd8ff2355e7ad67132986200a5a4e53537c 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -76,8 +76,6 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_DEFAULT_PASS "taosdata" #endif -#define SHELL_MAX_PASSWORD_LEN 20 - #define TSDB_TRUE 1 #define TSDB_FALSE 0 #define TSDB_OK 0 diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 367baef53f0d8ca24a7da4a23952e0f7bd5b633d..2ae9a118030f2f3f750c3a5b0ac753c20210eed8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -105,9 +105,9 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, epSet.epSet.eps[0].port = port; } - char* key = getClusterKey(user, secretEncrypt, ip, port); - SAppInstInfo** pInst = NULL; + char* key = getClusterKey(user, secretEncrypt, ip, port); + SAppInstInfo** pInst = NULL; taosThreadMutexLock(&appInfo.mutex); pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); @@ -840,10 +840,21 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 return code; } - int32_t* colLength = (int32_t*)pResultInfo->pData; - char* pStart = ((char*)pResultInfo->pData) + sizeof(int32_t) * numOfCols; + char* p = (char*) pResultInfo->pData; + + int32_t dataLen = *(int32_t*) p; + p += sizeof(int32_t); + + uint64_t groupId = *(uint64_t*) p; + p += sizeof(uint64_t); + + int32_t* colLength = (int32_t*)p; + p += sizeof(int32_t) * numOfCols; + + char* pStart = p; for (int32_t i = 0; i < numOfCols; ++i) { colLength[i] = htonl(colLength[i]); + ASSERT(colLength[i] < dataLen); if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { pResultInfo->pCol[i].offset = (int32_t*)pStart; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 800bcd70cee97cd4b02e364fb3e30520e19dc329..09452b584d5236f8e1265506d3adcc720c4b97d9 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -644,12 +644,12 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { /** * @refitem blockDataToBuf for the meta size - * * @param pBlock * @return */ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) { - return sizeof(int32_t) + pBlock->info.numOfCols * sizeof(int32_t); + // | total rows/total length | block group id | each column length | + return sizeof(int32_t) + sizeof(uint64_t) + pBlock->info.numOfCols * sizeof(int32_t); } double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { @@ -1219,7 +1219,6 @@ void colDataDestroy(SColumnInfoData* pColData) { taosMemoryFree(pColData->pData); } - static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { int32_t len = BitmapLen(total); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 46e9654a69ebcf4c5d5a0c5dc5055a79f0ed960e..33999ea14cc901c579a46e8f8cdddd7046dabfd6 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -32,13 +32,9 @@ int32_t tsVersion = 30000000; int32_t tsStatusInterval = 1; // second // common -int32_t tsRpcTimer = 300; -int32_t tsRpcMaxTime = 600; // seconds; -bool tsRpcForceTcp = true; // disable this, means query, show command use udp protocol as default int32_t tsMaxShellConns = 50000; int32_t tsMaxConnections = 50000; int32_t tsShellActivityTimer = 3; // second -int32_t tsMaxBinaryDisplayWidth = 30; bool tsEnableSlaveQuery = true; bool tsPrintAuth = false; @@ -105,14 +101,6 @@ int32_t tsCompressColData = -1; */ int32_t tsCompatibleModel = 1; -// client -int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN; -int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN; - -// the maximum number of results for projection query on super table that are returned from -// one virtual node, to order according to timestamp -int32_t tsMaxNumOfOrderedResults = 100000; - // 10 ms for sliding time, the value will changed in case of time precision changed int32_t tsMinSlidingTime = 10; @@ -314,19 +302,10 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1; if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1; if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "maxTmrCtrl", tsMaxTmrCtrl, 8, 2048, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcTimer", tsRpcTimer, 100, 3000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcMaxTime", tsRpcMaxTime, 100, 7200, 1) != 0) return -1; - if (cfgAddBool(pCfg, "rpcForceTcp", tsRpcForceTcp, 1) != 0) return -1; if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1) != 0) return -1; if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "maxWildCardsLength", tsMaxWildCardsLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "maxRegexStringLen", tsMaxRegexStringLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "maxNumOfOrderedRes", tsMaxNumOfOrderedResults, 128, TSDB_MAX_ALLOWED_SQL_LEN, 1) != 0) - return -1; if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 4; @@ -508,18 +487,10 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { return -1; } - tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32; - tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32; - tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32; - tsRpcForceTcp = cfgGetItem(pCfg, "rpcForceTcp")->i32; tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; - tsMaxWildCardsLen = cfgGetItem(pCfg, "maxWildCardsLength")->i32; - tsMaxRegexStringLen = cfgGetItem(pCfg, "maxRegexStringLen")->i32; - tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32; tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; - tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32; tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; return 0; @@ -710,6 +681,6 @@ void taosCfgDynamicOptions(const char *option, const char *value) { if (strcasecmp(option, "resetlog") == 0) { taosResetLog(); - cfgDumpCfg(tsCfg, 1, false); + cfgDumpCfg(tsCfg, 0, false); } } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 0d3009fc65815bcb47e3e9820768cda3a98899d8..efd4cbfcecc53176f658c66548523ae1b8f140b1 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1567,13 +1567,8 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq if (tEncodeI32(&encoder, pReq->codeLen) < 0) return -1; if (tEncodeI64(&encoder, pReq->signature) < 0) return -1; - int32_t codeSize = 0; if (pReq->pCode != NULL) { - codeSize = strlen(pReq->pCode) + 1; - } - if (tEncodeI32(&encoder, codeSize) < 0) return -1; - if (pReq->pCode != NULL) { - if (tEncodeCStr(&encoder, pReq->pCode) < 0) return -1; + if (tEncodeBinary(&encoder, pReq->pCode, pReq->codeLen) < 0) return -1; } int32_t commentSize = 0; @@ -1607,10 +1602,8 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR if (tDecodeI32(&decoder, &pReq->codeLen) < 0) return -1; if (tDecodeI64(&decoder, &pReq->signature) < 0) return -1; - int32_t codeSize = 0; - if (tDecodeI32(&decoder, &codeSize) < 0) return -1; - if (codeSize > 0) { - pReq->pCode = taosMemoryCalloc(1, codeSize); + if (pReq->codeLen > 0) { + pReq->pCode = taosMemoryCalloc(1, pReq->codeLen); if (pReq->pCode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -1733,7 +1726,7 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp * if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1; if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1; if (pInfo->codeSize) { - if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1; + if (tEncodeBinary(&encoder, pInfo->pCode, pInfo->codeSize) < 0) return -1; } if (pInfo->commentSize) { if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1; @@ -2090,10 +2083,15 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp if (tStartDecode(&decoder) < 0) return -1; int32_t num = 0; if (tDecodeI32(&decoder, &num) < 0) return -1; - pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr)); - if (NULL == pRsp->addrsList) return -1; + if (NULL == pRsp->addrsList) { + pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr)); + if (NULL == pRsp->addrsList) return -1; + } + for (int32_t i = 0; i < num; ++i) { - if (tDecodeSQueryNodeAddr(&decoder, TARRAY_GET_ELEM(pRsp->addrsList, i)) < 0) return -1; + SQueryNodeAddr addr = {0}; + if (tDecodeSQueryNodeAddr(&decoder, &addr) < 0) return -1; + taosArrayPush(pRsp->addrsList, &addr); } tEndDecode(&decoder); diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index b8ace4bf50a401b1cc230b289a21193a824e63c8..4b01c540f7e6019240aa3eff0ab547705420275e 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -106,7 +106,7 @@ static void dmPrintVersion() { static void dmDumpCfg() { SConfig *pCfg = taosGetCfg(); - cfgDumpCfg(pCfg, 0, 1); + cfgDumpCfg(pCfg, 0, true); } static SDnodeOpt dmGetOpt() { diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index e76d7fcd4f0393601988f0060ddf0766e928e33e..ca1b943fb22b478533036236352cb9907ae1ac8d 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -311,6 +311,9 @@ static void dmWatchUdfd(void *args) { } static int32_t dmStartUdfd(SDnode *pDnode) { + char dnodeId[8] = {0}; + snprintf(dnodeId, sizeof(dnodeId), "%d", pDnode->data.dnodeId); + uv_os_setenv("DNODE_ID", dnodeId); SUdfdData *pData = &pDnode->udfdData; if (pData->startCalled) { dInfo("dnode-mgmt start udfd already called"); @@ -320,8 +323,17 @@ static int32_t dmStartUdfd(SDnode *pDnode) { uv_barrier_init(&pData->barrier, 2); uv_thread_create(&pData->thread, dmWatchUdfd, pDnode); uv_barrier_wait(&pData->barrier); - pData->needCleanUp = true; - return pData->spawnErr; + int32_t err = atomic_load_32(&pData->spawnErr); + if (err != 0) { + uv_barrier_destroy(&pData->barrier); + uv_async_send(&pData->stopAsync); + uv_thread_join(&pData->thread); + pData->needCleanUp = false; + dInfo("dnode-mgmt udfd cleaned up after spawn err"); + } else { + pData->needCleanUp = true; + } + return err; } static int32_t dmStopUdfd(SDnode *pDnode) { @@ -336,7 +348,7 @@ static int32_t dmStopUdfd(SDnode *pDnode) { uv_barrier_destroy(&pData->barrier); uv_async_send(&pData->stopAsync); uv_thread_join(&pData->thread); - + dInfo("dnode-mgmt udfd cleaned up"); return 0; } @@ -371,9 +383,9 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { } dmReportStartup(pDnode, "dnode-transport", "initialized"); -// if (dmStartUdfd(pDnode) != 0) { -// dError("failed to start udfd"); -// } + if (dmStartUdfd(pDnode) != 0) { + dError("failed to start udfd"); + } dInfo("dnode-mgmt is initialized"); return 0; diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index 5eacffbdf7c187d2a81f9a72dffc65fa5ef33034..fb8e5e7fb23d39353eb317da9dec72416da8b01e 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -140,6 +140,12 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { return; } + if (msgType == TDMT_DND_NET_TEST) { + dTrace("net test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); + dmProcessServerStatusReq(pDnode, pMsg); + return; + } + if (pDnode->status != DND_STAT_RUNNING) { dError("msg:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); if (isReq) { diff --git a/source/dnode/mgmt/interface/inc/dmInt.h b/source/dnode/mgmt/interface/inc/dmInt.h index a2368f31735be850a03dcb98bc14613cc7f23d27..63bfaf5ad24809043f0a9ea83a6815310aa4d592 100644 --- a/source/dnode/mgmt/interface/inc/dmInt.h +++ b/source/dnode/mgmt/interface/inc/dmInt.h @@ -37,6 +37,7 @@ void dmSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgF void dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); void dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc); void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pMsg); +void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pMsg); void dmGetMonitorSysInfo(SMonSysInfo *pInfo); // dmFile.c diff --git a/source/dnode/mgmt/interface/src/dmInt.c b/source/dnode/mgmt/interface/src/dmInt.c index bab84effe572bb00246fba0e97751805abc14217..2d15a7a008c01f0035b26eb2396b4cc3e3efa506 100644 --- a/source/dnode/mgmt/interface/src/dmInt.c +++ b/source/dnode/mgmt/interface/src/dmInt.c @@ -171,6 +171,18 @@ static void dmGetServerStatus(SDnode *pDnode, SServerStatusRsp *pStatus) { } } +void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pRpc) { + dDebug("net test req is received"); + SRpcMsg rsp = {.handle = pRpc->handle, .refId = pRpc->refId, .ahandle = pRpc->ahandle, .code = 0}; + rsp.pCont = rpcMallocCont(pRpc->contLen); + if (rsp.pCont == NULL) { + rsp.code = TSDB_CODE_OUT_OF_MEMORY; + } else { + rsp.contLen = pRpc->contLen; + } + rpcSendResponse(&rsp); +} + void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pReq) { dDebug("server status req is received"); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index b434cf102e130737fa4493eb204613cb8fee602b..afe57e3d8ffd78f7d9684cbd7e52a264a1343668 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -177,6 +177,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_MND_QNODE_LIST, mmProcessReadMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index e784272581ff0c460ad50d622095af9b9303a8a5..687a799c575baeb13603f2d37edcb8e4e6895bd2 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" +#include "libs/function/function.h" SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId) { SVnodeObj *pVnode = NULL; @@ -275,7 +276,7 @@ static void vmCleanup(SMgmtWrapper *pWrapper) { pWrapper->pMgmt = NULL; // syncCleanUp(); - + udfcClose(); dInfo("vnode-mgmt is cleaned up"); } @@ -339,6 +340,10 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) { } dmReportStartup(pDnode, "vnode-vnodes", "initialized"); + if (udfcOpen() != 0) { + dError("failed to open udfc in dnode"); + } + code = 0; _OVER: diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 156d894a442a5e48cd54eb7a4d652190100df47d..09d05870192db3e494118d90265309b07c54d567 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -309,10 +309,10 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) { goto _OVER; } - if (createReq.pCode[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; - goto _OVER; - } + if (createReq.codeLen <= 1) { + terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; + goto _OVER; + } if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 1c03ca30f44ec969f57e834cce08d0c44e5d2920..cec5933ba32b2e2f3fbcfb8a5ab48a4f8109ad18 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -451,8 +451,9 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) { goto _OVER; } + void *pIter = NULL; while (1) { - void *pIter = sdbFetch(pSdb, SDB_QNODE, NULL, (void **)&pObj); + pIter = sdbFetch(pSdb, SDB_QNODE, pIter, (void **)&pObj); if (pIter == NULL) break; SQueryNodeAddr nodeAddr = {0}; @@ -472,7 +473,7 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) { } int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp); - void *pRsp = taosMemoryMalloc(rspLen); + void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index daf8dd431fad5d85a39c0a64da260db43b35027b..75caef2336e4b15388ae5a992b370e3cf44fe7ae 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -385,7 +385,11 @@ int32_t mndProcessMsg(SNodeMsg *pMsg) { terrno = code; mTrace("msg:%p, in progress, app:%p", pMsg, ahandle); } else if (code != 0) { - mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); + if (terrno != TSDB_CODE_OPS_NOT_SUPPORT) { + mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); + } else { + mTrace("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); + } } else { mTrace("msg:%p, is processed, app:%p", pMsg, ahandle); } diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 4db9411e877e79e756c8ab754208f8610cbd9da3..0473fa375e7ceef9bb8d0b3702b83e8684363871 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -22,17 +22,16 @@ class MndTestFunc : public ::testing::Test { void SetUp() override {} void TearDown() override {} - void SetCode(SCreateFuncReq* pReq, const char* pCode); + void SetCode(SCreateFuncReq* pReq, const char* pCode, int32_t size); void SetComment(SCreateFuncReq* pReq, const char* pComment); }; Testbase MndTestFunc::test; -void MndTestFunc::SetCode(SCreateFuncReq* pReq, const char* pCode) { - int32_t len = strlen(pCode); - pReq->pCode = (char*)taosMemoryCalloc(1, len + 1); - strcpy(pReq->pCode, pCode); - pReq->codeLen = len; +void MndTestFunc::SetCode(SCreateFuncReq *pReq, const char *pCode, int32_t size) { + pReq->pCode = (char*)taosMemoryMalloc(size); + memcpy(pReq->pCode, pCode, size); + pReq->codeLen = size; } void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) { @@ -79,7 +78,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - SetCode(&createReq, ""); + SetCode(&createReq, "", 1); SetComment(&createReq, "comment1"); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); @@ -95,7 +94,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - SetCode(&createReq, "code1"); + SetCode(&createReq, "code1", 6); SetComment(&createReq, "comment1"); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); @@ -111,7 +110,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - SetCode(&createReq, "code1"); + SetCode(&createReq, "code1", 6); SetComment(&createReq, "comment1"); createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1; @@ -128,7 +127,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { for (int32_t i = 0; i < 3; ++i) { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - SetCode(&createReq, "code1"); + SetCode(&createReq, "code1", 6); SetComment(&createReq, "comment1"); createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1; createReq.igExists = 0; @@ -253,7 +252,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { createReq.outputLen = 24; createReq.bufSize = 6; createReq.signature = 18; - SetCode(&createReq, "code2"); + SetCode(&createReq, "code2", 6); SetComment(&createReq, "comment2"); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); @@ -439,3 +438,70 @@ TEST_F(MndTestFunc, 04_Drop_Func) { test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); EXPECT_EQ(test.GetShowRows(), 1); } + +TEST_F(MndTestFunc, 05_Actual_code) { + { + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "udf1"); + char code[300] = {0}; + for (int32_t i = 0; i < sizeof(code); ++i) { + code[i] = (i) % 20; + } + SetCode(&createReq, code, 300); + SetComment(&createReq, "comment1"); + createReq.bufSize = 8; + createReq.igExists = 0; + createReq.funcType = 1; + createReq.scriptType = 2; + createReq.outputType = TSDB_DATA_TYPE_SMALLINT; + createReq.outputLen = 12; + createReq.bufSize = 4; + createReq.signature = 5; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); + tFreeSCreateFuncReq(&createReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 1; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, "udf1"); + + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + tFreeSRetrieveFuncReq(&retrieveReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + SRetrieveFuncRsp retrieveRsp = {0}; + tDeserializeSRetrieveFuncRsp(pRsp->pCont, pRsp->contLen, &retrieveRsp); + EXPECT_EQ(retrieveRsp.numOfFuncs, 1); + EXPECT_EQ(retrieveRsp.numOfFuncs, (int32_t)taosArrayGetSize(retrieveRsp.pFuncInfos)); + + SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0); + + EXPECT_STREQ(pFuncInfo->name, "udf1"); + EXPECT_EQ(pFuncInfo->funcType, 1); + EXPECT_EQ(pFuncInfo->scriptType, 2); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT); + EXPECT_EQ(pFuncInfo->outputLen, 12); + EXPECT_EQ(pFuncInfo->bufSize, 4); + EXPECT_EQ(pFuncInfo->signature, 5); + EXPECT_STREQ("comment1", pFuncInfo->pComment); + for (int32_t i = 0; i < 300; ++i) { + EXPECT_EQ(pFuncInfo->pCode[i], (i) % 20); + } + tFreeSRetrieveFuncRsp(&retrieveRsp); + } + +} \ No newline at end of file diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 7b299f1f3ce3fca46e2205f3a8c0226096ada826..907fddaec2529385856f09add338fa91159e7c69 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -17,6 +17,7 @@ #include "qndInt.h" #include "query.h" #include "qworker.h" +//#include "tudf.h" SQnode *qndOpen(const SQnodeOpt *pOption) { SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); @@ -25,6 +26,8 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { return NULL; } + //udfcOpen(); + if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) { taosMemoryFreeClear(pQnode); return NULL; @@ -37,13 +40,15 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { void qndClose(SQnode *pQnode) { qWorkerDestroy((void **)&pQnode->pQuery); + //udfcClose(); + taosMemoryFree(pQnode); } int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { - qTrace("message in query queue is processing"); + qTrace("message in qnode query queue is processing"); SReadHandle handle = {0}; switch (pMsg->msgType) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f70a4478b3a3c740cf8adff91b5a0d225e33e149..098b86975b7869aa59b61abf8129ffcbe7581b35 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3223,8 +3223,13 @@ void tsdbRetrieveDataBlockInfo(tsdbReaderT* pTsdbReadHandle, SDataBlockInfo* pDa tsdbDebug("data block generated, uid:%" PRIu64 " numOfRows:%d, tsrange:%" PRId64 " - %" PRId64 " %s", uid, cur->rows, cur->win.skey, cur->win.ekey, pHandle->idStr); - // pDataBlockInfo->uid = uid; // block Id may be over write by assigning uid fro this data block. Do NOT assign - // the table uid + pDataBlockInfo->uid = uid; + +#if 0 + // for multi-group data query processing test purpose + pDataBlockInfo->groupId = uid; +#endif + pDataBlockInfo->rows = cur->rows; pDataBlockInfo->window = cur->win; pDataBlockInfo->numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pHandle)); diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 273b7447ffe781075a3956a83237e9f563eb88e4..bc9fe46c0f34f1b9b293af212c221064752b7602 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -1510,7 +1510,6 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, } STSma *pTSma = pItem->pSma; - #endif STSmaReadH tReadH = {0}; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index aa90a04478ab37d5f377903d36c7b99d43411f29..e3c504c93d9195eec1111965fc65c05119d3f853 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -135,7 +135,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg } int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { - vTrace("message in query queue is processing"); + vTrace("message in vnode query queue is processing"); SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config}; switch (pMsg->msgType) { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 410527c9e6ac48c52e5926abfde59e321557eb7e..272c370ca3943c65d6163e9d1a169d19c9b80a82 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -494,7 +494,7 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray **out) { +int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray *out) { char *msg = NULL; int32_t msgLen = 0; @@ -526,7 +526,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt CTG_ERR_RET(code); } - ctgDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(*out)); + ctgDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(out)); return TSDB_CODE_SUCCESS; } @@ -2778,7 +2778,8 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, } if (pReq->qNodeRequired) { - CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pTrans, pMgmtEps, &pRsp->pEpSetList)); + pRsp->pQnodeList = taosArrayInit(10, sizeof(SQueryNodeAddr)); + CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pTrans, pMgmtEps, pRsp->pQnodeList)); } CTG_API_LEAVE(TSDB_CODE_SUCCESS); @@ -2807,7 +2808,7 @@ int32_t catalogGetQnodeList(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, &pQnodeList)); + CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, pQnodeList)); _return: diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 75084e76106ac69494605dd434370c7ef59a8d79..d17a5a59a444776c8465dac0487c6d73a2a29506 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -304,8 +304,8 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t memcpy(row.buf, tbuf, len); row.level = level; - row.len = len; - ctx->dataSize += len; + row.len = len; + ctx->dataSize += row.len; if (NULL == taosArrayPush(ctx->rows, &row)) { qError("taosArrayPush row to explain res rows failed"); @@ -756,7 +756,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { } int32_t colNum = 1; - int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; + int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); if (NULL == rsp) { qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); @@ -766,29 +766,38 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htonl(rowNum); - *(int32_t *)rsp->data = htonl(pCtx->dataSize); + // payload length + *(int32_t *)rsp->data = sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; - int32_t *offset = (int32_t *)((char *)rsp->data + sizeof(int32_t)); + // group id + *(uint64_t*)(rsp->data + sizeof(int32_t)) = 0; + + // column length + int32_t* colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t)); + + // varchar column offset segment + int32_t *offset = (int32_t *)((char *)colLength + sizeof(int32_t)); + + // varchar data real payload char *data = (char *)(offset + rowNum); - int32_t tOffset = 0; - + + char* start = data; for (int32_t i = 0; i < rowNum; ++i) { SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); - *offset = tOffset; - tOffset += row->len; + offset[i] = data - start; - memcpy(data, row->buf, row->len); - - ++offset; + varDataCopy(data, row->buf); + ASSERT(varDataTLen(row->buf) == row->len); data += row->len; } - *pRsp = rsp; + *colLength = htonl(data - start); + rsp->compLen = htonl(rspSize); + *pRsp = rsp; return TSDB_CODE_SUCCESS; } - int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { int32_t code = 0; SNodeListNode *plans = NULL; @@ -895,9 +904,7 @@ int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) { int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { QRY_ERR_RET(qExplainAppendGroupResRows(pCtx, pCtx->rootGroupId, 0)); - QRY_ERR_RET(qExplainAppendPlanRows(pCtx)); - QRY_ERR_RET(qExplainGetRspFromCtx(pCtx, pRsp)); return TSDB_CODE_SUCCESS; @@ -967,13 +974,10 @@ int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { SExplainCtx *pCtx = NULL; QRY_ERR_RET(qExplainPrepareCtx(pDag, &pCtx)); - QRY_ERR_JRET(qExplainGenerateRsp(pCtx, pRsp)); _return: - qExplainFreeCtx(pCtx); - QRY_RET(code); } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 834d37927a932aeabccaacf3edfc1fffe8594ad3..f0cb1c1107e8f473764472b44b25b0299a57dbf8 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -40,8 +40,6 @@ #define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str) -#define curTimeWindowIndex(_winres) ((_winres)->curIndex) - typedef struct SGroupResInfo { int32_t totalGroup; int32_t currentGroup; @@ -68,11 +66,16 @@ typedef struct SResultRowPosition { int32_t offset; } SResultRowPosition; +typedef struct SResKeyPos { + SResultRowPosition pos; + uint64_t groupId; + char key[]; +} SResKeyPos; + typedef struct SResultRowInfo { SResultRowPosition *pPosition; int32_t size; // number of result set int32_t capacity; // max capacity -// int32_t curPos; // current active result row index of pResult list SResultRowPosition cur; } SResultRowInfo; @@ -135,7 +138,7 @@ typedef struct { int32_t colId; } SStddevInterResult; -void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo); +void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, bool sortGroupResult); void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index c6d209e706744ae66a5be15d67b6c0cbafa1af78..db0dd1ffbb8e099f0743dc2e5ab55dd37a2cec02 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -347,10 +347,11 @@ typedef struct STableScanInfo { } STableScanInfo; typedef struct STagScanInfo { - SColumnInfo* pCols; - SSDataBlock* pRes; + SColumnInfo *pCols; + SSDataBlock *pRes; int32_t totalTables; int32_t curPos; + void *pReader; } STagScanInfo; typedef struct SStreamBlockScanInfo { @@ -376,13 +377,11 @@ typedef struct SSysTableScanInfo { SEpSet epSet; tsem_t ready; - int32_t accountId; - bool showRewrite; - SNode* pCondition; // db_name filter condition, to discard data that are not in current database - void* pCur; // cursor for iterate the local table meta store. - SArray* scanCols; // SArray scan column id list - -// int32_t type; // show type, TODO remove it + int32_t accountId; + bool showRewrite; + SNode* pCondition; // db_name filter condition, to discard data that are not in current database + void* pCur; // cursor for iterate the local table meta store. + SArray* scanCols; // SArray scan column id list SName name; SSDataBlock* pRes; int32_t capacity; @@ -628,7 +627,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfCols, int32_t dataLoadFlag, int32_t repeatTime, +SOperatorInfo* createTableScanOperatorInfo(void* pReaderHandle, int32_t order, int32_t numOfCols, int32_t dataLoadFlag, int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, SInterval* pInterval, double ratio, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, @@ -668,12 +667,12 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); #if 0 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createTagScanOperatorInfo(SReaderHandle* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput); #endif void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index e897bc8892e594368cc8a1b1d9a0f33d91f03c46..a5ce01dba28c07b0354d6e69c595048da511cd99 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -64,10 +64,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { } // data format: -// +----------------+--------------------------------------+-------------+-----------+-------------+-----------+ -// |SDataCacheEntry | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... -// | | sizeof(int32_t) * numOfCols | actual size | | actual size | | -// +----------------+--------------------------------------+-------------+-----------+-------------+-----------+ +// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+ +// |SDataCacheEntry | total length | group id | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... +// | | (4 bytes) |(8 bytes) | sizeof(int32_t) * numOfCols | actual size | | actual size | | +// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+ // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 98b72cf4c25a0deb6e25a21877aeb90b4f66d493..c3fa777779c979799e281761816a48698aebfc56 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -186,12 +186,50 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { pGroupResInfo->index = 0; } -void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo) { +static int32_t resultrowCompar1(const void* p1, const void* p2) { + SResKeyPos* pp1 = *(SResKeyPos**) p1; + SResKeyPos* pp2 = *(SResKeyPos**) p2; + + if (pp1->groupId == pp2->groupId) { + int64_t pts1 = *(int64_t*) pp1->key; + int64_t pts2 = *(int64_t*) pp2->key; + + if (pts1 == pts2) { + return 0; + } else { + return pts1 < pts2? -1:1; + } + } else { + return pp1->groupId < pp2->groupId? -1:1; + } +} + +void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, bool sortGroupResult) { if (pGroupResInfo->pRows != NULL) { taosArrayDestroy(pGroupResInfo->pRows); } - pGroupResInfo->pRows = taosArrayFromList(pResultInfo->pPosition, pResultInfo->size, sizeof(SResultRowPosition)); + // extract the result rows information from the hash map + void* pData = NULL; + pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES); + + size_t keyLen = 0; + while((pData = taosHashIterate(pHashmap, pData)) != NULL) { + void* key = taosHashGetKey(pData, &keyLen); + + SResKeyPos* p = taosMemoryMalloc(keyLen + sizeof(SResultRowPosition)); + + p->groupId = *(uint64_t*) key; + p->pos = *(SResultRowPosition*) pData; + memcpy(p->key, key + sizeof(uint64_t), keyLen - sizeof(uint64_t)); + + taosArrayPush(pGroupResInfo->pRows, &p); + } + + if (sortGroupResult) { + qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, resultrowCompar1); + } + pGroupResInfo->index = 0; assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 32c3a60b0fa6f1b2e023246adddeb7465fcc8221..6a5f5b2dc348545f4c350932b55a0b143b201696 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -235,7 +235,6 @@ static int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDi static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo); -static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); static void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* pTaskInfo); @@ -298,26 +297,6 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { return pBlock; } -static bool isSelectivityWithTagsQuery(SqlFunctionCtx* pCtx, int32_t numOfOutput) { - return true; - // bool hasTags = false; - // int32_t numOfSelectivity = 0; - // - // for (int32_t i = 0; i < numOfOutput; ++i) { - // int32_t functId = pCtx[i].functionId; - // if (functId == FUNCTION_TAG_DUMMY || functId == FUNCTION_TS_DUMMY) { - // hasTags = true; - // continue; - // } - // - // if ((aAggs[functId].status & FUNCSTATE_SELECTIVITY) != 0) { - // numOfSelectivity++; - // } - // } - // - // return (numOfSelectivity > 0 && hasTags); -} - static bool hasNull(SColumn* pColumn, SColumnDataAgg* pStatis) { if (TSDB_COL_IS_TAG(pColumn->flag) || TSDB_COL_IS_UD_COL(pColumn->flag) || pColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { @@ -433,6 +412,13 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, return pResultRow; } +/** + * the struct of key in hash table + * +----------+---------------+ + * | group id | key data | + * | 8 bytes | actual length | + * +----------+---------------+ + */ static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, int64_t uid, char* pData, int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) { @@ -1098,8 +1084,9 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { - pCtx[i].order = order; - pCtx[i].size = pBlock->info.rows; + pCtx[i].order = order; + pCtx[i].size = pBlock->info.rows; + pCtx[i].pSrcBlock = pBlock; pCtx[i].currentStage = MAIN_SCAN; SInputColumnInfoData* pInput = &pCtx[i].input; @@ -1474,7 +1461,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe SArray* pUpdated = NULL; if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - pUpdated = taosArrayInit(4, sizeof(SResultRowPosition)); + pUpdated = taosArrayInit(4, POINTER_BYTES); } int32_t step = 1; @@ -1486,8 +1473,6 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; - // assert(tsCols[0] == pSDataBlock->info.window.skey && tsCols[pSDataBlock->info.rows - 1] == - // pSDataBlock->info.window.ekey); } int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); @@ -1506,7 +1491,11 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*) pos->key = pResult->win.skey; + taosArrayPush(pUpdated, &pos); } @@ -1580,7 +1569,11 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*) pos->key = pResult->win.skey; + taosArrayPush(pUpdated, &pos); } @@ -1844,10 +1837,6 @@ void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* // set the output buffer for the selectivity + tag query static int32_t setCtxTagColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) { - if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) { - return TSDB_CODE_SUCCESS; - } - int32_t num = 0; int16_t tagLen = 0; @@ -1874,9 +1863,8 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } } if (p != NULL) { - p->subsidiaryRes.pCtx = pTagCtx; - p->subsidiaryRes.numOfCols = num; - p->subsidiaryRes.bufLen = tagLen; + p->subsidiaries.pCtx = pTagCtx; + p->subsidiaries.num = num; } else { taosMemoryFreeClear(pTagCtx); } @@ -1903,6 +1891,9 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, SqlFunctionCtx* pCtx = &pFuncCtx[i]; pCtx->functionId = -1; + pCtx->curBufPage = -1; + pCtx->pExpr = pExpr; + if (pExpr->pExpr->nodeType == QUERY_NODE_FUNCTION) { SFuncExecEnv env = {0}; pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId; @@ -1930,9 +1921,9 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, pCtx->pTsOutput = NULL; pCtx->resDataInfo.bytes = pFunct->resSchema.bytes; pCtx->resDataInfo.type = pFunct->resSchema.type; - pCtx->order = TSDB_ORDER_ASC; + pCtx->order = TSDB_ORDER_ASC; pCtx->start.key = INT64_MIN; - pCtx->end.key = INT64_MIN; + pCtx->end.key = INT64_MIN; pCtx->numOfParams = pExpr->base.numOfParams; pCtx->param = pFunct->pParam; @@ -1992,7 +1983,7 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } taosVariantDestroy(&pCtx[i].tag); - taosMemoryFreeClear(pCtx[i].subsidiaryRes.pCtx); + taosMemoryFreeClear(pCtx[i].subsidiaries.pCtx); } taosMemoryFreeClear(pCtx); @@ -2095,25 +2086,6 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { return status; } -static void doUpdateLastKey(STaskAttr* pQueryAttr) { - STimeWindow* win = &pQueryAttr->window; - - size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList); - for (int32_t i = 0; i < num; ++i) { - SArray* p1 = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); - - size_t len = taosArrayGetSize(p1); - for (int32_t j = 0; j < len; ++j) { - // STableKeyInfo* pInfo = taosArrayGet(p1, j); - // - // // update the new lastkey if it is equalled to the value of the old skey - // if (pInfo->lastKey == win->ekey) { - // pInfo->lastKey = win->skey; - // } - } - } -} - // static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool stableQuery) { // STaskAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; // @@ -2848,6 +2820,7 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { } } +// todo merged with the build group result. void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { for (int32_t i = 0; i < pResultRowInfo->size; ++i) { @@ -2882,40 +2855,36 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD } } +// todo merged with the build group result. void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList, int32_t* rowCellInfoOffset) { size_t num = taosArrayGetSize(pUpdateList); for (int32_t i = 0; i < num; ++i) { - SResultRowPosition* pPos = taosArrayGet(pUpdateList, i); - - SFilePage* bufPage = getBufPage(pBuf, pPos->pageId); - SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->offset); + SResKeyPos * pPos = taosArrayGetP(pUpdateList, i); + SFilePage* bufPage = getBufPage(pBuf, pPos->pos.pageId); + SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset); +// for (int32_t j = 0; j < numOfOutput; ++j) { pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset); - +// struct SResultRowEntryInfo* pResInfo = pCtx[j].resultInfo; - if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { - continue; - } - - if (pCtx[j].fpSet.process) { // TODO set the dummy function. - // pCtx[j].fpSet.finalize(&pCtx[j]); - pResInfo->initialized = true; - } - +// if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { +// continue; +// } +// +// if (pCtx[j].fpSet.process) { // TODO set the dummy function. +//// pCtx[j].fpSet.finalize(&pCtx[j]); +// pResInfo->initialized = true; +// } +// if (pRow->numOfRows < pResInfo->numOfRes) { pRow->numOfRows = pResInfo->numOfRes; } } releaseBufPage(pBuf, bufPage); - /* - * set the number of output results for group by normal columns, the number of output rows usually is 1 except - * the top and bottom query - */ - // buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput); } } @@ -3062,33 +3031,6 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p pAggInfo->groupId = groupId; } -void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - SExprBasicInfo* pExpr = &pExprInfo->base; - // if (pQueryAttr->stableQuery && (pRuntimeEnv->pTsBuf != NULL) && - // (pExpr->functionId == FUNCTION_TS || pExpr->functionId == FUNCTION_PRJ) && - // (pExpr->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_ID)) { - // assert(pExpr->numOfParams == 1); - // - // int16_t tagColId = (int16_t)pExprInfo->base.param[0].i; - // SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId); - // - // doSetTagValueInParam(pTable, tagColId, &pCtx->tag, pColInfo->type, pColInfo->bytes); - // - // int16_t tagType = pCtx[0].tag.nType; - // if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { - // //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", - // GET_TASKID(pRuntimeEnv), - //// pExprInfo->base.param[0].i, pCtx[0].tag.pz); - // } else { - // //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, - // GET_TASKID(pRuntimeEnv), - //// pExprInfo->base.param[0].i, pCtx[0].tag.i); - // } - // } -} - /* * There are two cases to handle: * @@ -3131,7 +3073,6 @@ void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWin } /** - * copyToOutputBuf support copy data in ascending/descending order * For interval query of both super table and table, copy the data in ascending order, since the output results are * ordered in SWindowResutl already. While handling the group by query for both table and super table, * all group result are completed already. @@ -3159,10 +3100,10 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased } for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { - SResultRowPosition* pPos = taosArrayGet(pGroupResInfo->pRows, i); - SFilePage* page = getBufPage(pBuf, pPos->pageId); + SResKeyPos *pPos = taosArrayGetP(pGroupResInfo->pRows, i); + SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); - SResultRow* pRow = (SResultRow*)((char*)page + pPos->offset); + SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; continue; @@ -3181,7 +3122,7 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased pCtx[j].resultInfo = getResultCell(pRow, j, rowCellOffset); if (pCtx[j].fpSet.process) { - pCtx[j].fpSet.finalize(&pCtx[j], pBlock, slotId); + pCtx[j].fpSet.finalize(&pCtx[j], pBlock); } else { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); @@ -3806,9 +3747,15 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI blockDataEnsureCapacity(pRes, numOfRows); if (pColList == NULL) { // data from other sources + int32_t dataLen = *(int32_t*) pData; + pData += sizeof(int32_t); + + pRes->info.groupId = *(uint64_t*) pData; + pData += sizeof(uint64_t); + int32_t* colLen = (int32_t*)pData; - char* pStart = pData + sizeof(int32_t) * numOfOutput; + char* pStart = pData + sizeof(int32_t) * numOfOutput; for (int32_t i = 0; i < numOfOutput; ++i) { colLen[i] = htonl(colLen[i]); ASSERT(colLen[i] > 0); @@ -3861,7 +3808,11 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI blockDataEnsureCapacity(&block, numOfRows); - int32_t* colLen = (int32_t*)pStart; + int32_t dataLen = *(int32_t*) pStart; + uint64_t groupId = *(uint64_t*) (pStart + sizeof(int32_t)); + pStart += sizeof(int32_t) + sizeof(uint64_t); + + int32_t* colLen = (int32_t*) (pStart); pStart += sizeof(int32_t) * numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { @@ -3903,6 +3854,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI } pRes->info.rows = numOfRows; + blockDataUpdateTsWindow(pRes); int64_t el = taosGetTimestampUs() - startTs; @@ -4318,18 +4270,6 @@ static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { // } } -static SExprInfo* exprArrayDup(SArray* pExprList) { - size_t numOfOutput = taosArrayGetSize(pExprList); - - SExprInfo* p = taosMemoryCalloc(numOfOutput, sizeof(SExprInfo)); - for (int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExpr = taosArrayGetP(pExprList, i); - assignExprInfo(&p[i], pExpr); - } - - return p; -} - // TODO merge aggregate super table static void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { @@ -4822,7 +4762,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { finalizeMultiTupleQueryResult(pAggInfo->binfo.pCtx, pOperator->numOfOutput, pAggInfo->aggSup.pResultBuf, &pAggInfo->binfo.resultRowInfo, pAggInfo->binfo.rowCellInfoOffset); - initGroupResInfo(&pAggInfo->groupResInfo, &pAggInfo->binfo.resultRowInfo); + initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, false); OPTR_SET_OPENED(pOperator); return TSDB_CODE_SUCCESS; } @@ -5125,7 +5065,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId); #if 0 // test for encode/decode result info if(pOperator->encodeResultRow){ @@ -5147,7 +5087,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); OPTR_SET_OPENED(pOperator); return TSDB_CODE_SUCCESS; } @@ -5276,7 +5216,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); // finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput); - initGroupResInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); +// initGroupedResultInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes); if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { @@ -5327,7 +5267,7 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); +// initGroupedResultInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); OPTR_SET_OPENED(pOperator); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); @@ -5458,7 +5398,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); - initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); @@ -5510,7 +5450,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); - initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); @@ -5699,6 +5639,11 @@ int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInf pBasicInfo->pRes = pResultBlock; doInitAggInfoSup(pAggSup, pBasicInfo->pCtx, numOfCols, keyBufSize, pkey); + + for(int32_t i = 0; i < numOfCols; ++i) { + pBasicInfo->pCtx[i].pBuf = pAggSup->pResultBuf; + } + return TSDB_CODE_SUCCESS; } @@ -5847,11 +5792,6 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { doDestroyBasicInfo(&pInfo->binfo, numOfOutput); } -static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { - STagScanInfo* pInfo = (STagScanInfo*)param; - pInfo->pRes = blockDataDestroy(pInfo->pRes); -} - static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param; pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock); @@ -5941,11 +5881,12 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; - pInfo->execModel = pTaskInfo->execModel; - pInfo->win = pTaskInfo->window; - pInfo->twAggSup = *pTwAggSupp; + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; +// pInfo->execModel = OPTR_EXEC_MODEL_STREAM; + pInfo->execModel = pTaskInfo->execModel; + pInfo->win = pTaskInfo->window; + pInfo->twAggSup = *pTwAggSupp; pInfo->primaryTsIndex = primaryTsSlotId; int32_t numOfRows = 4096; @@ -6204,157 +6145,6 @@ _error: return NULL; } -static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) { -#if 0 - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; - - STagScanInfo *pInfo = pOperator->info; - SSDataBlock *pRes = pInfo->pRes; - *newgroup = false; - - int32_t count = 0; - SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); - - int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]); - if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id - assert(pQueryAttr->numOfOutput == 1); - - SExprInfo* pExprInfo = &pOperator->pExpr[0]; - int32_t rsize = pExprInfo->base.resSchema.bytes; - - count = 0; - - int16_t bytes = pExprInfo->base.resSchema.bytes; - int16_t type = pExprInfo->base.resSchema.type; - - for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { - if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) { - bytes = pQueryAttr->tagColList[i].bytes; - type = pQueryAttr->tagColList[i].type; - break; - } - } - - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; - STableQueryInfo *item = taosArrayGetP(pa, i); - - char *output = pColInfo->pData + count * rsize; - varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); - - output = varDataVal(output); - STableId* id = TSDB_TABLEID(item->pTable); - - *(int16_t *)output = 0; - output += sizeof(int16_t); - - *(int64_t *)output = id->uid; // memory align problem, todo serialize - output += sizeof(id->uid); - - *(int32_t *)output = id->tid; - output += sizeof(id->tid); - - *(int32_t *)output = pQueryAttr->vgId; - output += sizeof(pQueryAttr->vgId); - - char* data = NULL; - if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes); - } - - doSetTagValueToResultBuf(output, data, type, bytes); - count += 1; - } - - //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count); - } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - *(int64_t*)pColInfo->pData = pInfo->totalTables; - count = 1; - - pOperator->status = OP_EXEC_DONE; - //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); - } else { // return only the tags|table name etc. - SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo - - count = 0; - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; - - STableQueryInfo* item = taosArrayGetP(pa, i); - - char *data = NULL, *dst = NULL; - int16_t type = 0, bytes = 0; - for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { - // not assign value in case of user defined constant output column - if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) { - continue; - } - - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); - type = pExprInfo[j].base.resSchema.type; - bytes = pExprInfo[j].base.resSchema.bytes; - - if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); - } - - dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; - doSetTagValueToResultBuf(dst, data, type, bytes); - } - - count += 1; - } - - if (pInfo->curPos >= pInfo->totalTables) { - pOperator->status = OP_EXEC_DONE; - } - - //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); - } - - if (pOperator->status == OP_EXEC_DONE) { - setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED); - } - - pRes->info.rows = count; - return (pRes->info.rows == 0)? NULL:pInfo->pRes; - -#endif - return TSDB_CODE_SUCCESS; -} - -SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { - STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); - size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); - assert(numOfGroup == 0 || numOfGroup == 1); - - pInfo->curPos = 0; - - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - pOperator->name = "SeqTableTagScan"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->getNextFn = doTagScan; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->closeFn = destroyTagScanOperatorInfo; - - return pOperator; -} static int32_t getColumnIndexInSource(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr, SColumnInfo* pTagCols) { int32_t j = 0; @@ -6562,6 +6352,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t numOfCols = 0; tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); + if (pDataReader == NULL) { + return NULL; + } SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); @@ -6576,8 +6369,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo }; return createTableScanOperatorInfo(pDataReader, pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC, - numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq[0], pTableScanNode->scanSeq[1], pColList, - pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo); + numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq[0], pTableScanNode->scanSeq[1], pColList, + pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc); @@ -6585,17 +6378,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. - int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, - queryId, taskId); + int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); SArray* tableIdList = extractTableIdList(pTableGroupInfo); SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); int32_t numOfCols = 0; - SArray* pColList = - extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); - SOperatorInfo* pOperator = - createStreamScanOperatorInfo(pHandle->reader, pResBlock, pColList, tableIdList, pTaskInfo); + SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo); taosArrayDestroy(tableIdList); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -6621,6 +6411,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo for (int32_t i = 0; i < size; ++i) { SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); + if (ops[i] == NULL) { + return NULL; + } } SOperatorInfo* pOptr = NULL; @@ -6988,8 +6781,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead goto _complete; } - STableGroupInfo group = {0}; - (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &group); + (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoGroupInfo); if (NULL == (*pTaskInfo)->pRoot) { code = terrno; goto _complete; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 47936007101bb0b7ec4eb642582d764c872a887a..52bdbea8a56e2be76a4ac61062dce3dbad81fe37 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -308,7 +308,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou // } blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity); - initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, false); while(1) { doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3a9742d48aecd515c360bfbad7324f16fbdfdc42..a70e40287159134edaa5addf186790a5a3755633 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -271,9 +271,7 @@ static void setupEnvForReverseScan(STableScanInfo* pTableScanInfo, SqlFunctionCt static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { STableScanInfo* pTableScanInfo = pOperator->info; - SSDataBlock* pBlock = pTableScanInfo->pResBlock; - STableGroupInfo* pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo; - + SSDataBlock* pBlock = pTableScanInfo->pResBlock; *newgroup = false; while (tsdbNextDataBlock(pTableScanInfo->dataReader)) { @@ -284,18 +282,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { pTableScanInfo->numOfBlocks += 1; tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info); - // todo opt - // if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { - // STableQueryInfo** pTableQueryInfo = - // (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.uid, sizeof(pBlock->info.uid)); - // if (pTableQueryInfo == NULL) { - // break; - // } - // - // doTableQueryInfoTimeWindowCheck(pTaskInfo, *pTableQueryInfo, pTableScanInfo->order); - // } - - // this function never returns error? uint32_t status = 0; int32_t code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status); // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); @@ -308,6 +294,8 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { continue; } + // reset the block to be 0 by default, this blockId is assigned by physical plan and is used by direct upstream operator. + pBlock->info.blockId = 0; return pBlock; } @@ -405,11 +393,11 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, int32_t order, int pOperator->name = "TableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->getNextFn = doTableScan; - pOperator->pTaskInfo = pTaskInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->getNextFn = doTableScan; + pOperator->pTaskInfo = pTaskInfo; static int32_t cost = 0; pOperator->cost.openCost = ++cost; @@ -824,12 +812,12 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { int32_t tableNameSlotId = 1; SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, tableNameSlotId); - char* name = NULL; + char* tb = NULL; int32_t numOfRows = 0; char n[TSDB_TABLE_NAME_LEN] = {0}; - while ((name = metaTbCursorNext(pInfo->pCur)) != NULL) { - STR_TO_VARSTR(n, name); + while ((tb = metaTbCursorNext(pInfo->pCur)) != NULL) { + STR_TO_VARSTR(n, tb); colDataAppend(pTableNameCol, numOfRows, n, false); numOfRows += 1; if (numOfRows >= pInfo->capacity) { @@ -992,3 +980,167 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB return pOperator; } + +static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) { +#if 0 + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; + + STagScanInfo *pInfo = pOperator->info; + SSDataBlock *pRes = pInfo->pRes; + *newgroup = false; + + int32_t count = 0; + SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); + + int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]); + if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id + assert(pQueryAttr->numOfOutput == 1); + + SExprInfo* pExprInfo = &pOperator->pExpr[0]; + int32_t rsize = pExprInfo->base.resSchema.bytes; + + count = 0; + + int16_t bytes = pExprInfo->base.resSchema.bytes; + int16_t type = pExprInfo->base.resSchema.type; + + for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { + if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) { + bytes = pQueryAttr->tagColList[i].bytes; + type = pQueryAttr->tagColList[i].type; + break; + } + } + + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); + + while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { + int32_t i = pInfo->curPos++; + STableQueryInfo *item = taosArrayGetP(pa, i); + + char *output = pColInfo->pData + count * rsize; + varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); + + output = varDataVal(output); + STableId* id = TSDB_TABLEID(item->pTable); + + *(int16_t *)output = 0; + output += sizeof(int16_t); + + *(int64_t *)output = id->uid; // memory align problem, todo serialize + output += sizeof(id->uid); + + *(int32_t *)output = id->tid; + output += sizeof(id->tid); + + *(int32_t *)output = pQueryAttr->vgId; + output += sizeof(pQueryAttr->vgId); + + char* data = NULL; + if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { + data = tsdbGetTableName(item->pTable); + } else { + data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes); + } + + doSetTagValueToResultBuf(output, data, type, bytes); + count += 1; + } + + //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count); + } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); + *(int64_t*)pColInfo->pData = pInfo->totalTables; + count = 1; + + pOperator->status = OP_EXEC_DONE; + //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); + } else { // return only the tags|table name etc. + SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo + + count = 0; + while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { + int32_t i = pInfo->curPos++; + + STableQueryInfo* item = taosArrayGetP(pa, i); + + char *data = NULL, *dst = NULL; + int16_t type = 0, bytes = 0; + for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { + // not assign value in case of user defined constant output column + if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) { + continue; + } + + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); + type = pExprInfo[j].base.resSchema.type; + bytes = pExprInfo[j].base.resSchema.bytes; + + if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { + data = tsdbGetTableName(item->pTable); + } else { + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); + } + + dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; + doSetTagValueToResultBuf(dst, data, type, bytes); + } + + count += 1; + } + + if (pInfo->curPos >= pInfo->totalTables) { + pOperator->status = OP_EXEC_DONE; + } + + //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); + } + + if (pOperator->status == OP_EXEC_DONE) { + setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED); + } + + pRes->info.rows = count; + return (pRes->info.rows == 0)? NULL:pInfo->pRes; + +#endif + return TSDB_CODE_SUCCESS; +} + +static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { + STagScanInfo* pInfo = (STagScanInfo*)param; + pInfo->pRes = blockDataDestroy(pInfo->pRes); +} + +SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { + STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->pReader = pReaderHandle; + pInfo->curPos = 0; + pOperator->name = "TagScanOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; + pOperator->blockingOptr = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->getNextFn = doTagScan; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pTaskInfo = pTaskInfo; + pOperator->closeFn = destroyTagScanOperatorInfo; + + return pOperator; + _error: + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; +} diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 40393a4eabb83c37bc555da0eddf35e0d5c2233e..0ad3730b5a44185e1da652c7b31404f911b7e95d 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -24,7 +24,7 @@ extern "C" { #include "functionMgt.h" bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); -int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); +int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); @@ -43,17 +43,17 @@ int32_t maxFunction(SqlFunctionCtx *pCtx); bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool avgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t avgFunction(SqlFunctionCtx* pCtx); -int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); +int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t stddevFunction(SqlFunctionCtx* pCtx); -int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); +int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t percentileFunction(SqlFunctionCtx *pCtx); -int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); +int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo); @@ -65,7 +65,7 @@ int32_t lastFunction(SqlFunctionCtx *pCtx); bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); int32_t topFunction(SqlFunctionCtx *pCtx); -int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); +int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); #ifdef __cplusplus } diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index 5f4f96c4ccc93b84036bb6ae14c1853231b92e10..b72905b87210c21dbef2d8944f283e09f94a2382 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -29,29 +29,20 @@ extern "C" { #define UDF_LISTEN_PIPE_NAME_LEN 32 #define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock." +#define UDF_DNODE_ID_ENV_NAME "DNODE_ID" //====================================================================================== //begin API to taosd and qworker enum { UDFC_CODE_STOPPING = -1, - UDFC_CODE_PIPE_READ_ERR = -3, + UDFC_CODE_PIPE_READ_ERR = -2, + UDFC_CODE_CONNECT_PIPE_ERR = -3, + UDFC_CODE_LOAD_UDF_FAILURE = -4, + UDFC_CODE_INVALID_STATE = -5 }; -typedef void *UdfcHandle; -typedef void *UdfcFuncHandle; -/** - * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf - * @return error code - */ -int32_t udfcOpen(int32_t dnodeId, UdfcHandle* proxyHandle); - -/** - * destroy udfd proxy - * @return error code - */ -int32_t udfcClose(UdfcHandle proxyhandle); /** @@ -60,7 +51,7 @@ int32_t udfcClose(UdfcHandle proxyhandle); * @param handle, out * @return error code */ -int32_t setupUdf(UdfcHandle proxyHandle, char udfName[], SEpSet *epSet, UdfcFuncHandle *handle); +int32_t setupUdf(char udfName[], UdfcFuncHandle *handle); typedef struct SUdfColumnMeta { int16_t type; diff --git a/source/libs/function/inc/tudfInt.h b/source/libs/function/inc/tudfInt.h index 2c16afbd0dd940e59fd9962961240e4cec4bab94..4e7178f7fd0a67e53552ae9d0e73f3357edb8430 100644 --- a/source/libs/function/inc/tudfInt.h +++ b/source/libs/function/inc/tudfInt.h @@ -39,7 +39,6 @@ enum { typedef struct SUdfSetupRequest { char udfName[TSDB_FUNC_NAME_LEN]; - SEpSet epSet; } SUdfSetupRequest; typedef struct SUdfSetupResponse { @@ -112,6 +111,7 @@ void freeUdfDataDataBlock(SUdfDataBlock *block); int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock); int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block); +int32_t getUdfdPipeName(char* pipeName, int32_t size); #ifdef __cplusplus } #endif diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 70087ee46bf3b2f49a8876372c1df75c8cdb582a..a52e39bd4f58343c0166145a9c444e7c366c0101 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -355,7 +355,7 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType) { .bytes = 64, .type = TSDB_DATA_TYPE_BINARY}; + pFunc->node.resType = (SDataType) { .bytes = 24, .type = TSDB_DATA_TYPE_BINARY}; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 63af2fb97225014483a328b9e700eaf58ef20683..0b9765ef15036c7744fe9693249a915812f23541 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -14,7 +14,7 @@ */ #include "builtinsimpl.h" -#include +#include "function.h" #include "querynodes.h" #include "taggfunction.h" #include "tdatablock.h" @@ -44,8 +44,6 @@ typedef struct STopBotResItem { } STopBotResItem; typedef struct STopBotRes { - int32_t pageId; -// int32_t num; STopBotResItem *pItems; } STopBotRes; @@ -92,18 +90,6 @@ typedef struct SDiffInfo { } \ } while (0); -#define DO_UPDATE_SUBSID_RES(ctx, ts) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->subsidiaryRes.numOfCols; ++_i) { \ - SqlFunctionCtx *__ctx = (ctx)->subsidiaryRes.pCtx[_i]; \ - if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ - __ctx->tag.i = (ts); \ - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ - } \ - __ctx->fpSet.process(__ctx); \ - } \ - } while (0) - #define UPDATE_DATA(ctx, left, right, num, sign, _ts) \ do { \ if (((left) < (right)) ^ (sign)) { \ @@ -139,7 +125,8 @@ bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { return true; } -int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { +int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -406,7 +393,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { +int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SInputColumnInfoData* pInput = &pCtx->input; int32_t type = pInput->pData[0]->info.type; SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -416,7 +403,7 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { pAvgRes->result = pAvgRes->sum.dsum / ((double) pAvgRes->count); } - return functionFinalize(pCtx, pBlock, slotId); + return functionFinalize(pCtx, pBlock); } EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow){ @@ -521,6 +508,49 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } +#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) +#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) + +#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ + do { \ + for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ + SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ + __ctx->fpSet.process(__ctx); \ + } \ + } while (0); + +#define DO_UPDATE_SUBSID_RES(ctx, ts) \ + do { \ + for (int32_t _i = 0; _i < (ctx)->subsidiaries.num; ++_i) { \ + SqlFunctionCtx* __ctx = (ctx)->subsidiaries.pCtx[_i]; \ + if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ + __ctx->tag.i = (ts); \ + __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ + } \ + __ctx->fpSet.process(__ctx); \ + } \ + } while (0) + +#define UPDATE_DATA(ctx, left, right, num, sign, _ts) \ + do { \ + if (((left) < (right)) ^ (sign)) { \ + (left) = (right); \ + DO_UPDATE_SUBSID_RES(ctx, _ts); \ + (num) += 1; \ + } \ + } while (0) + +#define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num) \ + do { \ + _t *d = (_t *)((_col)->pData); \ + for (int32_t i = (_start); i < (_nrow) + (_start); ++i) { \ + if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \ + continue; \ + } \ + TSKEY ts = (ctx)->ptsList != NULL ? GET_TS_DATA(ctx, i) : 0; \ + UPDATE_DATA(ctx, val, d[i], num, sign, ts); \ + } \ + } while (0) int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { int32_t numOfElems = 0; @@ -564,8 +594,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { int64_t val = GET_INT64_VAL(tval); if ((prev < val) ^ isMinFunc) { *(int64_t*) buf = val; - for (int32_t i = 0; i < (pCtx)->subsidiaryRes.numOfCols; ++i) { - SqlFunctionCtx* __ctx = pCtx->subsidiaryRes.pCtx[i]; + for (int32_t i = 0; i < (pCtx)->subsidiaries.num; ++i) { + SqlFunctionCtx* __ctx = pCtx->subsidiaries.pCtx[i]; if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor __ctx->tag.i = key; __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; @@ -581,8 +611,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { uint64_t val = GET_UINT64_VAL(tval); if ((prev < val) ^ isMinFunc) { *(uint64_t*) buf = val; - for (int32_t i = 0; i < (pCtx)->subsidiaryRes.numOfCols; ++i) { - SqlFunctionCtx* __ctx = pCtx->subsidiaryRes.pCtx[i]; + for (int32_t i = 0; i < (pCtx)->subsidiaries.num; ++i) { + SqlFunctionCtx* __ctx = pCtx->subsidiaries.pCtx[i]; if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor __ctx->tag.i = key; __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; @@ -797,7 +827,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { +int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SInputColumnInfoData* pInput = &pCtx->input; int32_t type = pInput->pData[0]->info.type; SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -810,7 +840,7 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId pStddevRes->result = sqrt(pStddevRes->quadraticDSum/((double)pStddevRes->count) - avg*avg); } - return functionFinalize(pCtx, pBlock, slotId); + return functionFinalize(pCtx, pBlock); } bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -923,7 +953,7 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } -int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { +int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SVariant* pVal = &pCtx->param[1].param; double v = pVal->nType == TSDB_DATA_TYPE_INT ? pVal->i : pVal->d; @@ -936,7 +966,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t sl } tMemBucketDestroy(pMemBucket); - return functionFinalize(pCtx, pBlock, slotId); + return functionFinalize(pCtx, pBlock); } bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -1353,15 +1383,16 @@ static STopBotRes *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { return pRes; } -static void doAddIntoResult(STopBotRes* pRes, int32_t maxSize, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, +static void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, uint64_t uid, SResultRowEntryInfo* pEntryInfo); +static void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem); +static void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem); + int32_t topFunction(SqlFunctionCtx *pCtx) { int32_t numOfElems = 0; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STopBotRes *pRes = getTopBotOutputInfo(pCtx); - // if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotRes) + POINTER_BYTES * pCtx->param[0].i)) { // buildTopBotStruct(pRes, pCtx); // } @@ -1381,7 +1412,7 @@ int32_t topFunction(SqlFunctionCtx *pCtx) { numOfElems++; char* data = colDataGetData(pCol, i); - doAddIntoResult(pRes, pCtx->param[1].param.i, data, i, NULL, type, pInput->uid, pResInfo); + doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, type, pInput->uid, pResInfo); } return TSDB_CODE_SUCCESS; @@ -1414,9 +1445,11 @@ static int32_t topBotResComparFn(const void *p1, const void *p2, const void *par return (val1->v.d > val2->v.d) ? 1 : -1; } +void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, + uint64_t uid, SResultRowEntryInfo* pEntryInfo) { + STopBotRes *pRes = getTopBotOutputInfo(pCtx); + int32_t maxSize = pCtx->param[1].param.i; -void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, - uint64_t uid, SResultRowEntryInfo* pEntryInfo) { SVariant val = {0}; taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); @@ -1428,22 +1461,9 @@ void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t row STopBotResItem* pItem = &pItems[pEntryInfo->numOfRes]; pItem->v = val; pItem->uid = uid; - pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer - - if (pRes->pageId == -1) { - SFilePage* pPage = getNewBufPage(NULL, 0, &pRes->pageId); - pPage->num = sizeof(SFilePage); - - // keep the current row data - for(int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); - bool isNull = colDataIsNull_s(pCol, rowIndex); - - - colDataGetData(pCol, rowIndex); - } - } + // save the data of this tuple + saveTupleData(pCtx, rowIndex, pSrcBlock, pItem); // allocate the buffer and keep the data of this row into the new allocated buffer pEntryInfo->numOfRes++; @@ -1452,22 +1472,100 @@ void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t row if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pItems[0].v.i) || (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) || (IS_FLOAT_TYPE(type) && val.d > pItems[0].v.d)) { + // replace the old data and the coresponding tuple data STopBotResItem* pItem = &pItems[0]; pItem->v = val; pItem->uid = uid; - pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer + + // save the data of this tuple by over writing the old data + copyTupleData(pCtx, rowIndex, pSrcBlock, pItem); taosheapadjust((void *) pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void *) &type, topBotResComparFn, NULL, false); } } } -int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { +void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem) { + SFilePage* pPage = NULL; + + int32_t completeRowSize = pSrcBlock->info.rowSize + pSrcBlock->info.numOfCols * sizeof(bool); + + if (pCtx->curBufPage == -1) { + pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage); + pPage->num = sizeof(SFilePage); + } else { + pPage = getBufPage(pCtx->pBuf, pCtx->curBufPage); + if (pPage->num + completeRowSize > getBufPageSize(pCtx->pBuf)) { + pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage); + pPage->num = sizeof(SFilePage); + } + } + + pItem->tuplePos.pageId = pCtx->curBufPage; + + // keep the current row data, extract method + int32_t offset = 0; + bool* nullList = (bool*)((char*)pPage + pPage->num); + char* pStart = (char*)(nullList + sizeof(bool) * pSrcBlock->info.numOfCols); + for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); + bool isNull = colDataIsNull_s(pCol, rowIndex); + if (isNull) { + nullList[i] = true; + continue; + } + + char* p = colDataGetData(pCol, rowIndex); + if (IS_VAR_DATA_TYPE(pCol->info.type)) { + memcpy(pStart + offset, p, varDataTLen(p)); + } else { + memcpy(pStart + offset, p, pCol->info.bytes); + } + + offset += pCol->info.bytes; + } + + pItem->tuplePos.offset = pPage->num; + pPage->num += completeRowSize; + + setBufPageDirty(pPage, true); + releaseBufPage(pCtx->pBuf, pPage); +} + +void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem) { + SFilePage* pPage = getBufPage(pCtx->pBuf, pItem->tuplePos.pageId); + + bool* nullList = (bool*)((char*)pPage + pItem->tuplePos.offset); + char* pStart = (char*)(nullList + pSrcBlock->info.numOfCols * sizeof(bool)); + + int32_t offset = 0; + for(int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); + if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) { + continue; + } + + char* p = colDataGetData(pCol, rowIndex); + if (IS_VAR_DATA_TYPE(pCol->info.type)) { + memcpy(pStart + offset, p, varDataTLen(p)); + } else { + memcpy(pStart + offset, p, pCol->info.bytes); + } + + offset += pCol->info.bytes; + } + + setBufPageDirty(pPage, true); + releaseBufPage(pCtx->pBuf, pPage); +} + +int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo *pEntryInfo = GET_RES_INFO(pCtx); STopBotRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); pEntryInfo->complete = true; int32_t type = pCtx->input.pData[0]->info.type; + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); // todo assign the tag value and the corresponding row data @@ -1476,19 +1574,45 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId case TSDB_DATA_TYPE_INT: { for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { STopBotResItem* pItem = &pRes->pItems[i]; - colDataAppendInt32(pCol, currentRow++, (int32_t*)&pItem->v.i); + colDataAppendInt32(pCol, currentRow, (int32_t*)&pItem->v.i); int32_t pageId = pItem->tuplePos.pageId; int32_t offset = pItem->tuplePos.offset; - if (pageId != -1) { - // todo + if (pItem->tuplePos.pageId != -1) { + SFilePage* pPage = getBufPage(pCtx->pBuf, pageId); + + bool* nullList = (bool*)((char*)pPage + offset); + char* pStart = (char*)(nullList + pCtx->pSrcBlock->info.numOfCols * sizeof(bool)); + + // todo set the offset value to optimize the performance. + for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { + SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; + + SFunctParam *pFuncParam = &pc->pExpr->base.pParam[0]; + int32_t srcSlotId = pFuncParam->pCol->slotId; + int32_t dstSlotId = pCtx->pExpr->base.resSchema.slotId; + + int32_t ps = 0; + for(int32_t k = 0; k < srcSlotId; ++k) { + SColumnInfoData* pSrcCol = taosArrayGet(pCtx->pSrcBlock->pDataBlock, k); + ps += pSrcCol->info.bytes; + } + + SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); + if (nullList[srcSlotId]) { + colDataAppendNULL(pDstCol, currentRow); + } else { + colDataAppend(pDstCol, currentRow, (pStart + ps), false); + } + } } + + currentRow += 1; } + break; } } return pEntryInfo->numOfRes; - -// return functionFinalize(pCtx, pBlock, slotId); } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 317339af04e6f93e16d29b43c1d18bfbaf0db1de..f8a7e77814ed11331a8d01f6dca0917517dd59a2 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -124,7 +124,7 @@ enum { int64_t gUdfTaskSeqNum = 0; typedef struct SUdfdProxy { - int32_t dnodeId; + char udfdPipeName[UDF_LISTEN_PIPE_NAME_LEN]; uv_barrier_t gUdfInitBarrier; uv_loop_t gUdfdLoop; @@ -137,11 +137,11 @@ typedef struct SUdfdProxy { int8_t gUdfcState; QUEUE gUdfTaskQueue; QUEUE gUvProcTaskQueue; - // int8_t gUdfcState = UDFC_STATE_INITAL; - // QUEUE gUdfTaskQueue = {0}; - // QUEUE gUvProcTaskQueue = {0}; + + int8_t initialized; } SUdfdProxy; +SUdfdProxy gUdfdProxy = {0}; typedef struct SUdfUvSession { SUdfdProxy *udfc; @@ -209,19 +209,27 @@ enum { UDFC_STATE_STARTNG, // starting after udfcOpen UDFC_STATE_READY, // started and begin to receive quests UDFC_STATE_STOPPING, // stopping after udfcClose - UDFC_STATUS_FINAL, // stopped }; +int32_t getUdfdPipeName(char* pipeName, int32_t size) { + char dnodeId[8] = {0}; + size_t dnodeIdSize; + int32_t err = uv_os_getenv(UDF_DNODE_ID_ENV_NAME, dnodeId, &dnodeIdSize); + if (err != 0) { + dnodeId[0] = '1'; + } + snprintf(pipeName, size, "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId); + return 0; +} + int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup) { int32_t len = 0; len += taosEncodeBinary(buf, setup->udfName, TSDB_FUNC_NAME_LEN); - len += taosEncodeSEpSet(buf, &setup->epSet); return len; } void* decodeUdfSetupRequest(const void* buf, SUdfSetupRequest *request) { buf = taosDecodeBinaryTo(buf, request->udfName, TSDB_FUNC_NAME_LEN); - buf = taosDecodeSEpSet((void*)buf, &request->epSet); return (void*)buf; } @@ -604,7 +612,7 @@ void onUdfcPipeClose(uv_handle_t *handle) { } int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvTask) { - debugPrint("%s", "get uv task result"); + fnDebug("udfc get uv task result. task: %p", task); if (uvTask->type == UV_TASK_REQ_RSP) { if (uvTask->rspBuf.base != NULL) { SUdfResponse rsp; @@ -647,7 +655,6 @@ int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvT } void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { - debugPrint("%s", "client allocate buffer to receive from pipe"); SClientUvConn *conn = handle->data; SClientConnBuf *connBuf = &conn->readBuf; @@ -662,7 +669,7 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf buf->base = connBuf->buf; buf->len = connBuf->cap; } else { - //TODO: log error + fnError("udfc allocate buffer failure. size: %d", msgHeadSize); buf->base = NULL; buf->len = 0; } @@ -674,13 +681,13 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf buf->base = connBuf->buf + connBuf->len; buf->len = connBuf->cap - connBuf->len; } else { - //TODO: log error free connBuf->buf + fnError("udfc re-allocate buffer failure. size: %d", connBuf->cap); buf->base = NULL; buf->len = 0; } } - debugPrint("\tconn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total); + fnTrace("conn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total); } @@ -689,6 +696,7 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { connBuf->total = *(int32_t *) (connBuf->buf); } if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { + fnTrace("udfc complete message is received, now handle it"); return true; } return false; @@ -696,10 +704,10 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { void udfcUvHandleRsp(SClientUvConn *conn) { SClientConnBuf *connBuf = &conn->readBuf; - int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen int32_t then seqnum + int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen then seqnum if (QUEUE_EMPTY(&conn->taskQueue)) { - //LOG error + fnError("udfc no task waiting for response on connection"); return; } bool found = false; @@ -713,7 +721,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) { found = true; taskFound = task; } else { - //LOG error; + fnError("udfc more than one task waiting for the same response"); continue; } } @@ -727,7 +735,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) { uv_sem_post(&taskFound->taskSem); QUEUE_REMOVE(&taskFound->procTaskQueue); } else { - //TODO: LOG error + fnError("no task is waiting for the response."); } connBuf->buf = NULL; connBuf->total = -1; @@ -751,7 +759,7 @@ void udfcUvHandleError(SClientUvConn *conn) { } void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { - debugPrint("%s, nread: %zd", "client read from pipe", nread); + fnTrace("udfc client %p, client read from pipe. nread: %zd", client, nread); if (nread == 0) return; SClientUvConn *conn = client->data; @@ -764,9 +772,9 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } if (nread < 0) { - debugPrint("\tclient read error: %s", uv_strerror(nread)); + fnError("udfc client pipe %p read error: %s", client, uv_strerror(nread)); if (nread == UV_EOF) { - //TODO: + fnError("udfc client pipe %p closed", client); } udfcUvHandleError(conn); } @@ -774,16 +782,15 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } void onUdfClientWrite(uv_write_t *write, int status) { - debugPrint("%s", "after writing to pipe"); SClientUvTaskNode *uvTask = write->data; + uv_pipe_t *pipe = uvTask->pipe; if (status == 0) { - uv_pipe_t *pipe = uvTask->pipe; SClientUvConn *conn = pipe->data; QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); } else { - //TODO Log error; + fnError("udfc client %p write error.", pipe); } - debugPrint("\tlength:%zu", uvTask->reqBuf.len); + fnTrace("udfc client %p write length:%zu", pipe, uvTask->reqBuf.len); taosMemoryFree(write); taosMemoryFree(uvTask->reqBuf.base); } @@ -841,7 +848,7 @@ int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN } int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { - debugPrint("%s, %d", "queue uv task", uvTask->type); + fnTrace("queue uv task to event loop, task: %d, %p", uvTask->type, uvTask); SUdfdProxy *udfc = uvTask->udfc; uv_mutex_lock(&udfc->gUdfTaskQueueMutex); QUEUE_INSERT_TAIL(&udfc->gUdfTaskQueue, &uvTask->recvTaskQueue); @@ -855,7 +862,7 @@ int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { } int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { - debugPrint("%s, type %d", "start uv task ", uvTask->type); + fnTrace("event loop start uv task. task: %d, %p", uvTask->type, uvTask); switch (uvTask->type) { case UV_TASK_CONNECT: { uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); @@ -874,8 +881,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t)); connReq->data = uvTask; - - uv_pipe_connect(connReq, pipe, "udf.sock", onUdfClientConnect); + uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfClientConnect); break; } case UV_TASK_REQ_RSP: { @@ -971,27 +977,37 @@ void constructUdfService(void *argsThread) { uv_loop_close(&udfc->gUdfdLoop); } -int32_t udfcOpen(int32_t dnodeId, UdfcHandle *udfc) { - SUdfdProxy *proxy = taosMemoryCalloc(1, sizeof(SUdfdProxy)); - proxy->dnodeId = dnodeId; +int32_t udfcOpen() { + int8_t old = atomic_val_compare_exchange_8(&gUdfdProxy.initialized, 0, 1); + if (old == 1) { + return 0; + } + SUdfdProxy *proxy = &gUdfdProxy; + getUdfdPipeName(proxy->udfdPipeName, UDF_LISTEN_PIPE_NAME_LEN); proxy->gUdfcState = UDFC_STATE_STARTNG; uv_barrier_init(&proxy->gUdfInitBarrier, 2); uv_thread_create(&proxy->gUdfLoopThread, constructUdfService, proxy); - uv_barrier_wait(&proxy->gUdfInitBarrier); + atomic_store_8(&proxy->gUdfcState, UDFC_STATE_READY); proxy->gUdfcState = UDFC_STATE_READY; - *udfc = proxy; + uv_barrier_wait(&proxy->gUdfInitBarrier); + fnInfo("udfc initialized") return 0; } -int32_t udfcClose(UdfcHandle udfcHandle) { - SUdfdProxy *udfc = udfcHandle; +int32_t udfcClose() { + int8_t old = atomic_val_compare_exchange_8(&gUdfdProxy.initialized, 1, 0); + if (old == 0) { + return 0; + } + + SUdfdProxy *udfc = &gUdfdProxy; udfc->gUdfcState = UDFC_STATE_STOPPING; uv_async_send(&udfc->gUdfLoopStopAsync); uv_thread_join(&udfc->gUdfLoopThread); uv_mutex_destroy(&udfc->gUdfTaskQueueMutex); uv_barrier_destroy(&udfc->gUdfInitBarrier); - udfc->gUdfcState = UDFC_STATUS_FINAL; - taosMemoryFree(udfc); + udfc->gUdfcState = UDFC_STATE_INITAL; + fnInfo("udfc cleaned up"); return 0; } @@ -1009,12 +1025,15 @@ int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) { return task->errCode; } -int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle *funcHandle) { - debugPrint("%s", "client setup udf"); +int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) { + fnInfo("udfc setup udf. udfName: %s", udfName); + if (gUdfdProxy.gUdfcState != UDFC_STATE_READY) { + return UDFC_CODE_INVALID_STATE; + } SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; task->session = taosMemoryMalloc(sizeof(SUdfUvSession)); - task->session->udfc = udfc; + task->session->udfc = &gUdfdProxy; task->type = UDF_TASK_SETUP; SUdfSetupRequest *req = &task->_setup.req; @@ -1022,15 +1041,20 @@ int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle int32_t errCode = udfcRunUvTask(task, UV_TASK_CONNECT); if (errCode != 0) { - //TODO: log error - return -1; + fnError("failed to connect to pipe. udfName: %s, pipe: %s", udfName, (&gUdfdProxy)->udfdPipeName); + return UDFC_CODE_CONNECT_PIPE_ERR; } udfcRunUvTask(task, UV_TASK_REQ_RSP); SUdfSetupResponse *rsp = &task->_setup.rsp; task->session->severHandle = rsp->udfHandle; - *funcHandle = task->session; + if (task->errCode != 0) { + fnError("failed to setup udf. err: %d", task->errCode) + } else { + fnInfo("sucessfully setup udf func handle. handle: %p", task->session); + *funcHandle = task->session; + } int32_t err = task->errCode; taosMemoryFree(task); return err; @@ -1038,7 +1062,7 @@ int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, SSDataBlock* output, SUdfInterBuf *newState) { - debugPrint("%s", "client call udf"); + fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle); SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; @@ -1076,35 +1100,37 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf udfcRunUvTask(task, UV_TASK_REQ_RSP); - SUdfCallResponse *rsp = &task->_call.rsp; - switch (callType) { - case TSDB_UDF_CALL_AGG_INIT: { - *newState = rsp->resultBuf; - break; - } - case TSDB_UDF_CALL_AGG_PROC: { - *newState = rsp->resultBuf; - break; - } - case TSDB_UDF_CALL_AGG_MERGE: { - *newState = rsp->resultBuf; - break; - } - case TSDB_UDF_CALL_AGG_FIN: { - *newState = rsp->resultBuf; - break; - } - case TSDB_UDF_CALL_SCALA_PROC: { - *output = rsp->resultData; - break; + if (task->errCode != 0) { + fnError("call udf failure. err: %d", task->errCode); + } else { + SUdfCallResponse *rsp = &task->_call.rsp; + switch (callType) { + case TSDB_UDF_CALL_AGG_INIT: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_AGG_PROC: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_AGG_MERGE: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_AGG_FIN: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_SCALA_PROC: { + *output = rsp->resultData; + break; + } } } - taosMemoryFree(task); return task->errCode; } -//TODO: translate these calls to callUdf int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) { int8_t callType = TSDB_UDF_CALL_AGG_INIT; @@ -1148,7 +1174,7 @@ int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t nu } int32_t teardownUdf(UdfcFuncHandle handle) { - debugPrint("%s", "client teardown udf"); + fnInfo("tear down udf. udf func handle: %p", handle); SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; @@ -1160,7 +1186,6 @@ int32_t teardownUdf(UdfcFuncHandle handle) { udfcRunUvTask(task, UV_TASK_REQ_RSP); - SUdfTeardownResponse *rsp = &task->_teardown.rsp; int32_t err = task->errCode; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 65408517580181d5f79eb5ba37f671e596414b1e..e4c4cb4893c57255439fe1ba90438708591fa638 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -20,6 +20,7 @@ #include "tudf.h" #include "tudfInt.h" +#include "tdatablock.h" #include "tdataformat.h" #include "tglobal.h" #include "tmsg.h" @@ -31,8 +32,9 @@ typedef struct SUdfdContext { uv_signal_t intrSignal; char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN]; uv_pipe_t listeningPipe; - void *clientRpc; + void *clientRpc; + SCorEpSet mgmtEp; uv_mutex_t udfsMutex; SHashObj *udfsHash; @@ -63,8 +65,13 @@ typedef struct SUdf { uv_mutex_t lock; uv_cond_t condReady; - char name[16]; - int8_t type; + char name[TSDB_FUNC_NAME_LEN]; + int8_t funcType; + int8_t scriptType; + int8_t outputType; + int32_t outputLen; + int32_t bufSize; + char path[PATH_MAX]; uv_lib_t lib; @@ -78,17 +85,17 @@ typedef struct SUdfcFuncHandle { SUdf *udf; } SUdfcFuncHandle; -int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf); +int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf); -int32_t udfdLoadUdf(char *udfName, SEpSet *pEpSet, SUdf *udf) { +int32_t udfdLoadUdf(char *udfName, SUdf *udf) { strcpy(udf->name, udfName); - udfdFillUdfInfoFromMNode(global.clientRpc, pEpSet, udf->name, udf); - + udfdFillUdfInfoFromMNode(global.clientRpc, udf->name, udf); + //strcpy(udf->path, "/home/slzhou/TDengine/debug/build/lib/libudf1.so"); int err = uv_dlopen(udf->path, &udf->lib); if (err != 0) { fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); - // TODO set error + return UDFC_CODE_LOAD_UDF_FAILURE; } // TODO: find all the functions char normalFuncName[TSDB_FUNC_NAME_LEN] = {0}; @@ -115,8 +122,8 @@ void udfdProcessRequest(uv_work_t *req) { SUdf *udf = NULL; uv_mutex_lock(&global.udfsMutex); - SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN); - if (*udfInHash) { + SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName)); + if (udfInHash) { ++(*udfInHash)->refCount; udf = *udfInHash; uv_mutex_unlock(&global.udfsMutex); @@ -128,14 +135,14 @@ void udfdProcessRequest(uv_work_t *req) { uv_mutex_init(&udfNew->lock); uv_cond_init(&udfNew->condReady); udf = udfNew; - taosHashPut(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN, &udfNew, sizeof(&udfNew)); + taosHashPut(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName), &udfNew, sizeof(&udfNew)); uv_mutex_unlock(&global.udfsMutex); } uv_mutex_lock(&udf->lock); if (udf->state == UDF_STATE_INIT) { udf->state = UDF_STATE_LOADING; - udfdLoadUdf(setup->udfName, &setup->epSet, udf); + udfdLoadUdf(setup->udfName, udf); udf->state = UDF_STATE_READY; uv_cond_broadcast(&udf->condReady); uv_mutex_unlock(&udf->lock); @@ -214,7 +221,7 @@ void udfdProcessRequest(uv_work_t *req) { udf->refCount--; if (udf->refCount == 0) { unloadUdf = true; - taosHashRemove(global.udfsHash, udf->name, TSDB_FUNC_NAME_LEN); + taosHashRemove(global.udfsHash, udf->name, strlen(udf->name)); } uv_mutex_unlock(&global.udfsMutex); if (unloadUdf) { @@ -393,7 +400,48 @@ void udfdIntrSignalHandler(uv_signal_t *handle, int signum) { void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; } -int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf) { +int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) { + pEpSet->version = 0; + + // init mnode ip set + SEpSet* mgmtEpSet = &(pEpSet->epSet); + mgmtEpSet->numOfEps = 0; + mgmtEpSet->inUse = 0; + + if (firstEp && firstEp[0] != 0) { + if (strlen(firstEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); + if (code != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return terrno; + } + + mgmtEpSet->numOfEps++; + } + + if (secondEp && secondEp[0] != 0) { + if (strlen(secondEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); + mgmtEpSet->numOfEps++; + } + + if (mgmtEpSet->numOfEps == 0) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + return 0; +} + +int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) { SRetrieveFuncReq retrieveReq = {0}; retrieveReq.numOfFuncs = 1; retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); @@ -410,15 +458,21 @@ int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC; SRpcMsg rpcRsp = {0}; - rpcSendRecv(clientRpc, pEpSet, &rpcMsg, &rpcRsp); + rpcSendRecv(clientRpc, &global.mgmtEp.epSet, &rpcMsg, &rpcRsp); SRetrieveFuncRsp retrieveRsp = {0}; tDeserializeSRetrieveFuncRsp(rpcRsp.pCont, rpcRsp.contLen, &retrieveRsp); SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0); + udf->funcType = pFuncInfo->funcType; + udf->scriptType = pFuncInfo->scriptType; + udf->outputType = pFuncInfo->funcType; + udf->outputLen = pFuncInfo->outputLen; + udf->bufSize = pFuncInfo->bufSize; + char path[PATH_MAX] = {0}; - taosGetTmpfilePath("/tmp", "libudf", path); - TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", udfName); + TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); // TODO check for failure of flush to disk taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); taosCloseFile(&file); @@ -531,15 +585,7 @@ static int32_t udfdUvInit() { uv_pipe_open(&global.ctrlPipe, 0); uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb); - char dnodeId[8] = {0}; - size_t dnodeIdSize; - int32_t err = uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize); - if (err != 0) { - dnodeId[0] = '1'; - } - char listenPipeName[32] = {0}; - snprintf(listenPipeName, sizeof(listenPipeName), "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId); - strcpy(global.listenPipeName, listenPipeName); + getUdfdPipeName(global.listenPipeName, UDF_LISTEN_PIPE_NAME_LEN); removeListeningPipe(); @@ -550,7 +596,7 @@ static int32_t udfdUvInit() { int r; fnInfo("bind to pipe %s", global.listenPipeName); - if ((r = uv_pipe_bind(&global.listeningPipe, listenPipeName))) { + if ((r = uv_pipe_bind(&global.listeningPipe, global.listenPipeName))) { fnError("Bind error %s", uv_err_name(r)); removeListeningPipe(); return -1; @@ -580,7 +626,7 @@ static int32_t udfdRun() { fnInfo("start the udfd"); int code = uv_run(global.loop, UV_RUN_DEFAULT); - fnInfo("udfd stopped. result: %s", uv_err_name(code)); + fnInfo("udfd stopped. result: %s, code: %d", uv_err_name(code), code); int codeClose = uv_loop_close(global.loop); fnDebug("uv loop close. result: %s", uv_err_name(codeClose)); udfdCloseClientRpc(); @@ -615,5 +661,6 @@ int main(int argc, char *argv[]) { return -1; } + initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp); return udfdRun(); } diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index 0727a4a1d26db2678c1bb63a7c193e0a91fd2321..8e5eac538e9bac83534e3ad5143482e008f51d91 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -1,61 +1,84 @@ #include #include #include - #include "uv.h" + +#include "fnLog.h" #include "os.h" -#include "tudf.h" #include "tdatablock.h" +#include "tglobal.h" +#include "tudf.h" -int main(int argc, char *argv[]) { - UdfcHandle udfc; - udfcOpen(1, &udfc); - uv_sleep(1000); - char path[256] = {0}; - size_t cwdSize = 256; - int err = uv_cwd(path, &cwdSize); - if (err != 0) { - fprintf(stderr, "err cwd: %s\n", uv_strerror(err)); - return err; - } - fprintf(stdout, "current working directory:%s\n", path); - strcat(path, "/libudf1.so"); - - UdfcFuncHandle handle; - SEpSet epSet; - setupUdf(udfc, "udf1", &epSet, &handle); - - SSDataBlock block = {0}; - SSDataBlock* pBlock = █ - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; - pBlock->info.rows = 4; - char data[16] = {0}; - char bitmap[4] = {0}; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfo.pData = data; - colInfo.nullbitmap = bitmap; - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - colDataAppendInt32(&colInfo, j, &j); +static int32_t parseArgs(int32_t argc, char *argv[]) { + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default is %s\n", configDir); + return -1; } - taosArrayPush(pBlock->pDataBlock, &colInfo); } + } - SScalarParam input = {0}; - input.numOfRows = pBlock->info.rows; - input.columnData = taosArrayGet(pBlock->pDataBlock, 0); - SScalarParam output = {0}; - callUdfScalarFunc(handle, &input, 1 , &output); + return 0; +} - SColumnInfoData *col = output.columnData; - for (int32_t i = 0; i < output.numOfRows; ++i) { - fprintf(stderr, "%d\t%d\n" , i, *(int32_t*)(col->pData + i *sizeof(int32_t))); +static int32_t initLog() { + char logName[12] = {0}; + snprintf(logName, sizeof(logName), "%slog", "udfc"); + return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0); +} + +int main(int argc, char *argv[]) { + parseArgs(argc, argv); + initLog(); + if (taosInitCfg(configDir, NULL, NULL, NULL, 0) != 0) { + fnError("failed to start since read config error"); + return -1; + } + + udfcOpen(); + uv_sleep(1000); + + UdfcFuncHandle handle; + + setupUdf("udf1", &handle); + + SSDataBlock block = {0}; + SSDataBlock *pBlock = █ + pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + pBlock->info.numOfCols = 1; + pBlock->info.rows = 4; + char data[16] = {0}; + char bitmap[4] = {0}; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData colInfo = {0}; + colInfo.info.type = TSDB_DATA_TYPE_INT; + colInfo.info.bytes = sizeof(int32_t); + colInfo.info.colId = 1; + colInfo.pData = data; + colInfo.nullbitmap = bitmap; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + colDataAppendInt32(&colInfo, j, &j); } - teardownUdf(handle); + taosArrayPush(pBlock->pDataBlock, &colInfo); + } + + SScalarParam input = {0}; + input.numOfRows = pBlock->info.rows; + input.columnData = taosArrayGet(pBlock->pDataBlock, 0); + SScalarParam output = {0}; + callUdfScalarFunc(handle, &input, 1, &output); - udfcClose(udfc); + SColumnInfoData *col = output.columnData; + for (int32_t i = 0; i < output.numOfRows; ++i) { + fprintf(stderr, "%d\t%d\n", i, *(int32_t *)(col->pData + i * sizeof(int32_t))); + } + teardownUdf(handle); + udfcClose(); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 31cc629e0a31d194d042e9eb1fde01a8e14585ad..e74a5ad6a9d111fcae0a84dee88455f57ff6ac1c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3673,7 +3673,9 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } - pQuery->precision = extractResultTsPrecision((SSelectStmt*)pQuery->pRoot); + if (nodeType(pQuery->pRoot) == QUERY_NODE_SELECT_STMT) { + pQuery->precision = extractResultTsPrecision((SSelectStmt*)pQuery->pRoot); + } } if (NULL != pCxt->pDbs) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index de805d2c772f43be55e328684d58e81086100b83..fd7c831399901a5c8d9922e011213e4267d748cd 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -342,28 +342,20 @@ PROCESS_META_OVER: int32_t queryProcessQnodeListRsp(void *output, char *msg, int32_t msgSize) { SQnodeListRsp out = {0}; - int32_t code = -1; + int32_t code = 0; if (NULL == output || NULL == msg || msgSize <= 0) { code = TSDB_CODE_TSC_INVALID_INPUT; - goto PROCESS_QLIST_OVER; + return code; } + out.addrsList = (SArray *)output; if (tDeserializeSQnodeListRsp(msg, msgSize, &out) != 0) { qError("invalid qnode list rsp msg, msgSize:%d", msgSize); code = TSDB_CODE_INVALID_MSG; - goto PROCESS_QLIST_OVER; + return code; } -PROCESS_QLIST_OVER: - - if (code != 0) { - tFreeSQnodeListRsp(&out); - out.addrsList = NULL; - } - - *(SArray **)output = out.addrsList; - return code; } diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 3d59ffd93d7ac7e59ac1aa9e81023f3d2cb30636..9142cc41c41f8e8e491549fbbf5c990c07264fd4 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -32,6 +32,9 @@ typedef struct SScalarCtx { #define SCL_DATA_TYPE_DUMMY_HASH 9000 #define SCL_DEFAULT_OP_NUM 10 +#define SCL_IS_CONST_NODE(_node) ((NULL == (_node)) || (QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type)) +#define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList) + #define sclFatal(...) qFatal(__VA_ARGS__) #define sclError(...) qError(__VA_ARGS__) #define sclWarn(...) qWarn(__VA_ARGS__) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 145afbe984a8f19e30871cd91bd62f9c5520cf32..820a4894b5ed756bcacfe030211b24b35e86b11e 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -244,23 +244,53 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t return TSDB_CODE_SUCCESS; } -int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *rowNum) { +int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *paramNum, int32_t *rowNum) { int32_t code = 0; - SScalarParam *paramList = taosMemoryCalloc(pParamList->length, sizeof(SScalarParam)); + if (NULL == pParamList) { + if (ctx->pBlockList) { + SSDataBlock *pBlock = taosArrayGet(ctx->pBlockList, 0); + *rowNum = pBlock->info.rows; + } else { + *rowNum = 1; + } + + *paramNum = 1; + } else { + *paramNum = pParamList->length; + } + + SScalarParam *paramList = taosMemoryCalloc(*paramNum, sizeof(SScalarParam)); if (NULL == paramList) { - sclError("calloc %d failed", (int32_t)(pParamList->length * sizeof(SScalarParam))); + sclError("calloc %d failed", (int32_t)((*paramNum) * sizeof(SScalarParam))); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SListCell *cell = pParamList->pHead; - for (int32_t i = 0; i < pParamList->length; ++i) { - if (NULL == cell || NULL == cell->pNode) { - sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); - SCL_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + if (pParamList) { + SNode *tnode = NULL; + int32_t i = 0; + if (SCL_IS_CONST_CALC(ctx)) { + WHERE_EACH (tnode, pParamList) { + if (!SCL_IS_CONST_NODE(tnode)) { + WHERE_NEXT; + } else { + SCL_ERR_JRET(sclInitParam(tnode, ¶mList[i], ctx, rowNum)); + ERASE_NODE(pParamList); + } + + ++i; + } + } else { + FOREACH(tnode, pParamList) { + SCL_ERR_JRET(sclInitParam(tnode, ¶mList[i], ctx, rowNum)); + ++i; + } } + } else { + paramList[0].numOfRows = *rowNum; + } - SCL_ERR_JRET(sclInitParam(cell->pNode, ¶mList[i], ctx, rowNum)); - cell = cell->pNext; + if (0 == *rowNum) { + taosMemoryFreeClear(paramList); } *pParams = paramList; @@ -299,37 +329,45 @@ _return: } int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *output) { - if (NULL == node->pParameterList || node->pParameterList->length <= 0) { - sclError("invalid function parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0); - SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } - - SScalarFuncExecFuncs ffpSet = {0}; - int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); - if (code) { - sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); - SCL_ERR_RET(code); - } - SScalarParam *params = NULL; int32_t rowNum = 0; - SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); - - output->columnData = createColumnInfoData(&node->node.resType, rowNum); - if (output->columnData == NULL) { - sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes)); - SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } + int32_t paramNum = 0; + int32_t code = 0; + SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, ¶mNum, &rowNum)); - code = (*ffpSet.process)(params, node->pParameterList->length, output); - if (code) { - sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + if (fmIsUserDefinedFunc(node->funcId)) { +#if 0 + UdfcFuncHandle udfHandle = NULL; + + SCL_ERR_JRET(setupUdf(node->functionName, &udfHandle)); + code = callUdfScalarFunc(udfHandle, params, paramNum, output); + teardownUdf(udfHandle); SCL_ERR_JRET(code); +#endif + } else { + SScalarFuncExecFuncs ffpSet = {0}; + code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); + if (code) { + sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + SCL_ERR_JRET(code); + } + + output->columnData = createColumnInfoData(&node->node.resType, rowNum); + if (output->columnData == NULL) { + sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes)); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + code = (*ffpSet.process)(params, paramNum, output); + if (code) { + sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + SCL_ERR_JRET(code); + } } _return: - for (int32_t i = 0; i < node->pParameterList->length; ++i) { + for (int32_t i = 0; i < paramNum; ++i) { // sclFreeParamNoData(params + i); } @@ -355,8 +393,13 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o SScalarParam *params = NULL; int32_t rowNum = 0; + int32_t paramNum = 0; int32_t code = 0; - SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); + SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, ¶mNum, &rowNum)); + if (NULL == params) { + output->numOfRows = 0; + return TSDB_CODE_SUCCESS; + } int32_t type = node->node.resType.type; output->numOfRows = rowNum; @@ -369,25 +412,41 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o } bool value = false; + bool complete = true; for (int32_t i = 0; i < rowNum; ++i) { - for (int32_t m = 0; m < node->pParameterList->length; ++m) { + complete = true; + for (int32_t m = 0; m < paramNum; ++m) { + if (NULL == params[m].columnData) { + complete = false; + continue; + } char* p = colDataGetData(params[m].columnData, i); GET_TYPED_DATA(value, bool, params[m].columnData->info.type, p); if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) { + complete = true; break; } else if (LOGIC_COND_TYPE_OR == node->condType && value) { + complete = true; break; } else if (LOGIC_COND_TYPE_NOT == node->condType) { value = !value; } } - colDataAppend(output->columnData, i, (char*) &value, false); + if (complete) { + colDataAppend(output->columnData, i, (char*) &value, false); + } + } + + if (SCL_IS_CONST_CALC(ctx) && (false == complete)) { + sclFreeParam(output); + output->numOfRows = 0; } _return: - for (int32_t i = 0; i < node->pParameterList->length; ++i) { + + for (int32_t i = 0; i < paramNum; ++i) { // sclFreeParamNoData(params + i); } @@ -426,6 +485,17 @@ _return: EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)*pNode; + SNode* tnode = NULL; + if (fmIsUserDefinedFunc(node->funcId)) { + return DEAL_RES_CONTINUE; + } + + FOREACH(tnode, node->pParameterList) { + if (!SCL_IS_CONST_NODE(tnode)) { + return DEAL_RES_CONTINUE; + } + } + SScalarParam output = {0}; ctx->code = sclExecFunction(node, ctx, &output); @@ -470,6 +540,10 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } + if (0 == output.numOfRows) { + return DEAL_RES_CONTINUE; + } + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); @@ -498,6 +572,14 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)*pNode; + if (!SCL_IS_CONST_NODE(node->pLeft)) { + return DEAL_RES_CONTINUE; + } + + if (!SCL_IS_CONST_NODE(node->pRight)) { + return DEAL_RES_CONTINUE; + } + SScalarParam output = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; ctx->code = sclExecOperator(node, ctx, &output); if (ctx->code) { @@ -530,7 +612,7 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { } EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) { + if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) { return DEAL_RES_CONTINUE; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 156e71a845a3a84679dea73607fc9d97f140148b..08b58cf8100a5561917d965eac10d15156f2e6bc 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -32,8 +32,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu float *in = (float *)pInputData->pData; float *out = (float *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0)? in[i] : -in[i]; @@ -45,8 +45,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu double *in = (double *)pInputData->pData; double *out = (double *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0)? in[i] : -in[i]; @@ -58,8 +58,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int8_t *in = (int8_t *)pInputData->pData; int8_t *out = (int8_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0)? in[i] : -in[i]; @@ -71,8 +71,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int16_t *in = (int16_t *)pInputData->pData; int16_t *out = (int16_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0)? in[i] : -in[i]; @@ -84,8 +84,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int32_t *in = (int32_t *)pInputData->pData; int32_t *out = (int32_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0)? in[i] : -in[i]; @@ -97,8 +97,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int64_t *in = (int64_t *)pInputData->pData; int64_t *out = (int64_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0)? in[i] : -in[i]; @@ -129,8 +129,8 @@ static int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SS double *out = (double *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = valFn(getValueFn(pInputData->pData, i)); @@ -157,9 +157,9 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S double *out = (double *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData[0]->nullbitmap, i) || - colDataIsNull_f(pInputData[1]->nullbitmap, 0)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData[0], i) || + colDataIsNull_s(pInputData[1], 0)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, 0)); @@ -184,8 +184,8 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP float *out = (float *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = f1(in[i]); @@ -198,8 +198,8 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP double *out = (double *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = d1(in[i]); @@ -301,7 +301,7 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + colDataAppendNULL(pOutputData, i); continue; } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 04ab26d8a499dee44f8e7537ddca2279753725a9..ed52e48f6d8186666163d7ff2ea714640dc14143 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -137,6 +137,11 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in rnode->node.resType.bytes = dataBytes; rnode->dataBlockId = 0; + if (NULL == block) { + *pNode = (SNode *)rnode; + return; + } + if (NULL == *block) { SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = 3; @@ -889,6 +894,8 @@ TEST(constantTest, int_greater_int_is_true2) { } TEST(constantTest, greater_and_lower) { + scltInitLogFile(); + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; bool eRes[5] = {false, false, true, true, true}; int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; @@ -913,6 +920,115 @@ TEST(constantTest, greater_and_lower) { nodesDestroyNode(res); } +TEST(constantTest, column_and_value1) { + scltInitLogFile(); + + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *v = (SLogicConditionNode *)res; + ASSERT_EQ(v->condType, LOGIC_COND_TYPE_AND); + ASSERT_EQ(v->pParameterList->length, 1); + nodesDestroyNode(res); +} + +TEST(constantTest, column_and_value2) { + scltInitLogFile(); + + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); +} + +TEST(constantTest, column_and_value3) { + scltInitLogFile(); + + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_OR, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); +} + +TEST(constantTest, column_and_value4) { + scltInitLogFile(); + + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_OR, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *v = (SLogicConditionNode *)res; + ASSERT_EQ(v->condType, LOGIC_COND_TYPE_OR); + ASSERT_EQ(v->pParameterList->length, 1); + nodesDestroyNode(res); +} + + void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){ char keyVar[32] = {0}; memcpy(varDataVal(keyVar), key, strlen(key)); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index a2352a1831218d9d2648268d22226955911233e2..deb158cbaef59a70364bf6d4c87c8c0acb978b09 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -170,7 +170,6 @@ static int32_t syncIOStartInternal(SSyncIO *io) { taosBlockSIGPIPE(); rpcInit(); - tsRpcForceTcp = 1; // cient rpc init { diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index cf3c95e6589c81268e7363581e5aa757ba969cc1..148529170c7d50a633993efb4bf4a1496bc29683 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -119,7 +119,7 @@ int taosSetConsoleEcho(bool on) { #endif } -void setTerminalMode() { +void taosSetTerminalMode() { #if defined(WINDOWS) #else @@ -152,7 +152,7 @@ void setTerminalMode() { #endif } -int32_t getOldTerminalMode() { +int32_t taosGetOldTerminalMode() { #if defined(WINDOWS) #else @@ -170,7 +170,7 @@ int32_t getOldTerminalMode() { #endif } -void resetTerminalMode() { +void taosResetTerminalMode() { #if defined(WINDOWS) #else diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 1c655fc2bf765500f4e4ef5e413590c58511d615..2d741b18f6ce71cb98aaa47e19f1c262fb1a35e0 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -476,6 +476,7 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t return (void*)buf; } +// todo remove it // order array void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) { taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 6f6ae5d699bd7af6b3e71da06ea2c29584d2151c..699a7aa080fdc5e581555b7193fe257473330995 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -513,6 +513,8 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { for (int32_t i = 0; i < size; ++i) { SConfigItem *pItem = taosArrayGet(pCfg->array, i); if (tsc && !pItem->tsc) continue; + if (dump && strcmp(pItem->name, "scriptDir") == 0) continue; + if (dump && strcmp(pItem->name, "simDebugFlag") == 0) continue; tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); for (int32_t i = 0; i < CFG_SRC_PRINT_LEN; ++i) { if (src[i] == 0) src[i] = ' '; @@ -551,10 +553,10 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { break; case CFG_DTYPE_FLOAT: if (dump) { - printf("%s %s %f", src, name, pItem->fval); + printf("%s %s %.2f", src, name, pItem->fval); printf("\n"); } else { - uInfo("%s %s %f", src, name, pItem->fval); + uInfo("%s %s %.2f", src, name, pItem->fval); } break; case CFG_DTYPE_STRING: diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 34a59a7d4847cd96bae7368bd66324c865722d13..e06d7d8c898f7fdbcc2464d19f8330e2cffff273 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -110,7 +110,7 @@ typedef struct time_wheel_t { tmr_obj_t** slots; } time_wheel_t; -int32_t tsMaxTmrCtrl = 512; +static int32_t tsMaxTmrCtrl = 512; static TdThreadOnce tmrModuleInit = PTHREAD_ONCE_INIT; static TdThreadMutex tmrCtrlMutex; diff --git a/tests/script/tsim/parser/groupby-basic.sim b/tests/script/tsim/parser/groupby-basic.sim index f073200a05e31148d6ba6e9d89c8902297270190..4d6b33612f54e55061ceee71e3c621a5012acd27 100644 --- a/tests/script/tsim/parser/groupby-basic.sim +++ b/tests/script/tsim/parser/groupby-basic.sim @@ -115,19 +115,19 @@ endi if $data00 != 10 then return -1 endi -if $data01 != 0 then +if $data10 != 10 then return -1 endi -if $data10 != 10 then +if $data90 != 10 then return -1 endi -if $data11 != 1 then +if $data01 != 7 then return -1 endi -if $data90 != 10 then +if $data11 != 6 then return -1 endi -if $data91 != 9 then +if $data91 != 3 then return -1 endi @@ -143,16 +143,16 @@ if $row != 10 then return -1 endi -if $data00 != @22-01-01 00:00:00.000@ then +if $data00 != @22-01-01 00:00:00.007@ then return -1 endi -if $data01 != 0 then +if $data01 != 7 then return -1 endi -if $data90 != @22-01-01 00:00:00.009@ then +if $data90 != @22-01-01 00:00:00.003@ then return -1 endi -if $data91 != 9 then +if $data91 != 3 then return -1 endi diff --git a/tests/script/tsim/query/crash_sql.sim b/tests/script/tsim/query/crash_sql.sim new file mode 100644 index 0000000000000000000000000000000000000000..44671fbb0db9a4c583e874fb0158783cfbd3343e --- /dev/null +++ b/tests/script/tsim/query/crash_sql.sim @@ -0,0 +1,103 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi + +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +print =============== create database +sql create database db +sql show databases +if $rows != 2 then + return -1 +endi + +sql use db + +print =============== create super table and child table +sql create table stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int) +sql show stables +print $rows $data00 $data01 $data02 +if $rows != 1 then + return -1 +endi + +sql create table ct1 using stb1 tags ( 1 ) +sql create table ct2 using stb1 tags ( 2 ) +sql create table ct3 using stb1 tags ( 3 ) +sql create table ct4 using stb1 tags ( 4 ) +sql show tables +print $rows $data00 $data10 $data20 +if $rows != 4 then + return -1 +endi + +print =============== insert data into child table ct1 (s) +sql insert into ct1 values ( '2022-01-01 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a ) +sql insert into ct1 values ( '2022-01-01 01:01:06.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a ) +sql insert into ct1 values ( '2022-01-01 01:01:10.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a ) +sql insert into ct1 values ( '2022-01-01 01:01:16.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a ) +sql insert into ct1 values ( '2022-01-01 01:01:20.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a ) +sql insert into ct1 values ( '2022-01-01 01:01:26.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a ) +sql insert into ct1 values ( '2022-01-01 01:01:30.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", now+7a ) +sql insert into ct1 values ( '2022-01-01 01:01:36.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", now+8a ) + +print =============== insert data into child table ct2 (d) +sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a ) +sql insert into ct2 values ( '2022-01-01 10:00:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a ) +sql insert into ct2 values ( '2022-01-01 20:00:01.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a ) +sql insert into ct2 values ( '2022-01-02 10:00:01.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a ) +sql insert into ct2 values ( '2022-01-02 20:00:01.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a ) +sql insert into ct2 values ( '2022-01-03 10:00:01.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", now+6a ) +sql insert into ct2 values ( '2022-01-03 20:00:01.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", now+7a ) + +print =============== insert data into child table ct3 (n) +sql insert into ct3 values ( '2021-12-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) +sql insert into ct3 values ( '2021-12-31 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a ) +sql insert into ct3 values ( '2022-01-01 01:01:06.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a ) +sql insert into ct3 values ( '2022-01-07 01:01:10.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a ) +sql insert into ct3 values ( '2022-01-31 01:01:16.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a ) +sql insert into ct3 values ( '2022-02-01 01:01:20.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a ) +sql insert into ct3 values ( '2022-02-28 01:01:26.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a ) +sql insert into ct3 values ( '2022-03-01 01:01:30.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) +sql insert into ct3 values ( '2022-03-08 01:01:36.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + +print =============== insert data into child table ct4 (y) +sql insert into ct4 values ( '2019-01-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) +sql insert into ct4 values ( '2019-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a ) +sql insert into ct4 values ( '2019-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a ) +sql insert into ct4 values ( '2020-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a ) +sql insert into ct4 values ( '2020-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a ) +sql insert into ct4 values ( '2020-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a ) +sql insert into ct4 values ( '2020-12-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) +sql insert into ct4 values ( '2021-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a ) +sql insert into ct4 values ( '2021-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) +sql insert into ct4 values ( '2021-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) +sql insert into ct4 values ( '2022-02-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) +sql insert into ct4 values ( '2022-05-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + + +print ================ start query ====================== +print ================ SQL used to cause taosd or taos shell crash +sql select sum(c1) ,count(c1) from ct4 group by c1 having sum(c10) between 0 and 1 ; + + + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/stable/disk.sim b/tests/script/tsim/stable/disk.sim index 97ef779ff218812ca4befa96bae4a0bd939cea8a..c1ced6ae1076b3b1cc5e8a79f31188c076a93f59 100644 --- a/tests/script/tsim/stable/disk.sim +++ b/tests/script/tsim/stable/disk.sim @@ -53,6 +53,7 @@ endi sql select count(tbcol) from $mt print select count(tbcol) from $mt ===> $data00 if $data00 != $totalNum then + print expect $totalNum , actual: $data00 return -1 endi diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py index f09e7d1f630c1f1427960b07160bcf07df947425..fb0f22eaf8ba20b5229ec6f885c7af1202a85b0b 100644 --- a/tests/system-test/2-query/cast.py +++ b/tests/system-test/2-query/cast.py @@ -6,6 +6,7 @@ import inspect from util.log import * from util.sql import * from util.cases import * +from util.dnodes import * @@ -15,56 +16,50 @@ class TDTestCase: tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring - tdSql.prepare() + def __cast_to_bigint(self, col_name, tbname): + __sql = f"select cast({col_name} as bigint), {col_name} from {tbname}" + tdSql.query(sql=__sql) + data_tb_col = [result[1] for result in tdSql.queryResult] + for i in range(len(tdSql.queryRows)): + tdSql.checkData( i, 0, None ) if data_tb_col[i] is None else tdSql.checkData( i, 0, int(data_tb_col[i]) ) + + def __range_to_bigint(self,cols,tables): + for col in cols: + for table in tables: + self.__cast_to_bigint(col_name=col, tbname=table) + + def __cast_to_timestamp(self, col_name, tbname): + __sql = f"select cast({col_name} as timestamp), {col_name} from {tbname}" + tdSql.query(sql=__sql) + data_tb_col = [result[1] for result in tdSql.queryResult] + for i in range(len(tdSql.queryRows)): + if data_tb_col[i] is None: + tdSql.checkData( i, 0 , None ) + if (col_name == "c2" or col_name == "double" ) and tbname == "t1" and i == 10: + continue + else: + utc_zone = datetime.timezone.utc + utc_8 = datetime.timezone(datetime.timedelta(hours=8)) + date_init_stamp = datetime.datetime.utcfromtimestamp(data_tb_col[i]/1000) + date_data = date_init_stamp.replace(tzinfo=utc_zone).astimezone(utc_8).strftime("%Y-%m-%d %H:%M:%S.%f") + tdSql.checkData( i, 0, date_data) - tdLog.printNoPrefix("==========step1:create table") - tdSql.execute( - '''create table stb1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - tags (t1 int) - ''' - ) - tdSql.execute( - ''' - create table t1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - ''' - ) - for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + def __range_to_timestamp(self, cols, tables): + for col in cols: + for table in tables: + self.__cast_to_timestamp(col_name=col, tbname=table) - tdLog.printNoPrefix("==========step2:insert data") - for i in range(9): - tdSql.execute( - f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" - ) - tdSql.execute( - f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" - ) - tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") - tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + def __test_bigint(self): + __table_list = ["ct1", "ct4", "t1"] + __col_list = ["c1","c2","c3","c4","c5","c6","c7","c10","c1+c2"] + self.__range_to_bigint(cols=__col_list, tables=__table_list) - tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + def __test_timestamp(self): + __table_list = ["ct1", "ct4", "t1"] + __col_list = ["c1","c2","c3","c4","c5","c6","c7","c1+c2"] + self.__range_to_timestamp(cols=__col_list, tables=__table_list) - tdSql.execute( - f'''insert into t1 values - ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) - ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) - ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) - ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) - ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) - ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) - ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) - ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) - ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) - ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ''' - ) + def all_test(self): tdSql.query("select c1 from ct4") data_ct4_c1 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] @@ -82,9 +77,9 @@ class TDTestCase: tdLog.printNoPrefix("==========step5: cast int to binary, expect changes to str(int) ") - tdSql.query("select cast(c1 as binary(32)) as b from ct4") - for i in range(len(data_ct4_c1)): - tdSql.checkData( i, 0, str(data_ct4_c1[i]) ) + #tdSql.query("select cast(c1 as binary(32)) as b from ct4") + #for i in range(len(data_ct4_c1)): + # tdSql.checkData( i, 0, str(data_ct4_c1[i]) ) tdSql.query("select cast(c1 as binary(32)) as b from t1") for i in range(len(data_t1_c1)): tdSql.checkData( i, 0, str(data_t1_c1[i]) ) @@ -240,7 +235,7 @@ class TDTestCase: tdSql.checkData( i, 0, date_data) - tdLog.printNoPrefix("==========step16: cast smallint to bigint, expect no changes") + tdLog.printNoPrefix("==========step16: cast tinyint to bigint, expect no changes") tdSql.query("select c4 from ct4") data_ct4_c4 = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] tdSql.query("select c4 from t1") @@ -254,7 +249,7 @@ class TDTestCase: tdSql.checkData( i, 0, data_t1_c4[i]) - tdLog.printNoPrefix("==========step17: cast smallint to binary, expect changes to str(int) ") + tdLog.printNoPrefix("==========step17: cast tinyint to binary, expect changes to str(int) ") tdSql.query("select cast(c4 as binary(32)) as b from ct4") for i in range(len(data_ct4_c4)): @@ -263,7 +258,7 @@ class TDTestCase: for i in range(len(data_t1_c4)): tdSql.checkData( i, 0, str(data_t1_c4[i]) ) - tdLog.printNoPrefix("==========step18: cast smallint to nchar, expect changes to str(int) ") + tdLog.printNoPrefix("==========step18: cast tinyint to nchar, expect changes to str(int) ") tdSql.query("select cast(c4 as nchar(32)) as b from ct4") for i in range(len(data_ct4_c4)): @@ -272,7 +267,7 @@ class TDTestCase: for i in range(len(data_t1_c4)): tdSql.checkData( i, 0, str(data_t1_c4[i]) ) - tdLog.printNoPrefix("==========step19: cast smallint to timestamp, expect changes to timestamp ") + tdLog.printNoPrefix("==========step19: cast tinyint to timestamp, expect changes to timestamp ") tdSql.query("select cast(c4 as timestamp) as b from ct4") for i in range(len(data_ct4_c4)): @@ -624,7 +619,67 @@ class TDTestCase: tdSql.error("select cast(c8 as timestamp ) as b from ct4") tdSql.error("select cast(c9 as timestamp ) as b from ct4") tdSql.error("select cast(c9 as binary(64) ) as b from ct4") + pass + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + tdLog.printNoPrefix("==========step2:insert data") + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + self.all_test() + + tdDnodes.stop(1) + tdDnodes.start(1) + + tdSql.execute("use db") + self.all_test() def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/sum.py b/tests/system-test/2-query/sum.py new file mode 100644 index 0000000000000000000000000000000000000000..757f5c3c90a072bbc5589a20768b7836c549bf80 --- /dev/null +++ b/tests/system-test/2-query/sum.py @@ -0,0 +1,232 @@ +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + + +INT_COL = "c1" +BINT_COL = "c2" +SINT_COL = "c3" +TINT_COL = "c4" +FLOAT_COL = "c5" +DOUBLE_COL = "c6" +BOOL_COL = "c7" + +BINARY_COL = "c8" +NCHAR_COL = "c9" +TS_COL = "c10" + +NUM_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] +UN_NUM_COL = [BOOL_COL, BINARY_COL, NCHAR_COL, ] +TS_TYPE_COL = [TS_COL] + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def __sum_condition(self): + sum_condition = [] + for num_col in NUM_COL: + sum_condition.extend( + ( + num_col, + f"ceil( {num_col} )", + ) + ) + sum_condition.extend( f"{num_col} + {num_col_2}" for num_col_2 in NUM_COL ) + sum_condition.extend( f"{num_col} + {un_num_col} " for un_num_col in UN_NUM_COL ) + + sum_condition.append(1) + + return sum_condition + + def __where_condition(self, col): + return f" where abs( {col} ) < 1000000 " + + def __group_condition(self, col, having = ""): + return f" group by {col} having {having}" if having else f" group by {col} " + + def __sum_current_check(self, tbname): + sum_condition = self.__sum_condition() + for condition in sum_condition: + where_condition = self.__where_condition(condition) + group_condition = self.__group_condition(condition, having=f"{condition} is not null " ) + + tdSql.query(f"select {condition} from {tbname} {where_condition} ") + datas = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] + sum_data = sum(filter(None, datas)) + tdSql.query(f"select sum( {condition} ) from {tbname} {where_condition} ") + tdSql.checkData(0, 0, sum_data) + + tdSql.query(f"select {condition} from {tbname} {where_condition} {group_condition} ") + + def __sum_err_check(self,tbanme): + sqls = [] + + for un_num_col in UN_NUM_COL: + sqls.extend( + ( + f"select sum( {un_num_col} ) from {tbanme} ", + f"select sum(ceil( {un_num_col} )) from {tbanme} ", + ) + ) + sqls.extend( f"select sum( {un_num_col} + {un_num_col_2} ) from {tbanme} " for un_num_col_2 in UN_NUM_COL ) + + sqls.extend( f"select sum( {num_col} + {ts_col} ) from {tbanme} " for num_col in NUM_COL for ts_col in TS_TYPE_COL) + sqls.extend( + ( + f"select sum() from {tbanme} ", + f"select sum(*) from {tbanme} ", + f"select sum(ccccccc) from {tbanme} ", + f"select sum('test') from {tbanme} ", + ) + ) + + return sqls + + def __test_current(self): + tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") + tbname = ["ct1", "ct2", "ct4", "t1"] + for tb in tbname: + self.__sum_current_check(tb) + tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========") + + def __test_error(self): + tdLog.printNoPrefix("==========err sql condition check , must return error==========") + tbname = ["ct1", "ct2", "ct4", "t1"] + + for tb in tbname: + for errsql in self.__sum_err_check(tb): + tdSql.error(sql=errsql) + tdLog.printNoPrefix(f"==========err sql condition check in {tb} over==========") + + + def all_test(self): + self.__test_current() + self.__test_error() + + + def __create_tb(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + create_stb_sql = f'''create table stb1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + ) tags (t1 int) + ''' + create_ntb_sql = f'''create table t1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + ) + ''' + tdSql.execute(create_stb_sql) + tdSql.execute(create_ntb_sql) + + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + def __insert_data(self, rows): + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct2 values ( now()-{i*90}d, {-1*i}, {-11111*i}, {-111*i}, {-11*i}, {-1.11*i}, {-11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + '''insert into ct1 values + ( now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a ) + ( now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a ) + ''' + ) + + tdSql.execute( + f'''insert into ct4 values + ( now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( now()+{rows * 9}d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( + now()+{rows * 9-10}d, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nachar_limit-1", now()-1d + ) + ( + now()+{rows * 9-20}d, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nachar_limit-2", now()-2d + ) + ''' + ) + + tdSql.execute( + f'''insert into ct2 values + ( now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( now()+{rows * 9}d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( + now()+{rows * 9-10}d, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, + { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nachar_limit-1", now()-1d + ) + ( + now()+{rows * 9-20}d, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, + { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nachar_limit-2", now()-2d + ) + ''' + ) + + for i in range(rows): + insert_data = f'''insert into t1 values + ( now()-{i}h, {i}, {i}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, + "binary_{i}", "nchar_{i}", now()-{i}s ) + ''' + tdSql.execute(insert_data) + tdSql.execute( + f'''insert into t1 values + ( now() + 3h, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( now()-{ ( rows // 2 ) * 60 + 30 }m, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( now()-{rows}h, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( now() + 2h, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, + "binary_limit-1", "nachar_limit-1", now()-1d + ) + ( + now() + 1h , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, + "binary_limit-2", "nachar_limit-2", now()-2d + ) + ''' + ) + + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + self.__create_tb() + + tdLog.printNoPrefix("==========step2:insert data") + self.__insert_data(100) + + tdLog.printNoPrefix("==========step3:all check") + self.all_test() + + # tdDnodes.stop(1) + # tdDnodes.start(1) + + # tdSql.execute("use db") + + # tdLog.printNoPrefix("==========step4:after wal, all check again ") + # self.all_test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 65e4785b5d10cbe1baf6ec5e3f3816fd9b7c21d9..2954358feb6bcf77e6ed1406e89f5bdaeab4eb3d 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -2,6 +2,6 @@ set -e #python3 ./test.py -f 2-query/between.py -python3 ./test.py -f 2-query/distinct.py +#python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py -python3 ./test.py -f 2-query/cast.py +#python3 ./test.py -f 2-query/cast.py diff --git a/tools/shell/inc/shell.h b/tools/shell/inc/shell.h deleted file mode 100644 index 866cd63bdbfdfe3337d7075b798a56dc043b3fa3..0000000000000000000000000000000000000000 --- a/tools/shell/inc/shell.h +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_SHELL_H_ -#define _TD_SHELL_H_ - -#include "os.h" - -#include "taos.h" -#include "taosdef.h" - -#define MAX_USERNAME_SIZE 64 -#define MAX_DBNAME_SIZE 64 -#define MAX_IP_SIZE 20 -#define MAX_HISTORY_SIZE 1000 -#define MAX_COMMAND_SIZE 1048586 -#define HISTORY_FILE ".taos_history" - -#define DEFAULT_RES_SHOW_NUM 100 - -typedef struct SShellHistory { - char* hist[MAX_HISTORY_SIZE]; - int hstart; - int hend; -} SShellHistory; - -typedef struct SShellArguments { - char* host; - char* password; - char* user; - char* auth; - char* database; - char* timezone; - bool is_raw_time; - bool is_use_passwd; - bool dump_config; - char file[TSDB_FILENAME_LEN]; - char dir[TSDB_FILENAME_LEN]; - int threadNum; - int check; - bool status; - bool verbose; - char* commands; - int abort; - int port; - int pktLen; - int pktNum; - char* pktType; - char* netTestRole; -} SShellArguments; - -/**************** Function declarations ****************/ -extern void shellParseArgument(int argc, char* argv[], SShellArguments* arguments); -extern TAOS* shellInit(SShellArguments* args); -extern void* shellLoopQuery(void* arg); -extern void taos_error(TAOS_RES* tres, int64_t st); -extern int regex_match(const char* s, const char* reg, int cflags); -int32_t shellReadCommand(TAOS* con, char command[]); -int32_t shellRunCommand(TAOS* con, char* command); -void shellRunCommandOnServer(TAOS* con, char command[]); -void read_history(); -void write_history(); -void source_file(TAOS* con, char* fptr); -void source_dir(TAOS* con, SShellArguments* args); -void get_history_path(char* history); -void shellCheck(TAOS* con, SShellArguments* args); -void cleanup_handler(void* arg); -void exitShell(); -int shellDumpResult(TAOS_RES* con, char* fname, int* error_no, bool printMode); -void shellGetGrantInfo(void *con); -int isCommentLine(char *line); - -/**************** Global variable declarations ****************/ -extern char PROMPT_HEADER[]; -extern char CONTINUE_PROMPT[]; -extern int prompt_size; -extern SShellHistory history; -extern SShellArguments args; -extern int64_t result; - -#endif diff --git a/tools/shell/inc/shellCommand.h b/tools/shell/inc/shellCommand.h deleted file mode 100644 index 49f7dc01331e810864f87bd5b65fb13815fc7719..0000000000000000000000000000000000000000 --- a/tools/shell/inc/shellCommand.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_SHELL_COMMAND_H_ -#define _TD_SHELL_COMMAND_H_ - -#include "shell.h" - -#define LEFT 1 -#define RIGHT 2 -#define UP 3 -#define DOWN 4 - -typedef struct Command Command; -struct Command { - char * buffer; - char * command; - unsigned commandSize; - unsigned bufferSize; - unsigned cursorOffset; - unsigned screenOffset; - unsigned endOffset; -}; - -extern void backspaceChar(Command *cmd); -extern void clearLineBefore(Command *cmd); -extern void clearLineAfter(Command *cmd); -extern void deleteChar(Command *cmd); -extern void moveCursorLeft(Command *cmd); -extern void moveCursorRight(Command *cmd); -extern void positionCursorHome(Command *cmd); -extern void positionCursorEnd(Command *cmd); -extern void showOnScreen(Command *cmd); -extern void updateBuffer(Command *cmd); -extern int isReadyGo(Command *cmd); -extern void resetCommand(Command *cmd, const char s[]); - -int countPrefixOnes(unsigned char c); -void clearScreen(int ecmd_pos, int cursor_pos); -void printChar(char c, int times); -void positionCursor(int step, int direction); - -#endif diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h new file mode 100644 index 0000000000000000000000000000000000000000..af283735101e3b9bb82acccd6ce2bd83e01165f1 --- /dev/null +++ b/tools/shell/inc/shellInt.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_SHELL_INT_H_ +#define _TD_SHELL_INT_H_ + +#include "os.h" +#include "taos.h" +#include "taosdef.h" +#include "taoserror.h" +#include "tconfig.h" +#include "tglobal.h" +#include "trpc.h" +#include "ttypes.h" +#include "tutil.h" + +#define SHELL_MAX_HISTORY_SIZE 1000 +#define SHELL_MAX_COMMAND_SIZE 1048586 +#define SHELL_HISTORY_FILE ".taos_history" +#define SHELL_DEFAULT_RES_SHOW_NUM 100 +#define SHELL_DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30 +#define SHELL_MAX_PKG_LEN 2 * 1024 * 1024 +#define SHELL_MIN_PKG_LEN 1 +#define SHELL_DEF_PKG_LEN 1024 +#define SHELL_MAX_PKG_NUM 1 * 1024 * 1024 +#define SHELL_MIN_PKG_NUM 1 +#define SHELL_DEF_PKG_NUM 100 + +typedef struct { + char* hist[SHELL_MAX_HISTORY_SIZE]; + char file[TSDB_FILENAME_LEN]; + int32_t hstart; + int32_t hend; +} SShellHistory; + +typedef struct { + const char* host; + const char* user; + const char* auth; + const char* database; + const char* cfgdir; + const char* commands; + const char* netrole; + char file[PATH_MAX]; + char password[TSDB_USET_PASSWORD_LEN]; + bool is_gen_auth; + bool is_raw_time; + bool is_version; + bool is_dump_config; + bool is_check; + bool is_startup; + bool is_help; + uint16_t port; + int32_t pktLen; + int32_t pktNum; + int32_t displayWidth; + int32_t abort; +} SShellArgs; + +typedef struct { + const char* clientVersion; + const char* promptHeader; + const char* promptContinue; + const char* osname; + int32_t promptSize; + char programVersion[32]; +} SShellOsDetails; + +typedef struct { + SShellArgs args; + SShellHistory history; + SShellOsDetails info; + TAOS* conn; + TdThread pid; + tsem_t cancelSem; + int64_t result; +} SShellObj; + +// shellArguments.c +int32_t shellParseArgs(int32_t argc, char* argv[]); + +// shellCommand.c +int32_t shellReadCommand(char* command); + +// shellEngine.c +int32_t shellExecute(); + +// shellUtil.c +int32_t shellCheckIntSize(); +void shellPrintVersion(); +void shellPrintHelp(); +void shellGenerateAuth(); +void shellDumpConfig(); +void shellCheckServerStatus(); +bool shellRegexMatch(const char* s, const char* reg, int32_t cflags); +void shellExit(); + +// shellNettest.c +void shellTestNetWork(); + +// shellMain.c +extern SShellObj shell; +extern void taos_init(); + +#endif /*_TD_SHELL_INT_H_*/ diff --git a/tools/shell/inc/syncMsg.h b/tools/shell/inc/syncMsg.h deleted file mode 100644 index 85ac9c78affa5282d5ca703caffc1bc5c24461bb..0000000000000000000000000000000000000000 --- a/tools/shell/inc/syncMsg.h +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_SYNC_MSG_H -#define TDENGINE_SYNC_MSG_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "tsync.h" - -typedef enum { - TAOS_SMSG_START = 0, - TAOS_SMSG_SYNC_DATA = 1, - TAOS_SMSG_SYNC_DATA_RSP = 2, - TAOS_SMSG_SYNC_FWD = 3, - TAOS_SMSG_SYNC_FWD_RSP = 4, - TAOS_SMSG_SYNC_REQ = 5, - TAOS_SMSG_SYNC_REQ_RSP = 6, - TAOS_SMSG_SYNC_MUST = 7, - TAOS_SMSG_SYNC_MUST_RSP = 8, - TAOS_SMSG_STATUS = 9, - TAOS_SMSG_STATUS_RSP = 10, - TAOS_SMSG_SETUP = 11, - TAOS_SMSG_SETUP_RSP = 12, - TAOS_SMSG_SYNC_FILE = 13, - TAOS_SMSG_SYNC_FILE_RSP = 14, - TAOS_SMSG_TEST = 15, - TAOS_SMSG_END = 16 -} ESyncMsgType; - -typedef enum { - SYNC_STATUS_BROADCAST, - SYNC_STATUS_BROADCAST_RSP, - SYNC_STATUS_SETUP_CONN, - SYNC_STATUS_SETUP_CONN_RSP, - SYNC_STATUS_EXCHANGE_DATA, - SYNC_STATUS_EXCHANGE_DATA_RSP, - SYNC_STATUS_CHECK_ROLE, - SYNC_STATUS_CHECK_ROLE_RSP -} ESyncStatusType; - -#pragma pack(push, 1) - -typedef struct { - int8_t type; // msg type - int8_t protocol; // protocol version - uint16_t signature; // fixed value - int32_t code; // - int32_t cId; // cluster Id - int32_t vgId; // vg ID - int32_t len; // content length, does not include head - uint32_t cksum; -} SSyncHead; - -typedef struct { - SSyncHead head; - uint16_t port; - uint16_t tranId; - int32_t sourceId; // only for arbitrator - char fqdn[TSDB_FQDN_LEN]; -} SSyncMsg; - -typedef struct { - SSyncHead head; - int8_t sync; - int8_t reserved; - uint16_t tranId; - int8_t reserverd[4]; -} SSyncRsp; - -typedef struct { - int8_t role; - uint64_t version; -} SPeerStatus; - -typedef struct { - SSyncHead head; - int8_t role; - int8_t ack; - int8_t type; - int8_t reserved[3]; - uint16_t tranId; - uint64_t version; - SPeerStatus peersStatus[TAOS_SYNC_MAX_REPLICA]; -} SPeersStatus; - -typedef struct { - SSyncHead head; - uint64_t fversion; -} SFileVersion; - -typedef struct { - SSyncHead head; - int8_t ack; -} SFileAck; - -typedef struct { - SSyncHead head; - uint64_t version; - int32_t code; -} SFwdRsp; - -#pragma pack(pop) - -#define SYNC_PROTOCOL_VERSION 1 -#define SYNC_SIGNATURE ((uint16_t)(0xCDEF)) - -extern char *statusType[]; - -uint16_t syncGenTranId(); -int32_t syncCheckHead(SSyncHead *pHead); - -void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len); -void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code); -void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId); -void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId); -void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId); -void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId); -void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId); - -void syncBuildFileAck(SFileAck *pMsg, int32_t vgId); -void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_VNODEPEER_H diff --git a/tools/shell/inc/tsync.h b/tools/shell/inc/tsync.h deleted file mode 100644 index d1b68e3f5a1e27b63dd08a4d1d4862c7b1e68179..0000000000000000000000000000000000000000 --- a/tools/shell/inc/tsync.h +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_SYNC_H -#define TDENGINE_SYNC_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define TAOS_SYNC_MAX_REPLICA 5 -#define TAOS_SYNC_MAX_INDEX 0x7FFFFFFF - -typedef enum { - TAOS_SYNC_ROLE_OFFLINE = 0, - TAOS_SYNC_ROLE_UNSYNCED = 1, - TAOS_SYNC_ROLE_SYNCING = 2, - TAOS_SYNC_ROLE_SLAVE = 3, - TAOS_SYNC_ROLE_MASTER = 4 -} ESyncRole; - -typedef enum { - TAOS_SYNC_STATUS_INIT = 0, - TAOS_SYNC_STATUS_START = 1, - TAOS_SYNC_STATUS_FILE = 2, - TAOS_SYNC_STATUS_CACHE = 3 -} ESyncStatus; - -typedef struct { - uint32_t nodeId; // node ID assigned by TDengine - uint16_t nodePort; // node sync Port - char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN -} SNodeInfo; - -typedef struct { - int8_t quorum; // number of confirms required, >=1 - int8_t replica; // number of replications, >=1 - SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA]; -} SSyncCfg; - -typedef struct { - int32_t selfIndex; - uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; - int32_t role[TAOS_SYNC_MAX_REPLICA]; -} SNodesRole; - -// get the wal file from index or after -// return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file -typedef int32_t (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); - -// when a forward pkt is received, call this to handle data -typedef int32_t (*FWriteToCache)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg); - -// when forward is confirmed by peer, master call this API to notify app -typedef void (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code); - -// when role is changed, call this to notify app -typedef void (*FNotifyRole)(int32_t vgId, int8_t role); - -// if a number of retrieving data failed, call this to start flow control -typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level); - -// when data file is synced successfully, notity app -typedef void (*FStartSyncFile)(int32_t vgId); -typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion); - -// get file version -typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver); - -typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd); -typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd); - -typedef struct { - int32_t vgId; // vgroup ID - uint64_t version; // initial version - SSyncCfg syncCfg; // configuration from mgmt - char path[TSDB_FILENAME_LEN]; // path to the file - void * pTsdb; - FGetWalInfo getWalInfoFp; - FWriteToCache writeToCacheFp; - FConfirmForward confirmForward; - FNotifyRole notifyRoleFp; - FNotifyFlowCtrl notifyFlowCtrlFp; - FStartSyncFile startSyncFileFp; - FStopSyncFile stopSyncFileFp; - FGetVersion getVersionFp; - FSendFile sendFileFp; - FRecvFile recvFileFp; -} SSyncInfo; - -typedef void *tsync_h; - -int32_t syncInit(); -void syncCleanUp(); - -int64_t syncStart(const SSyncInfo *); -void syncStop(int64_t rid); -int32_t syncReconfig(int64_t rid, const SSyncCfg *); -int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype, bool force); -void syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force); -void syncRecover(int64_t rid); // recover from other nodes: -int32_t syncGetNodesRole(int64_t rid, SNodesRole *); - -extern char *syncRole[]; - -//global configurable parameters -extern int32_t sDebugFlag; -extern char tsArbitrator[]; -extern uint16_t tsSyncPort; - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_SYNC_H diff --git a/tools/shell/src/backup/shellCheck.c b/tools/shell/src/backup/shellCheck.c deleted file mode 100644 index d1f0683fea51df5c4d3d4ff977cf218988c42ec3..0000000000000000000000000000000000000000 --- a/tools/shell/src/backup/shellCheck.c +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _GNU_SOURCE -#define _XOPEN_SOURCE -#define _DEFAULT_SOURCE - -#include "os.h" -#include "shell.h" -#include "shellCommand.h" -#include "tglobal.h" -#include "tutil.h" - -#define SHELL_SQL_LEN 1024 -static int32_t tbNum = 0; -static int32_t tbMallocNum = 0; -static char ** tbNames = NULL; -static int32_t checkedNum = 0; -static int32_t errorNum = 0; - -typedef struct { - TdThread threadID; - int threadIndex; - int totalThreads; - void * taos; - char * db; -} ShellThreadObj; - -static int32_t shellUseDb(TAOS *con, char *db) { - if (db == NULL) { - fprintf(stdout, "no dbname input\n"); - return -1; - } - - char sql[SHELL_SQL_LEN] = {0}; - snprintf(sql, SHELL_SQL_LEN, "use %s", db); - - TAOS_RES *pSql = taos_query(con, sql); - int32_t code = taos_errno(pSql); - if (code != 0) { - fprintf(stdout, "failed to execute sql:%s since %s", sql, taos_errstr(pSql)); - } - - taos_free_result(pSql); - return code; -} - -static int32_t shellShowTables(TAOS *con, char *db) { - char sql[SHELL_SQL_LEN] = {0}; - snprintf(sql, SHELL_SQL_LEN, "show %s.tables", db); - - TAOS_RES *pSql = taos_query(con, sql); - int32_t code = taos_errno(pSql); - - if (code != 0) { - fprintf(stdout, "failed to execute sql:%s since %s\n", sql, taos_errstr(pSql)); - } else { - TAOS_ROW row; - while ((row = taos_fetch_row(pSql))) { - int32_t tbIndex = tbNum++; - if (tbMallocNum < tbNum) { - tbMallocNum = (tbMallocNum * 2 + 1); - char** tbNames1 = taosMemoryRealloc(tbNames, tbMallocNum * sizeof(char *)); - if (tbNames1 == NULL) { - fprintf(stdout, "failed to malloc tablenames, num:%d\n", tbMallocNum); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - break; - } - tbNames = tbNames1; - } - - tbNames[tbIndex] = taosMemoryMalloc(TSDB_TABLE_NAME_LEN); - strncpy(tbNames[tbIndex], (const char *)row[0], TSDB_TABLE_NAME_LEN); - if (tbIndex % 100000 == 0 && tbIndex != 0) { - fprintf(stdout, "%d tablenames fetched\n", tbIndex); - } - } - } - - taos_free_result(pSql); - - fprintf(stdout, "total %d tablenames fetched, over\n", tbNum); - return code; -} - -static void shellFreeTbnames() { - for (int32_t i = 0; i < tbNum; ++i) { - taosMemoryFree(tbNames[i]); - } - taosMemoryFree(tbNames); -} - -static void *shellCheckThreadFp(void *arg) { - ShellThreadObj *pThread = (ShellThreadObj *)arg; - - setThreadName("shellCheckThrd"); - - int32_t interval = tbNum / pThread->totalThreads + 1; - int32_t start = pThread->threadIndex * interval; - int32_t end = (pThread->threadIndex + 1) * interval; - - if (end > tbNum) end = tbNum + 1; - - char file[32] = {0}; - snprintf(file, 32, "tb%d.txt", pThread->threadIndex); - - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (!fp) { - fprintf(stdout, "failed to open %s, reason:%s", file, strerror(errno)); - return NULL; - } - - char sql[SHELL_SQL_LEN]; - for (int32_t t = start; t < end; ++t) { - char *tbname = tbNames[t]; - if (tbname == NULL) break; - - snprintf(sql, SHELL_SQL_LEN, "select * from %s limit 1", tbname); - - TAOS_RES *pSql = taos_query(pThread->taos, sql); - int32_t code = taos_errno(pSql); - if (code != 0) { - int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname); - taosWriteFile(pFile, sql, len); - atomic_add_fetch_32(&errorNum, 1); - } - - int32_t cnum = atomic_add_fetch_32(&checkedNum, 1); - if (cnum % 5000 == 0 && cnum != 0) { - fprintf(stdout, "%d tables checked\n", cnum); - } - - taos_free_result(pSql); - } - - taosFsync(pFile); - taosCloseFile(&pFile); - - return NULL; -} - -static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) { - TdThreadAttr thattr; - ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj)); - for (int t = 0; t < _args->threadNum; ++t) { - ShellThreadObj *pThread = threadObj + t; - pThread->threadIndex = t; - pThread->totalThreads = _args->threadNum; - pThread->taos = con; - pThread->db = _args->database; - - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - - if (taosThreadCreate(&(pThread->threadID), &thattr, shellCheckThreadFp, (void *)pThread) != 0) { - fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); - exit(0); - } - } - - for (int t = 0; t < _args->threadNum; ++t) { - taosThreadJoin(threadObj[t].threadID, NULL); - } - - for (int t = 0; t < _args->threadNum; ++t) { - taos_close(threadObj[t].taos); - } - taosMemoryFree(threadObj); -} - -void shellCheck(TAOS *con, SShellArguments *_args) { - int64_t start = taosGetTimestampMs(); - - if (shellUseDb(con, _args->database) != 0) { - shellFreeTbnames(); - return; - } - - if (shellShowTables(con, _args->database) != 0) { - shellFreeTbnames(); - return; - } - - fprintf(stdout, "total %d tables will be checked by %d threads\n", tbNum, _args->threadNum); - shellRunCheckThreads(con, _args); - - int64_t end = taosGetTimestampMs(); - fprintf(stdout, "total %d tables checked, failed:%d, time spent %.2f seconds\n", checkedNum, errorNum, - (end - start) / 1000.0); -} diff --git a/tools/shell/src/backup/shellDarwin.c b/tools/shell/src/backup/shellDarwin.c deleted file mode 100644 index 93335776bad446d4ad9575efa838ed1723421901..0000000000000000000000000000000000000000 --- a/tools/shell/src/backup/shellDarwin.c +++ /dev/null @@ -1,498 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define __USE_XOPEN - -#include "os.h" - -#include "shell.h" -#include "shellCommand.h" -#include "tbase64.h" - -#include "tscLog.h" - -#define OPT_ABORT 1 /* �Cabort */ - -int indicator = 1; -struct termios oldtio; - -void insertChar(Command *cmd, char *c, int size); - - -void printHelp() { - char indent[10] = " "; - printf("taos shell is used to test the TDengine database\n"); - - printf("%s%s\n", indent, "-h"); - printf("%s%s%s\n", indent, indent, "TDengine server IP address to connect. The default host is localhost."); - printf("%s%s\n", indent, "-p"); - printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server."); - printf("%s%s\n", indent, "-P"); - printf("%s%s%s\n", indent, indent, "The TCP/IP port number to use for the connection"); - printf("%s%s\n", indent, "-u"); - printf("%s%s%s\n", indent, indent, "The user name to use when connecting to the server."); - printf("%s%s\n", indent, "-c"); - printf("%s%s%s\n", indent, indent, "Configuration directory."); - printf("%s%s\n", indent, "-s"); - printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell."); - printf("%s%s\n", indent, "-r"); - printf("%s%s%s\n", indent, indent, "Output time as unsigned long.."); - printf("%s%s\n", indent, "-f"); - printf("%s%s%s\n", indent, indent, "Script to run without enter the shell."); - printf("%s%s\n", indent, "-d"); - printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server."); - printf("%s%s\n", indent, "-t"); - printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local."); - printf("%s%s\n", indent, "-D"); - printf("%s%s%s\n", indent, indent, "Use multi-thread to import all SQL files in the directory separately."); - printf("%s%s\n", indent, "-T"); - printf("%s%s%s\n", indent, indent, "Number of threads when using multi-thread to import data."); - - exit(EXIT_SUCCESS); -} - -char DARWINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; -char g_password[SHELL_MAX_PASSWORD_LEN]; - -void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { - wordexp_t full_path; - for (int i = 1; i < argc; i++) { - // for host - if (strcmp(argv[i], "-h") == 0) { - if (i < argc - 1) { - arguments->host = argv[++i]; - } else { - fprintf(stderr, "option -h requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // for password - else if ((strncmp(argv[i], "-p", 2) == 0) - || (strncmp(argv[i], "--password", 10) == 0)) { - strcpy(tsOsName, "Darwin"); - printf(DARWINCLIENT_VERSION, tsOsName, taos_get_client_info()); - if ((strlen(argv[i]) == 2) - || (strncmp(argv[i], "--password", 10) == 0)) { - printf("Enter password: "); - taosSetConsoleEcho(false); - if (scanf("%s", g_password) > 1) { - fprintf(stderr, "password read error\n"); - } - taosSetConsoleEcho(true); - getchar(); - } else { - tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); - } - arguments->password = g_password; - arguments->is_use_passwd = true; - strcpy(argv[i], ""); - argc -= 1; - } - // for management port - else if (strcmp(argv[i], "-P") == 0) { - if (i < argc - 1) { - arguments->port = atoi(argv[++i]); - } else { - fprintf(stderr, "option -P requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // for user - else if (strcmp(argv[i], "-u") == 0) { - if (i < argc - 1) { - arguments->user = argv[++i]; - } else { - fprintf(stderr, "option -u requires an argument\n"); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { - if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) { - fprintf(stderr, "config file path: %s overflow max len %d\n", argv[i], TSDB_FILENAME_LEN - 1); - exit(EXIT_FAILURE); - } - strcpy(configDir, argv[i]); - } else { - fprintf(stderr, "Option -c requires an argument\n"); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-s") == 0) { - if (i < argc - 1) { - arguments->commands = argv[++i]; - } else { - fprintf(stderr, "option -s requires an argument\n"); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-r") == 0) { - arguments->is_raw_time = true; - } - // For temperory batch commands to run TODO - else if (strcmp(argv[i], "-f") == 0) { - if (i < argc - 1) { - strcpy(arguments->file, argv[++i]); - } else { - fprintf(stderr, "option -f requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // for default database - else if (strcmp(argv[i], "-d") == 0) { - if (i < argc - 1) { - arguments->database = argv[++i]; - } else { - fprintf(stderr, "option -d requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // For time zone - else if (strcmp(argv[i], "-t") == 0) { - if (i < argc - 1) { - arguments->timezone = argv[++i]; - } else { - fprintf(stderr, "option -t requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // For import directory - else if (strcmp(argv[i], "-D") == 0) { - if (i < argc - 1) { - if (wordexp(argv[++i], &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", argv[i]); - exit(EXIT_FAILURE); - } - strcpy(arguments->dir, full_path.we_wordv[0]); - wordfree(&full_path); - } else { - fprintf(stderr, "option -D requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // For time zone - else if (strcmp(argv[i], "-T") == 0) { - if (i < argc - 1) { - arguments->threadNum = atoi(argv[++i]); - } else { - fprintf(stderr, "option -T requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // For temperory command TODO - else if (strcmp(argv[i], "--help") == 0) { - printHelp(); - exit(EXIT_FAILURE); - } else { - fprintf(stderr, "wrong options\n"); - printHelp(); - exit(EXIT_FAILURE); - } - } -} - -int32_t shellReadCommand(TAOS *con, char *command) { - unsigned hist_counter = history.hend; - char utf8_array[10] = "\0"; - Command cmd; - memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); - showOnScreen(&cmd); - - // Read input. - char c; - while (1) { - c = getchar(); - - if (c < 0) { // For UTF-8 - int count = countPrefixOnes(c); - utf8_array[0] = c; - for (int k = 1; k < count; k++) { - c = getchar(); - utf8_array[k] = c; - } - insertChar(&cmd, utf8_array, count); - } else if (c < '\033') { - // Ctrl keys. TODO: Implement ctrl combinations - switch (c) { - case 1: // ctrl A - positionCursorHome(&cmd); - break; - case 3: - printf("\n"); - resetCommand(&cmd, ""); - kill(0, SIGINT); - break; - case 4: // EOF or Ctrl+D - printf("\n"); - taos_close(con); - // write the history - write_history(); - exitShell(); - break; - case 5: // ctrl E - positionCursorEnd(&cmd); - break; - case 8: - backspaceChar(&cmd); - break; - case '\n': - case '\r': - printf("\n"); - if (isReadyGo(&cmd)) { - sprintf(command, "%s%s", cmd.buffer, cmd.command); - taosMemoryFreeClear(cmd.buffer); - taosMemoryFreeClear(cmd.command); - return 0; - } else { - updateBuffer(&cmd); - } - break; - case 11: // Ctrl + K; - clearLineAfter(&cmd); - break; - case 12: // Ctrl + L; - system("clear"); - showOnScreen(&cmd); - break; - case 21: // Ctrl + U - clearLineBefore(&cmd); - break; - } - } else if (c == '\033') { - c = getchar(); - switch (c) { - case '[': - c = getchar(); - switch (c) { - case 'A': // Up arrow - if (hist_counter != history.hstart) { - hist_counter = (hist_counter + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE; - resetCommand(&cmd, (history.hist[hist_counter] == NULL) ? "" : history.hist[hist_counter]); - } - break; - case 'B': // Down arrow - if (hist_counter != history.hend) { - int next_hist = (hist_counter + 1) % MAX_HISTORY_SIZE; - - if (next_hist != history.hend) { - resetCommand(&cmd, (history.hist[next_hist] == NULL) ? "" : history.hist[next_hist]); - } else { - resetCommand(&cmd, ""); - } - hist_counter = next_hist; - } - break; - case 'C': // Right arrow - moveCursorRight(&cmd); - break; - case 'D': // Left arrow - moveCursorLeft(&cmd); - break; - case '1': - if ((c = getchar()) == '~') { - // Home key - positionCursorHome(&cmd); - } - break; - case '2': - if ((c = getchar()) == '~') { - // Insert key - } - break; - case '3': - if ((c = getchar()) == '~') { - // Delete key - deleteChar(&cmd); - } - break; - case '4': - if ((c = getchar()) == '~') { - // End key - positionCursorEnd(&cmd); - } - break; - case '5': - if ((c = getchar()) == '~') { - // Page up key - } - break; - case '6': - if ((c = getchar()) == '~') { - // Page down key - } - break; - case 72: - // Home key - positionCursorHome(&cmd); - break; - case 70: - // End key - positionCursorEnd(&cmd); - break; - } - break; - } - } else if (c == 0x7f) { - // press delete key - backspaceChar(&cmd); - } else { - insertChar(&cmd, &c, 1); - } - } - - return 0; -} - -void *shellLoopQuery(void *arg) { - if (indicator) { - getOldTerminalMode(); - indicator = 0; - } - - TAOS *con = (TAOS *)arg; - - setThreadName("shellLoopQuery"); - - taosThreadCleanupPush(cleanup_handler, NULL); - - char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); - if (command == NULL){ - tscError("failed to malloc command"); - return NULL; - } - - int32_t err = 0; - - do { - // Read command from shell. - memset(command, 0, MAX_COMMAND_SIZE); - setTerminalMode(); - err = shellReadCommand(con, command); - if (err) { - break; - } - resetTerminalMode(); - } while (shellRunCommand(con, command) == 0); - - taosMemoryFreeClear(command); - exitShell(); - - taosThreadCleanupPop(1); - - return NULL; -} - -void get_history_path(char *history) { sprintf(history, "%s/%s", getpwuid(getuid())->pw_dir, HISTORY_FILE); } - -void clearScreen(int ecmd_pos, int cursor_pos) { - struct winsize w; - if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - //fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); - w.ws_col = 120; - w.ws_row = 30; - } - - int cursor_x = cursor_pos / w.ws_col; - int cursor_y = cursor_pos % w.ws_col; - int command_x = ecmd_pos / w.ws_col; - positionCursor(cursor_y, LEFT); - positionCursor(command_x - cursor_x, DOWN); - fprintf(stdout, "\033[2K"); - for (int i = 0; i < command_x; i++) { - positionCursor(1, UP); - fprintf(stdout, "\033[2K"); - } - fflush(stdout); -} - -void showOnScreen(Command *cmd) { - struct winsize w; - if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - //fprintf(stderr, "No stream device\n"); - w.ws_col = 120; - w.ws_row = 30; - } - - TdWchar wc; - int size = 0; - - // Print out the command. - char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE); - memset(total_string, '\0', MAX_COMMAND_SIZE); - if (strcmp(cmd->buffer, "") == 0) { - sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command); - } else { - sprintf(total_string, "%s%s", CONTINUE_PROMPT, cmd->command); - } - - int remain_column = w.ws_col; - /* size = cmd->commandSize + prompt_size; */ - for (char *str = total_string; size < cmd->commandSize + prompt_size;) { - int ret = taosMbToWchar(&wc, str, MB_CUR_MAX); - if (ret < 0) break; - size += ret; - /* assert(size >= 0); */ - int width = taosWcharWidth(wc); - if (remain_column > width) { - printf("%lc", wc); - remain_column -= width; - } else { - if (remain_column == width) { - printf("%lc\n\r", wc); - remain_column = w.ws_col; - } else { - printf("\n\r%lc", wc); - remain_column = w.ws_col - width; - } - } - - str = total_string + size; - } - - taosMemoryFree(total_string); - /* for (int i = 0; i < size; i++){ */ - /* char c = total_string[i]; */ - /* if (k % w.ws_col == 0) { */ - /* printf("%c\n\r", c); */ - /* } */ - /* else { */ - /* printf("%c", c); */ - /* } */ - /* k += 1; */ - /* } */ - - // Position the cursor - int cursor_pos = cmd->screenOffset + prompt_size; - int ecmd_pos = cmd->endOffset + prompt_size; - - int cursor_x = cursor_pos / w.ws_col; - int cursor_y = cursor_pos % w.ws_col; - // int cursor_y = cursor % w.ws_col; - int command_x = ecmd_pos / w.ws_col; - int command_y = ecmd_pos % w.ws_col; - // int command_y = (command.size() + prompt_size) % w.ws_col; - positionCursor(command_y, LEFT); - positionCursor(command_x, UP); - positionCursor(cursor_x, DOWN); - positionCursor(cursor_y, RIGHT); - fflush(stdout); -} - -void cleanup_handler(void *arg) { resetTerminalMode(); } - -void exitShell() { - resetTerminalMode(); - exit(EXIT_SUCCESS); -} diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c deleted file mode 100644 index 130c72a20bbdb831fb1fe3fa8d62ca93f5aa45dd..0000000000000000000000000000000000000000 --- a/tools/shell/src/backup/shellImport.c +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _GNU_SOURCE -#define _XOPEN_SOURCE -#define _DEFAULT_SOURCE - -#include "os.h" -#include "shell.h" -#include "shellCommand.h" -#include "tglobal.h" -#include "tutil.h" - -static char **shellSQLFiles = NULL; -static int32_t shellSQLFileNum = 0; -static char shellTablesSQLFile[TSDB_FILENAME_LEN] = {0}; - -typedef struct { - TdThread threadID; - int threadIndex; - int totalThreads; - void *taos; -} ShellThreadObj; - -static int shellGetFilesNum(const char *directoryName, const char *prefix) -{ - char cmd[1024] = { 0 }; - sprintf(cmd, "ls %s/*.%s | wc -l ", directoryName, prefix); - - char buf[1024] = { 0 }; - if (taosSystem(cmd, buf, sizeof(buf)) < 0) { - fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno)); - exit(0); - } - - int fileNum = 0; - if (sscanf(buf, "%d", &fileNum) != 1) { - fprintf(stderr, "ERROR: failed to execute:%s, parse result error\n", cmd); - exit(0); - } - - if (fileNum <= 0) { - fprintf(stderr, "ERROR: directory:%s is empry\n", directoryName); - exit(0); - } - - return fileNum; -} - -static void shellParseDirectory(const char *directoryName, const char *prefix, char **fileArray, int totalFiles) -{ - char cmd[1024] = { 0 }; - sprintf(cmd, "ls %s/*.%s | sort", directoryName, prefix); - - char buf[1024] = { 0 }; - if (taosSystem(cmd, buf, sizeof(buf)) < 0) { - fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno)); - exit(0); - } - - int fileNum = 0; - while (sscanf(buf, "%128s", fileArray[fileNum++])) { - if (strcmp(fileArray[fileNum-1], shellTablesSQLFile) == 0) { - fileNum--; - } - if (fileNum >= totalFiles) { - break; - } - } - - if (fileNum != totalFiles) { - fprintf(stderr, "ERROR: directory:%s changed while read\n", directoryName); - exit(0); - } -} - -static void shellCheckTablesSQLFile(const char *directoryName) -{ - sprintf(shellTablesSQLFile, "%s/tables.sql", directoryName); - - if (taosFStatFile(shellTablesSQLFile, NULL, NULL) < 0) { - shellTablesSQLFile[0] = 0; - } -} - -static void shellMallocSQLFiles() -{ - shellSQLFiles = (char**)taosMemoryCalloc(shellSQLFileNum, sizeof(char*)); - for (int i = 0; i < shellSQLFileNum; i++) { - shellSQLFiles[i] = taosMemoryCalloc(1, TSDB_FILENAME_LEN); - } -} - -static void shellGetDirectoryFileList(char *inputDir) -{ - if (!taosDirExist(inputDir)) { - fprintf(stderr, "ERROR: %s not exist\n", inputDir); - exit(0); - } - - if (taosIsDir(inputDir)) { - shellCheckTablesSQLFile(inputDir); - shellSQLFileNum = shellGetFilesNum(inputDir, "sql"); - int totalSQLFileNum = shellSQLFileNum; - if (shellTablesSQLFile[0] != 0) { - shellSQLFileNum--; - } - shellMallocSQLFiles(); - shellParseDirectory(inputDir, "sql", shellSQLFiles, shellSQLFileNum); - fprintf(stdout, "\nstart to dispose %d files in %s\n", totalSQLFileNum, inputDir); - } - else { - fprintf(stderr, "ERROR: %s is not a directory\n", inputDir); - exit(0); - } -} - -static void shellSourceFile(TAOS *con, char *fptr) { - wordexp_t full_path; - int read_len = 0; - char * cmd = taosMemoryMalloc(tsMaxSQLStringLen); - size_t cmd_len = 0; - char * line = NULL; - - if (wordexp(fptr, &full_path, 0) != 0) { - fprintf(stderr, "ERROR: illegal file name\n"); - taosMemoryFree(cmd); - return; - } - - char *fname = full_path.we_wordv[0]; - if (fname == NULL) { - fprintf(stderr, "ERROR: invalid filename\n"); - taosMemoryFree(cmd); - return; - } - - /* - if (access(fname, F_OK) != 0) { - fprintf(stderr, "ERROR: file %s is not exist\n", fptr); - - wordfree(&full_path); - taosMemoryFree(cmd); - return; - } - - if (access(fname, R_OK) != 0) { - fprintf(stderr, "ERROR: file %s is not readable\n", fptr); - - wordfree(&full_path); - taosMemoryFree(cmd); - return; - } - */ - - // FILE *f = fopen(fname, "r"); - TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) { - fprintf(stderr, "ERROR: failed to open file %s\n", fname); - wordfree(&full_path); - taosMemoryFree(cmd); - return; - } - - fprintf(stdout, "begin import file:%s\n", fname); - - int lineNo = 0; - while ((read_len = taosGetLineFile(pFile, &line)) != -1) { - ++lineNo; - if (read_len >= tsMaxSQLStringLen) continue; - line[--read_len] = '\0'; - - if (read_len == 0 || isCommentLine(line)) { // line starts with # - continue; - } - - if (line[read_len - 1] == '\\') { - line[read_len - 1] = ' '; - memcpy(cmd + cmd_len, line, read_len); - cmd_len += read_len; - continue; - } - - memcpy(cmd + cmd_len, line, read_len); - - TAOS_RES* pSql = taos_query(con, cmd); - int32_t code = taos_errno(pSql); - - if (code != 0) { - fprintf(stderr, "DB error: %s: %s (%d)\n", taos_errstr(pSql), fname, lineNo); - } - - /* free local resouce: allocated memory/metric-meta refcnt */ - taos_free_result(pSql); - - memset(cmd, 0, MAX_COMMAND_SIZE); - cmd_len = 0; - } - - taosMemoryFree(cmd); - if(line != NULL) taosMemoryFree(line); - wordfree(&full_path); - taosCloseFile(&pFile); -} - -void* shellImportThreadFp(void *arg) -{ - ShellThreadObj *pThread = (ShellThreadObj*)arg; - setThreadName("shellImportThrd"); - - for (int f = 0; f < shellSQLFileNum; ++f) { - if (f % pThread->totalThreads == pThread->threadIndex) { - char *SQLFileName = shellSQLFiles[f]; - shellSourceFile(pThread->taos, SQLFileName); - } - } - - return NULL; -} - -static void shellRunImportThreads(SShellArguments* _args) -{ - TdThreadAttr thattr; - ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj)); - for (int t = 0; t < _args->threadNum; ++t) { - ShellThreadObj *pThread = threadObj + t; - pThread->threadIndex = t; - pThread->totalThreads = _args->threadNum; - pThread->taos = taos_connect(_args->host, _args->user, _args->password, _args->database, tsDnodeShellPort); - if (pThread->taos == NULL) { - fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, "null taos"/*taos_errstr(pThread->taos)*/); - exit(0); - } - - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - - if (taosThreadCreate(&(pThread->threadID), &thattr, shellImportThreadFp, (void*)pThread) != 0) { - fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); - exit(0); - } - } - - for (int t = 0; t < _args->threadNum; ++t) { - taosThreadJoin(threadObj[t].threadID, NULL); - } - - for (int t = 0; t < _args->threadNum; ++t) { - taos_close(threadObj[t].taos); - } - taosMemoryFree(threadObj); -} - -void source_dir(TAOS* con, SShellArguments* _args) { - shellGetDirectoryFileList(_args->dir); - int64_t start = taosGetTimestampMs(); - - if (shellTablesSQLFile[0] != 0) { - shellSourceFile(con, shellTablesSQLFile); - int64_t end = taosGetTimestampMs(); - fprintf(stdout, "import %s finished, time spent %.2f seconds\n", shellTablesSQLFile, (end - start) / 1000.0); - } - - shellRunImportThreads(_args); - int64_t end = taosGetTimestampMs(); - fprintf(stdout, "import %s finished, time spent %.2f seconds\n", _args->dir, (end - start) / 1000.0); -} diff --git a/tools/shell/src/backup/shellWindows.c b/tools/shell/src/backup/shellWindows.c deleted file mode 100644 index 92ac7fd7213b13b34d94714b32216b1fb0435477..0000000000000000000000000000000000000000 --- a/tools/shell/src/backup/shellWindows.c +++ /dev/null @@ -1,325 +0,0 @@ -/******************************************************************* -* Copyright (c) 2017 by TAOS Technologies, Inc. -* All rights reserved. -* -* This file is proprietary and confidential to TAOS Technologies. -* No part of this file may be reproduced, stored, transmitted, -* disclosed or used in any form or by any means other than as -* expressly provided by the written permission from Jianhui Tao -* -* ****************************************************************/ - -#include -#include -#include -#include "../../../../include/client/taos.h" -#include "os.h" -#include "shell.h" -#include "shellCommand.h" - -extern char configDir[]; - -char WINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; - -void printVersion() { - printf("version: %s\n", version); -} - -void printHelp() { - char indent[10] = " "; - printf("taos shell is used to test the TDengine database\n"); - - printf("%s%s\n", indent, "-h"); - printf("%s%s%s\n", indent, indent, "TDengine server FQDN to connect. The default host is localhost."); - printf("%s%s\n", indent, "-p"); - printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server."); - printf("%s%s\n", indent, "-P"); - printf("%s%s%s\n", indent, indent, "The TCP/IP port number to use for the connection"); - printf("%s%s\n", indent, "-u"); - printf("%s%s%s\n", indent, indent, "The user name to use when connecting to the server."); - printf("%s%s\n", indent, "-A"); - printf("%s%s%s\n", indent, indent, "The user auth to use when connecting to the server."); - printf("%s%s\n", indent, "-c"); - printf("%s%s%s\n", indent, indent, "Configuration directory."); - printf("%s%s\n", indent, "-C"); - printf("%s%s%s\n", indent, indent, "Dump configuration."); - printf("%s%s\n", indent, "-s"); - printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell."); - printf("%s%s\n", indent, "-r"); - printf("%s%s%s\n", indent, indent, "Output time as unsigned long.."); - printf("%s%s\n", indent, "-f"); - printf("%s%s%s\n", indent, indent, "Script to run without enter the shell."); - printf("%s%s\n", indent, "-d"); - printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server."); - printf("%s%s\n", indent, "-t"); - printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local."); - printf("%s%s\n", indent, "-n"); - printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speed|fqdn."); - printf("%s%s\n", indent, "-l"); - printf("%s%s%s\n", indent, indent, "Packet length used for net test, default is 1000 bytes."); - printf("%s%s\n", indent, "-N"); - printf("%s%s%s\n", indent, indent, "Packet numbers used for net test, default is 100."); - printf("%s%s\n", indent, "-S"); - printf("%s%s%s\n", indent, indent, "Packet type used for net test, default is TCP."); - printf("%s%s\n", indent, "-V"); - printf("%s%s%s\n", indent, indent, "Print program version."); - - exit(EXIT_SUCCESS); -} - -char g_password[SHELL_MAX_PASSWORD_LEN]; - -void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { - for (int i = 1; i < argc; i++) { - // for host - if (strcmp(argv[i], "-h") == 0) { - if (i < argc - 1) { - arguments->host = argv[++i]; - } else { - fprintf(stderr, "option -h requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // for password - else if ((strncmp(argv[i], "-p", 2) == 0) - || (strncmp(argv[i], "--password", 10) == 0)) { - arguments->is_use_passwd = true; - strcpy(tsOsName, "Windows"); - printf(WINCLIENT_VERSION, tsOsName, taos_get_client_info()); - if ((strlen(argv[i]) == 2) - || (strncmp(argv[i], "--password", 10) == 0)) { - printf("Enter password: "); - taosSetConsoleEcho(false); - if (scanf("%s", g_password) > 1) { - fprintf(stderr, "password read error!\n"); - } - taosSetConsoleEcho(true); - getchar(); - } else { - tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); - } - arguments->password = g_password; - strcpy(argv[i], ""); - argc -= 1; - } - // for management port - else if (strcmp(argv[i], "-P") == 0) { - if (i < argc - 1) { - arguments->port = atoi(argv[++i]); - } else { - fprintf(stderr, "option -P requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // for user - else if (strcmp(argv[i], "-u") == 0) { - if (i < argc - 1) { - arguments->user = argv[++i]; - } else { - fprintf(stderr, "option -u requires an argument\n"); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-A") == 0) { - if (i < argc - 1) { - arguments->auth = argv[++i]; - } else { - fprintf(stderr, "option -A requires an argument\n"); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { - char *tmp = argv[++i]; - if (strlen(tmp) >= TSDB_FILENAME_LEN) { - fprintf(stderr, "config file path: %s overflow max len %d\n", tmp, TSDB_FILENAME_LEN - 1); - exit(EXIT_FAILURE); - } - strcpy(configDir, tmp); - } else { - fprintf(stderr, "Option -c requires an argument\n"); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-C") == 0) { - arguments->dump_config = true; - } else if (strcmp(argv[i], "-s") == 0) { - if (i < argc - 1) { - arguments->commands = argv[++i]; - } else { - fprintf(stderr, "option -s requires an argument\n"); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-r") == 0) { - arguments->is_raw_time = true; - } - // For temperory batch commands to run TODO - else if (strcmp(argv[i], "-f") == 0) { - if (i < argc - 1) { - strcpy(arguments->file, argv[++i]); - } else { - fprintf(stderr, "option -f requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // for default database - else if (strcmp(argv[i], "-d") == 0) { - if (i < argc - 1) { - arguments->database = argv[++i]; - } else { - fprintf(stderr, "option -d requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // For time zone - else if (strcmp(argv[i], "-t") == 0) { - if (i < argc - 1) { - arguments->timezone = argv[++i]; - } else { - fprintf(stderr, "option -t requires an argument\n"); - exit(EXIT_FAILURE); - } - } - else if (strcmp(argv[i], "-n") == 0) { - if (i < argc - 1) { - arguments->netTestRole = argv[++i]; - } else { - fprintf(stderr, "option -n requires an argument\n"); - exit(EXIT_FAILURE); - } - } - else if (strcmp(argv[i], "-l") == 0) { - if (i < argc - 1) { - arguments->pktLen = atoi(argv[++i]); - } else { - fprintf(stderr, "option -l requires an argument\n"); - exit(EXIT_FAILURE); - } - } - else if (strcmp(argv[i], "-N") == 0) { - if (i < argc - 1) { - arguments->pktNum = atoi(argv[++i]); - } else { - fprintf(stderr, "option -N requires an argument\n"); - exit(EXIT_FAILURE); - } - } - else if (strcmp(argv[i], "-S") == 0) { - if (i < argc - 1) { - arguments->pktType = argv[++i]; - } else { - fprintf(stderr, "option -S requires an argument\n"); - exit(EXIT_FAILURE); - } - } - else if (strcmp(argv[i], "-V") == 0) { - printVersion(); - exit(EXIT_SUCCESS); - } - // For temperory command TODO - else if (strcmp(argv[i], "--help") == 0) { - printHelp(); - exit(EXIT_SUCCESS); - } else { - fprintf(stderr, "wrong options\n"); - printHelp(); - exit(EXIT_FAILURE); - } - } -} - -void shellPrintContinuePrompt() { printf("%s", CONTINUE_PROMPT); } - -void shellPrintPrompt() { printf("%s", PROMPT_HEADER); } - -void updateBuffer(Command *cmd) { - if (regex_match(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " "); - strcat(cmd->buffer, cmd->command); - - memset(cmd->command, 0, MAX_COMMAND_SIZE); - cmd->cursorOffset = 0; -} - -int isReadyGo(Command *cmd) { - char *total = taosMemoryMalloc(MAX_COMMAND_SIZE); - memset(total, 0, MAX_COMMAND_SIZE); - sprintf(total, "%s%s", cmd->buffer, cmd->command); - - char *reg_str = - "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" - "\\s*clear\\s*$)"; - if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) { - taosMemoryFree(total); - return 1; - } - - taosMemoryFree(total); - return 0; -} - -void insertChar(Command *cmd, char c) { - // TODO: Check if the length enough. - if (cmd->cursorOffset >= MAX_COMMAND_SIZE) { - fprintf(stdout, "sql is larger than %d bytes", MAX_COMMAND_SIZE); - return; - } - - cmd->command[cmd->cursorOffset++] = c; -} - -int32_t shellReadCommand(TAOS *con, char command[]) { - Command cmd; - memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); - - // Read input. - char c; - while (1) { - c = getchar(); - - switch (c) { - case '\n': - case '\r': - if (isReadyGo(&cmd)) { - sprintf(command, "%s%s", cmd.buffer, cmd.command); - taosMemoryFree(cmd.buffer); - cmd.buffer = NULL; - taosMemoryFree(cmd.command); - cmd.command = NULL; - return 0; - } else { - shellPrintContinuePrompt(); - updateBuffer(&cmd); - } - break; - default: - insertChar(&cmd, c); - } - } - - return 0; -} - -void *shellLoopQuery(void *arg) { - TAOS *con = (TAOS *)arg; - char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); - if (command == NULL) return NULL; - - int32_t err = 0; - - do { - memset(command, 0, MAX_COMMAND_SIZE); - shellPrintPrompt(); - - // Read command from shell. - err = shellReadCommand(con, command); - if (err) { - break; - } - } while (shellRunCommand(con, command) == 0); - - return NULL; -} - -void get_history_path(char *history) { sprintf(history, "C:/TDengine/%s", HISTORY_FILE); } - -void exitShell() { exit(EXIT_SUCCESS); } diff --git a/tools/shell/src/backup/tnettest.h b/tools/shell/src/backup/tnettest.h deleted file mode 100644 index 8a03b67628ffd460a4aa95ad4de8110b71472496..0000000000000000000000000000000000000000 --- a/tools/shell/src/backup/tnettest.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TNETTEST_H -#define TDENGINE_TNETTEST_H - -#ifdef __cplusplus -extern "C" { -#endif - -void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TNETTEST_H diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c new file mode 100644 index 0000000000000000000000000000000000000000..5391d28277cf3858fc8b403c91ebad247fbbe869 --- /dev/null +++ b/tools/shell/src/shellArguments.c @@ -0,0 +1,354 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "shellInt.h" + +#define SHELL_HOST "The auth string to use when connecting to the server." +#define SHELL_PORT "The TCP/IP port number to use for the connection." +#define SHELL_USER "The user name to use when connecting to the server." +#define SHELL_PASSWORD "The password to use when connecting to the server." +#define SHELL_AUTH "The auth string to use when connecting to the server." +#define SHELL_GEN_AUTH "Generate auth string from password." +#define SHELL_CFG_DIR "Configuration directory." +#define SHELL_DMP_CFG "Dump configuration." +#define SHELL_CMD "Commands to run without enter the shell." +#define SHELL_RAW_TIME "Output time as uint64_t." +#define SHELL_FILE "Script to run without enter the shell." +#define SHELL_DB "Database to use when connecting to the server." +#define SHELL_CHECK "Check the service status." +#define SHELL_STARTUP "Check the details of the service status." +#define SHELL_WIDTH "Set the default binary display width, default is 30." +#define SHELL_NET_ROLE "Net role when network connectivity test, options: client|server." +#define SHELL_PKG_LEN "Packet length used for net test, default is 1024 bytes." +#define SHELL_PKT_NUM "Packet numbers used for net test, default is 100." +#define SHELL_VERSION "Print program version." +#define SHELL_EMAIL "" + +void shellPrintHelp() { + char indent[] = " "; + printf("Usage: taos [OPTION...] \n\n"); + printf("%s%s%s%s\n", indent, "-a,", indent, SHELL_AUTH); + printf("%s%s%s%s\n", indent, "-A,", indent, SHELL_GEN_AUTH); + printf("%s%s%s%s\n", indent, "-c,", indent, SHELL_CFG_DIR); + printf("%s%s%s%s\n", indent, "-C,", indent, SHELL_DMP_CFG); + printf("%s%s%s%s\n", indent, "-d,", indent, SHELL_DB); + printf("%s%s%s%s\n", indent, "-f,", indent, SHELL_FILE); + printf("%s%s%s%s\n", indent, "-h,", indent, SHELL_HOST); + printf("%s%s%s%s\n", indent, "-k,", indent, SHELL_CHECK); + printf("%s%s%s%s\n", indent, "-l,", indent, SHELL_PKG_LEN); + printf("%s%s%s%s\n", indent, "-n,", indent, SHELL_NET_ROLE); + printf("%s%s%s%s\n", indent, "-N,", indent, SHELL_PKT_NUM); + printf("%s%s%s%s\n", indent, "-p,", indent, SHELL_PASSWORD); + printf("%s%s%s%s\n", indent, "-P,", indent, SHELL_PORT); + printf("%s%s%s%s\n", indent, "-r,", indent, SHELL_RAW_TIME); + printf("%s%s%s%s\n", indent, "-s,", indent, SHELL_CMD); + printf("%s%s%s%s\n", indent, "-t,", indent, SHELL_STARTUP); + printf("%s%s%s%s\n", indent, "-u,", indent, SHELL_USER); + printf("%s%s%s%s\n", indent, "-w,", indent, SHELL_WIDTH); + printf("%s%s%s%s\n", indent, "-V,", indent, SHELL_VERSION); + printf("\n\nReport bugs to %s.\n", SHELL_EMAIL); +} + +static int32_t shellParseSingleOpt(int32_t key, char *arg) { + SShellArgs *pArgs = &shell.args; + + switch (key) { + case 'h': + pArgs->host = arg; + break; + case 'P': + pArgs->port = atoi(arg); + break; + case 'u': + pArgs->user = arg; + break; + case 'p': + break; + case 'a': + pArgs->auth = arg; + break; + case 'A': + pArgs->is_gen_auth = true; + break; + case 'c': + pArgs->cfgdir = arg; + break; + case 'C': + pArgs->is_dump_config = true; + break; + case 's': + pArgs->commands = arg; + break; + case 'r': + pArgs->is_raw_time = true; + break; + case 'f': + tstrncpy(pArgs->file, arg, sizeof(pArgs->file)); + break; + case 'd': + pArgs->database = arg; + break; + case 'k': + pArgs->is_check = true; + break; + case 't': + pArgs->is_startup = true; + break; + case 'w': + pArgs->displayWidth = atoi(arg); + break; + case 'n': + pArgs->netrole = arg; + break; + case 'l': + pArgs->pktLen = atoi(arg); + break; + case 'N': + pArgs->pktNum = atoi(arg); + break; + case 'V': + pArgs->is_version = true; + break; + case '?': + pArgs->is_help = true; + break; + case 1: + pArgs->abort = 1; + break; + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { + SShellArgs *pArgs = &shell.args; + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0) { + shellParseSingleOpt('?', NULL); + return 0; + } + + char *key = argv[i]; + int32_t keyLen = strlen(key); + if (keyLen != 2) { + fprintf(stderr, "invalid option %s\n", key); + return -1; + } + if (key[0] != '-') { + fprintf(stderr, "invalid option %s\n", key); + return -1; + } + + if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' || key[1] == 'a' || key[1] == 'c' || key[1] == 's' || + key[1] == 'f' || key[1] == 'd' || key[1] == 'w' || key[1] == 'n' || key[1] == 'l' || key[1] == 'N') { + if (i + 1 >= argc) { + fprintf(stderr, "option %s requires an argument\n", key); + return -1; + } + char *val = argv[i + 1]; + if (val[0] == '-') { + fprintf(stderr, "option %s requires an argument\n", key); + return -1; + } + shellParseSingleOpt(key[1], val); + i++; + } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'c' || key[1] == 'r' || key[1] == 'k' || key[1] == 't' || + key[1] == 'V') { + shellParseSingleOpt(key[1], NULL); + } else { + fprintf(stderr, "invalid option %s\n", key); + return -1; + } + } + + return 0; +} + +#ifdef LINUX +#include +#include + +const char *argp_program_version = version; +const char *argp_program_bug_address = SHELL_EMAIL; + +static struct argp_option shellOptions[] = { + {"host", 'h', "HOST", 0, SHELL_HOST}, + {"port", 'P', "PORT", 0, SHELL_PORT}, + {"user", 'u', "USER", 0, SHELL_USER}, + {0, 'p', 0, 0, SHELL_PASSWORD}, + {"auth", 'a', "AUTH", 0, SHELL_AUTH}, + {"generate-auth", 'A', 0, 0, SHELL_GEN_AUTH}, + {"config-dir", 'c', "DIR", 0, SHELL_CFG_DIR}, + {"dump-config", 'C', 0, 0, SHELL_DMP_CFG}, + {"commands", 's', "COMMANDS", 0, SHELL_CMD}, + {"raw-time", 'r', 0, 0, SHELL_RAW_TIME}, + {"file", 'f', "FILE", 0, SHELL_FILE}, + {"database", 'd', "DATABASE", 0, SHELL_DB}, + {"check", 'k', 0, 0, SHELL_CHECK}, + {"startup", 't', 0, 0, SHELL_STARTUP}, + {"display-width", 'w', "WIDTH", 0, SHELL_WIDTH}, + {"netrole", 'n', "NETROLE", 0, SHELL_NET_ROLE}, + {"pktlen", 'l', "PKTLEN", 0, SHELL_PKG_LEN}, + {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, + {0}, +}; + +static error_t shellParseOpt(int32_t key, char *arg, struct argp_state *state) { return shellParseSingleOpt(key, arg); } + +static struct argp shellArgp = {shellOptions, shellParseOpt, "", ""}; + +static void shellParseArgsUseArgp(int argc, char *argv[]) { + argp_program_version = shell.info.programVersion; + argp_parse(&shellArgp, argc, argv, 0, 0, &shell.args); +} + +#endif + +static void shellInitArgs(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strncmp(argv[i], "-p", 2) == 0) { + printf(shell.info.clientVersion, tsOsName, taos_get_client_info()); + if (strlen(argv[i]) == 2) { + printf("Enter password: "); + taosSetConsoleEcho(false); + if (scanf("%20s", shell.args.password) > 1) { + fprintf(stderr, "password reading error\n"); + } + taosSetConsoleEcho(true); + if (EOF == getchar()) { + fprintf(stderr, "getchar() return EOF\n"); + } + } else { + tstrncpy(shell.args.password, (char *)(argv[i] + 2), sizeof(shell.args.password)); + strcpy(argv[i], "-p"); + } + } + } + if (strlen(shell.args.password) == 0) { + tstrncpy(shell.args.password, TSDB_DEFAULT_PASS, sizeof(shell.args.password)); + } + + SShellArgs *pArgs = &shell.args; + pArgs->user = TSDB_DEFAULT_USER; + pArgs->pktLen = SHELL_DEF_PKG_LEN; + pArgs->pktNum = SHELL_DEF_PKG_NUM; + pArgs->displayWidth = SHELL_DEFAULT_MAX_BINARY_DISPLAY_WIDTH; +} + +static int32_t shellCheckArgs() { + SShellArgs *pArgs = &shell.args; + if (pArgs->host != NULL && (strlen(pArgs->host) <= 0 || strlen(pArgs->host) > TSDB_FQDN_LEN)) { + printf("Invalid host:%s\n", pArgs->host); + return -1; + } + + if (pArgs->user != NULL && (strlen(pArgs->user) <= 0 || strlen(pArgs->user) > TSDB_USER_LEN)) { + printf("Invalid user:%s\n", pArgs->user); + return -1; + } + + if (pArgs->auth != NULL && (strlen(pArgs->auth) <= 0 || strlen(pArgs->auth) > TSDB_PASSWORD_LEN)) { + printf("Invalid auth:%s\n", pArgs->auth); + return -1; + } + + if (pArgs->database != NULL && (strlen(pArgs->database) <= 0 || strlen(pArgs->database) > TSDB_DB_NAME_LEN)) { + printf("Invalid database:%s\n", pArgs->database); + return -1; + } + + if (pArgs->file[0] != 0) { + char fullname[PATH_MAX] = {0}; + if (taosExpandDir(pArgs->file, fullname, PATH_MAX) == 0) { + tstrncpy(pArgs->file, fullname, PATH_MAX); + } + } + + if (pArgs->cfgdir != NULL) { + if (strlen(pArgs->cfgdir) <= 0 || strlen(pArgs->cfgdir) >= PATH_MAX) { + printf("Invalid cfgdir:%s\n", pArgs->cfgdir); + return -1; + } else { + if (taosExpandDir(pArgs->cfgdir, configDir, PATH_MAX) != 0) { + tstrncpy(configDir, pArgs->cfgdir, PATH_MAX); + } + } + } + + if (pArgs->commands != NULL && (strlen(pArgs->commands) <= 0)) { + printf("Invalid commands:%s\n", pArgs->commands); + return -1; + } + + if (pArgs->netrole != NULL && !(strcmp(pArgs->netrole, "client") == 0 || strcmp(pArgs->netrole, "server") == 0)) { + printf("Invalid netrole:%s\n", pArgs->netrole); + return -1; + } + + if (pArgs->password != NULL && (strlen(pArgs->password) <= 0)) { + printf("Invalid password\n"); + return -1; + } + + if (pArgs->pktLen < SHELL_MIN_PKG_LEN || pArgs->pktLen > SHELL_MAX_PKG_LEN) { + printf("Invalid pktLen:%d, range:[%d, %d]\n", pArgs->pktLen, SHELL_MIN_PKG_LEN, SHELL_MAX_PKG_LEN); + return -1; + } + + if (pArgs->pktNum < SHELL_MIN_PKG_NUM || pArgs->pktNum > SHELL_MAX_PKG_NUM) { + printf("Invalid pktNum:%d, range:[%d, %d]\n", pArgs->pktNum, SHELL_MIN_PKG_NUM, SHELL_MAX_PKG_NUM); + return -1; + } + + if (pArgs->displayWidth <= 0 || pArgs->displayWidth > 10 * 1024) { + printf("Invalid displayWidth:%d, range:[1, 10 * 1024]\n", pArgs->displayWidth); + return -1; + } + + return 0; +} + +int32_t shellParseArgs(int32_t argc, char *argv[]) { + shellInitArgs(argc, argv); + shell.info.clientVersion = + "Welcome to the TDengine shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; + shell.info.promptHeader = "taos> "; + shell.info.promptContinue = " -> "; + shell.info.promptSize = 6; + snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "version: %s", version); + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + shell.info.osname = "Windows"; + snprintf(shell.history.file, TSDB_FILENAME_LEN, "C:/TDengine/%s", SHELL_HISTORY_FILE); + if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1; +#elif defined(_TD_DARWIN_64) + shell.info.osname = "Darwin"; + snprintf(shell.history.file, TSDB_FILENAME_LEN, "%s/%s", getpwuid(getuid())->pw_dir, SHELL_HISTORY_FILE); + if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1; +#else + shell.info.osname = "Linux"; + snprintf(shell.history.file, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), SHELL_HISTORY_FILE); + shellParseArgsUseArgp(argc, argv); + // if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1; + if (shell.args.abort) { + return -1; + } +#endif + + return shellCheckArgs(); +} diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 546b19f83cec33df8a0649b283046f70b0803f5e..c34ee7a22c6da4caf03ceae4b9a24744074eb4a0 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -14,22 +14,122 @@ */ #define __USE_XOPEN +#include "shellInt.h" -#include "shellCommand.h" -#include "os.h" -#include "shell.h" - -#include +#define LEFT 1 +#define RIGHT 2 +#define UP 3 +#define DOWN 4 +#define PSIZE shell.info.promptSize typedef struct { - char widthInString; - char widthOnScreen; -} UTFCodeInfo; + char *buffer; + char *command; + uint32_t commandSize; + uint32_t bufferSize; + uint32_t cursorOffset; + uint32_t screenOffset; + uint32_t endOffset; +} SShellCmd; + +static int32_t shellCountPrefixOnes(uint8_t c); +static void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width); +static void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width); +static void shellInsertChar(SShellCmd *cmd, char *c, int size); +static void shellBackspaceChar(SShellCmd *cmd); +static void shellClearLineBefore(SShellCmd *cmd); +static void shellClearLineAfter(SShellCmd *cmd); +static void shellDeleteChar(SShellCmd *cmd); +static void shellMoveCursorLeft(SShellCmd *cmd); +static void shellMoveCursorRight(SShellCmd *cmd); +static void shellPositionCursorHome(SShellCmd *cmd); +static void shellPositionCursorEnd(SShellCmd *cmd); +static void shellPrintChar(char c, int32_t times); +static void shellPositionCursor(int32_t step, int32_t direction); +static void shellUpdateBuffer(SShellCmd *cmd); +static int32_t shellIsReadyGo(SShellCmd *cmd); +static void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width); +static void shellResetCommand(SShellCmd *cmd, const char s[]); +static void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos); +static void shellShowOnScreen(SShellCmd *cmd); + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +static void shellPrintContinuePrompt() { printf("%s", shell.args.promptContinue); } +static void shellPrintPrompt() { printf("%s", shell.args.promptHeader); } + +void shellUpdateBuffer(SShellCmd *cmd) { + if (shellRegexMatch(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " "); + strcat(cmd->buffer, cmd->command); + + memset(cmd->command, 0, SHELL_MAX_COMMAND_SIZE); + cmd->cursorOffset = 0; +} + +int shellIsReadyGo(SShellCmd *cmd) { + char *total = taosMemoryMalloc(SHELL_MAX_COMMAND_SIZE); + memset(total, 0, SHELL_MAX_COMMAND_SIZE); + sprintf(total, "%s%s", cmd->buffer, cmd->command); -int countPrefixOnes(unsigned char c) { - unsigned char mask = 127; + char *reg_str = + "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" + "\\s*clear\\s*$)"; + if (shellRegexMatch(total, reg_str, REG_EXTENDED | REG_ICASE)) { + taosMemoryFree(total); + return 1; + } + + taosMemoryFree(total); + return 0; +} + +void shellInsertChar(SShellCmd *cmd, char c) { + if (cmd->cursorOffset >= SHELL_MAX_COMMAND_SIZE) { + fprintf(stdout, "sql is larger than %d bytes", SHELL_MAX_COMMAND_SIZE); + return; + } + cmd->command[cmd->cursorOffset++] = c; +} + +int32_t shellReadCommand(char command[]) { + SShellCmd cmd; + memset(&cmd, 0, sizeof(cmd)); + cmd.buffer = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); + cmd.command = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); + + // Read input. + char c; + while (1) { + c = getchar(); + + switch (c) { + case '\n': + case '\r': + if (shellIsReadyGo(&cmd)) { + sprintf(command, "%s%s", cmd.buffer, cmd.command); + taosMemoryFree(cmd.buffer); + cmd.buffer = NULL; + taosMemoryFree(cmd.command); + cmd.command = NULL; + return 0; + } else { + shellPrintContinuePrompt(); + shellUpdateBuffer(&cmd); + } + break; + default: + shellInsertChar(&cmd, c); + } + } + + return 0; +} + +#else + +int32_t shellCountPrefixOnes(uint8_t c) { + uint8_t mask = 127; mask = ~mask; - int ret = 0; + int32_t ret = 0; while ((c & mask) != 0) { ret++; c <<= 1; @@ -38,7 +138,7 @@ int countPrefixOnes(unsigned char c) { return ret; } -void getPrevCharSize(const char *str, int pos, int *size, int *width) { +void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) { assert(pos > 0); TdWchar wc; @@ -48,16 +148,16 @@ void getPrevCharSize(const char *str, int pos, int *size, int *width) { while (--pos >= 0) { *size += 1; - if (str[pos] > 0 || countPrefixOnes((unsigned char)str[pos]) > 1) break; + if (str[pos] > 0 || shellCountPrefixOnes((uint8_t)str[pos]) > 1) break; } - int rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); + int32_t rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); assert(rc == *size); *width = taosWcharWidth(wc); } -void getNextCharSize(const char *str, int pos, int *size, int *width) { +void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) { assert(pos >= 0); TdWchar wc; @@ -65,13 +165,13 @@ void getNextCharSize(const char *str, int pos, int *size, int *width) { *width = taosWcharWidth(wc); } -void insertChar(Command *cmd, char *c, int size) { +void shellInsertChar(SShellCmd *cmd, char *c, int32_t size) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); TdWchar wc; if (taosMbToWchar(&wc, c, size) < 0) return; - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); /* update the buffer */ memmove(cmd->command + cmd->cursorOffset + size, cmd->command + cmd->cursorOffset, cmd->commandSize - cmd->cursorOffset); @@ -81,122 +181,122 @@ void insertChar(Command *cmd, char *c, int size) { cmd->cursorOffset += size; cmd->screenOffset += taosWcharWidth(wc); cmd->endOffset += taosWcharWidth(wc); - showOnScreen(cmd); + shellShowOnScreen(cmd); } -void backspaceChar(Command *cmd) { +void shellBackspaceChar(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if (cmd->cursorOffset > 0) { - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); - int size = 0; - int width = 0; - getPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); + int32_t size = 0; + int32_t width = 0; + shellGetPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width); memmove(cmd->command + cmd->cursorOffset - size, cmd->command + cmd->cursorOffset, cmd->commandSize - cmd->cursorOffset); cmd->commandSize -= size; cmd->cursorOffset -= size; cmd->screenOffset -= width; cmd->endOffset -= width; - showOnScreen(cmd); + shellShowOnScreen(cmd); } } -void clearLineBefore(Command *cmd) { +void shellClearLineBefore(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); memmove(cmd->command, cmd->command + cmd->cursorOffset, cmd->commandSize - cmd->cursorOffset); cmd->commandSize -= cmd->cursorOffset; cmd->cursorOffset = 0; cmd->screenOffset = 0; cmd->endOffset = cmd->commandSize; - showOnScreen(cmd); + shellShowOnScreen(cmd); } -void clearLineAfter(Command *cmd) { +void shellClearLineAfter(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); cmd->commandSize -= cmd->endOffset - cmd->cursorOffset; cmd->endOffset = cmd->cursorOffset; - showOnScreen(cmd); + shellShowOnScreen(cmd); } -void deleteChar(Command *cmd) { +void shellDeleteChar(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if (cmd->cursorOffset < cmd->commandSize) { - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); - int size = 0; - int width = 0; - getNextCharSize(cmd->command, cmd->cursorOffset, &size, &width); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); + int32_t size = 0; + int32_t width = 0; + shellGetNextCharSize(cmd->command, cmd->cursorOffset, &size, &width); memmove(cmd->command + cmd->cursorOffset, cmd->command + cmd->cursorOffset + size, cmd->commandSize - cmd->cursorOffset - size); cmd->commandSize -= size; cmd->endOffset -= width; - showOnScreen(cmd); + shellShowOnScreen(cmd); } } -void moveCursorLeft(Command *cmd) { +void shellMoveCursorLeft(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if (cmd->cursorOffset > 0) { - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); - int size = 0; - int width = 0; - getPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); + int32_t size = 0; + int32_t width = 0; + shellGetPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width); cmd->cursorOffset -= size; cmd->screenOffset -= width; - showOnScreen(cmd); + shellShowOnScreen(cmd); } } -void moveCursorRight(Command *cmd) { +void shellMoveCursorRight(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if (cmd->cursorOffset < cmd->commandSize) { - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); - int size = 0; - int width = 0; - getNextCharSize(cmd->command, cmd->cursorOffset, &size, &width); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); + int32_t size = 0; + int32_t width = 0; + shellGetNextCharSize(cmd->command, cmd->cursorOffset, &size, &width); cmd->cursorOffset += size; cmd->screenOffset += width; - showOnScreen(cmd); + shellShowOnScreen(cmd); } } -void positionCursorHome(Command *cmd) { +void shellPositionCursorHome(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if (cmd->cursorOffset > 0) { - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); cmd->cursorOffset = 0; cmd->screenOffset = 0; - showOnScreen(cmd); + shellShowOnScreen(cmd); } } -void positionCursorEnd(Command *cmd) { +void shellPositionCursorEnd(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if (cmd->cursorOffset < cmd->commandSize) { - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); cmd->cursorOffset = cmd->commandSize; cmd->screenOffset = cmd->endOffset; - showOnScreen(cmd); + shellShowOnScreen(cmd); } } -void printChar(char c, int times) { - for (int i = 0; i < times; i++) { +void shellPrintChar(char c, int32_t times) { + for (int32_t i = 0; i < times; i++) { fprintf(stdout, "%c", c); } fflush(stdout); } -void positionCursor(int step, int direction) { +void shellPositionCursor(int32_t step, int32_t direction) { if (step > 0) { if (direction == LEFT) { fprintf(stdout, "\033[%dD", step); @@ -211,32 +311,32 @@ void positionCursor(int step, int direction) { } } -void updateBuffer(Command *cmd) { +void shellUpdateBuffer(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - if (regex_match(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " "); + if (shellRegexMatch(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " "); strcat(cmd->buffer, cmd->command); cmd->bufferSize += cmd->commandSize; - memset(cmd->command, 0, MAX_COMMAND_SIZE); + memset(cmd->command, 0, SHELL_MAX_COMMAND_SIZE); cmd->cursorOffset = 0; cmd->screenOffset = 0; cmd->commandSize = 0; cmd->endOffset = 0; - showOnScreen(cmd); + shellShowOnScreen(cmd); } -int isReadyGo(Command *cmd) { +int32_t shellIsReadyGo(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - char *total = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); - memset(cmd->command + cmd->commandSize, 0, MAX_COMMAND_SIZE - cmd->commandSize); + char *total = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); + memset(cmd->command + cmd->commandSize, 0, SHELL_MAX_COMMAND_SIZE - cmd->commandSize); sprintf(total, "%s%s", cmd->buffer, cmd->command); char *reg_str = "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" "\\s*clear\\s*$)"; - if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) { + if (shellRegexMatch(total, reg_str, REG_EXTENDED | REG_ICASE)) { taosMemoryFree(total); return 1; } @@ -245,28 +345,268 @@ int isReadyGo(Command *cmd) { return 0; } -void getMbSizeInfo(const char *str, int *size, int *width) { - TdWchar *wc = (TdWchar *)taosMemoryCalloc(sizeof(TdWchar), MAX_COMMAND_SIZE); +void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width) { + TdWchar *wc = (TdWchar *)taosMemoryCalloc(sizeof(TdWchar), SHELL_MAX_COMMAND_SIZE); *size = strlen(str); - taosMbsToWchars(wc, str, MAX_COMMAND_SIZE); - *width = taosWcharsWidth(wc, MAX_COMMAND_SIZE); + taosMbsToWchars(wc, str, SHELL_MAX_COMMAND_SIZE); + *width = taosWcharsWidth(wc, SHELL_MAX_COMMAND_SIZE); taosMemoryFree(wc); } -void resetCommand(Command *cmd, const char s[]) { +void shellResetCommand(SShellCmd *cmd, const char s[]) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); - memset(cmd->buffer, 0, MAX_COMMAND_SIZE); - memset(cmd->command, 0, MAX_COMMAND_SIZE); - strncpy(cmd->command, s, MAX_COMMAND_SIZE); - int size = 0; - int width = 0; - getMbSizeInfo(s, &size, &width); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); + memset(cmd->buffer, 0, SHELL_MAX_COMMAND_SIZE); + memset(cmd->command, 0, SHELL_MAX_COMMAND_SIZE); + strncpy(cmd->command, s, SHELL_MAX_COMMAND_SIZE); + int32_t size = 0; + int32_t width = 0; + shellGetMbSizeInfo(s, &size, &width); cmd->bufferSize = 0; cmd->commandSize = size; cmd->cursorOffset = size; cmd->screenOffset = width; cmd->endOffset = width; - showOnScreen(cmd); + shellShowOnScreen(cmd); +} + +void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos) { + struct winsize w; + if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { + // fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); + w.ws_col = 120; + w.ws_row = 30; + } + + int32_t cursor_x = cursor_pos / w.ws_col; + int32_t cursor_y = cursor_pos % w.ws_col; + int32_t command_x = ecmd_pos / w.ws_col; + shellPositionCursor(cursor_y, LEFT); + shellPositionCursor(command_x - cursor_x, DOWN); + fprintf(stdout, "\033[2K"); + for (int32_t i = 0; i < command_x; i++) { + shellPositionCursor(1, UP); + fprintf(stdout, "\033[2K"); + } + fflush(stdout); } + +void shellShowOnScreen(SShellCmd *cmd) { + struct winsize w; + if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { + fprintf(stderr, "No stream device\n"); + w.ws_col = 120; + w.ws_row = 30; + } + + TdWchar wc; + int32_t size = 0; + + // Print out the command. + char *total_string = taosMemoryMalloc(SHELL_MAX_COMMAND_SIZE); + memset(total_string, '\0', SHELL_MAX_COMMAND_SIZE); + if (strcmp(cmd->buffer, "") == 0) { + sprintf(total_string, "%s%s", shell.info.promptHeader, cmd->command); + } else { + sprintf(total_string, "%s%s", shell.info.promptContinue, cmd->command); + } + + int32_t remain_column = w.ws_col; + for (char *str = total_string; size < cmd->commandSize + PSIZE;) { + int32_t ret = taosMbToWchar(&wc, str, MB_CUR_MAX); + if (ret < 0) break; + size += ret; + /* assert(size >= 0); */ + int32_t width = taosWcharWidth(wc); + if (remain_column > width) { + printf("%lc", wc); + remain_column -= width; + } else { + if (remain_column == width) { + printf("%lc\n\r", wc); + remain_column = w.ws_col; + } else { + printf("\n\r%lc", wc); + remain_column = w.ws_col - width; + } + } + + str = total_string + size; + } + + taosMemoryFree(total_string); + + // Position the cursor + int32_t cursor_pos = cmd->screenOffset + PSIZE; + int32_t ecmd_pos = cmd->endOffset + PSIZE; + + int32_t cursor_x = cursor_pos / w.ws_col; + int32_t cursor_y = cursor_pos % w.ws_col; + // int32_t cursor_y = cursor % w.ws_col; + int32_t command_x = ecmd_pos / w.ws_col; + int32_t command_y = ecmd_pos % w.ws_col; + // int32_t command_y = (command.size() + PSIZE) % w.ws_col; + shellPositionCursor(command_y, LEFT); + shellPositionCursor(command_x, UP); + shellPositionCursor(cursor_x, DOWN); + shellPositionCursor(cursor_y, RIGHT); + fflush(stdout); +} + +int32_t shellReadCommand(char *command) { + SShellHistory *pHistory = &shell.history; + SShellCmd cmd = {0}; + uint32_t hist_counter = pHistory->hend; + char utf8_array[10] = "\0"; + + cmd.buffer = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); + cmd.command = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); + shellShowOnScreen(&cmd); + + // Read input. + char c; + while (1) { + c = (char)getchar(); // getchar() return an 'int32_t' value + + if (c == EOF) { + return c; + } + + if (c < 0) { // For UTF-8 + int32_t count = shellCountPrefixOnes(c); + utf8_array[0] = c; + for (int32_t k = 1; k < count; k++) { + c = (char)getchar(); + utf8_array[k] = c; + } + shellInsertChar(&cmd, utf8_array, count); + } else if (c < '\033') { + // Ctrl keys. TODO: Implement ctrl combinations + switch (c) { + case 1: // ctrl A + shellPositionCursorHome(&cmd); + break; + case 3: + printf("\n"); + shellResetCommand(&cmd, ""); + kill(0, SIGINT); + break; + case 4: // EOF or Ctrl+D + printf("\n"); + return -1; + case 5: // ctrl E + shellPositionCursorEnd(&cmd); + break; + case 8: + shellBackspaceChar(&cmd); + break; + case '\n': + case '\r': + printf("\n"); + if (shellIsReadyGo(&cmd)) { + sprintf(command, "%s%s", cmd.buffer, cmd.command); + taosMemoryFreeClear(cmd.buffer); + taosMemoryFreeClear(cmd.command); + return 0; + } else { + shellUpdateBuffer(&cmd); + } + break; + case 11: // Ctrl + K; + shellClearLineAfter(&cmd); + break; + case 12: // Ctrl + L; + system("clear"); + shellShowOnScreen(&cmd); + break; + case 21: // Ctrl + U; + shellClearLineBefore(&cmd); + break; + } + } else if (c == '\033') { + c = (char)getchar(); + switch (c) { + case '[': + c = (char)getchar(); + switch (c) { + case 'A': // Up arrow + if (hist_counter != pHistory->hstart) { + hist_counter = (hist_counter + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; + shellResetCommand(&cmd, (pHistory->hist[hist_counter] == NULL) ? "" : pHistory->hist[hist_counter]); + } + break; + case 'B': // Down arrow + if (hist_counter != pHistory->hend) { + int32_t next_hist = (hist_counter + 1) % SHELL_MAX_HISTORY_SIZE; + + if (next_hist != pHistory->hend) { + shellResetCommand(&cmd, (pHistory->hist[next_hist] == NULL) ? "" : pHistory->hist[next_hist]); + } else { + shellResetCommand(&cmd, ""); + } + hist_counter = next_hist; + } + break; + case 'C': // Right arrow + shellMoveCursorRight(&cmd); + break; + case 'D': // Left arrow + shellMoveCursorLeft(&cmd); + break; + case '1': + if ((c = (char)getchar()) == '~') { + // Home key + shellPositionCursorHome(&cmd); + } + break; + case '2': + if ((c = (char)getchar()) == '~') { + // Insert key + } + break; + case '3': + if ((c = (char)getchar()) == '~') { + // Delete key + shellDeleteChar(&cmd); + } + break; + case '4': + if ((c = (char)getchar()) == '~') { + // End key + shellPositionCursorEnd(&cmd); + } + break; + case '5': + if ((c = (char)getchar()) == '~') { + // Page up key + } + break; + case '6': + if ((c = (char)getchar()) == '~') { + // Page down key + } + break; + case 72: + // Home key + shellPositionCursorHome(&cmd); + break; + case 70: + // End key + shellPositionCursorEnd(&cmd); + break; + } + break; + } + } else if (c == 0x7f) { + // press delete key + shellBackspaceChar(&cmd); + } else { + shellInsertChar(&cmd, &c, 1); + } + } + + return 0; +} + +#endif \ No newline at end of file diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 12206ce29067226e0ce15fd0c9c61e26e86e489d..ac2be3c3cf89d45b032a3b26f3bafe7c218fce29 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -17,134 +17,35 @@ #define _GNU_SOURCE #define _XOPEN_SOURCE #define _DEFAULT_SOURCE - -#include "os.h" -#include "shell.h" -#include "shellCommand.h" -#include "taosdef.h" -#include "taoserror.h" -#include "tconfig.h" -#include "tglobal.h" -#include "ttypes.h" -#include "tutil.h" - -#include - -/**************** Global variables ****************/ -#ifdef _TD_POWER_ -char CLIENT_VERSION[] = - "Welcome to the PowerDB shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by PowerDB, Inc. All rights reserved.\n\n"; -char PROMPT_HEADER[] = "power> "; - -char CONTINUE_PROMPT[] = " -> "; -int prompt_size = 7; -#elif (_TD_TQ_ == true) -char CLIENT_VERSION[] = - "Welcome to the TQ shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TQ, Inc. All rights reserved.\n\n"; -char PROMPT_HEADER[] = "tq> "; - -char CONTINUE_PROMPT[] = " -> "; -int prompt_size = 4; -#else -char CLIENT_VERSION[] = - "Welcome to the TDengine shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; -char PROMPT_HEADER[] = "taos> "; - -char CONTINUE_PROMPT[] = " -> "; -int prompt_size = 6; -#endif - -int64_t result = 0; -SShellHistory history; - -#define DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30 -extern int32_t tsMaxBinaryDisplayWidth; -extern TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); - -/* - * FUNCTION: Initialize the shell. - */ -TAOS *shellInit(SShellArguments *_args) { - printf("\n"); - if (!_args->is_use_passwd) { -#ifdef TD_WINDOWS - strcpy(tsOsName, "Windows"); -#elif defined(TD_DARWIN) - strcpy(tsOsName, "Darwin"); -#endif - printf(CLIENT_VERSION, tsOsName, taos_get_client_info()); - } - - fflush(stdout); - - // set options before initializing - if (_args->timezone != NULL) { - taos_options(TSDB_OPTION_TIMEZONE, _args->timezone); - } - - if (!_args->is_use_passwd) { - _args->password = TSDB_DEFAULT_PASS; - } - - if (_args->user == NULL) { - _args->user = TSDB_DEFAULT_USER; - } - - // Connect to the database. - TAOS *con = NULL; - if (_args->auth == NULL) { - con = taos_connect(_args->host, _args->user, _args->password, _args->database, _args->port); - } else { - con = taos_connect_auth(_args->host, _args->user, _args->auth, _args->database, _args->port); - } - - if (con == NULL) { - fflush(stdout); - return con; - } - - /* Read history TODO : release resources here*/ - read_history(); - - // Check if it is temperory run - if (_args->commands != NULL || _args->file[0] != 0) { - if (_args->commands != NULL) { - printf("%s%s\n", PROMPT_HEADER, _args->commands); - shellRunCommand(con, _args->commands); - } - - if (_args->file[0] != 0) { - source_file(con, _args->file); - } - - taos_close(con); - write_history(); - exit(EXIT_SUCCESS); - } - -#if 0 -#ifndef WINDOWS - if (_args->dir[0] != 0) { - source_dir(con, _args); - taos_close(con); - exit(EXIT_SUCCESS); - } - - if (_args->check != 0) { - shellCheck(con, _args); - taos_close(con); - exit(EXIT_SUCCESS); - } -#endif -#endif - - return con; -} - -static bool isEmptyCommand(const char *cmd) { +#include "shellInt.h" + +static bool shellIsEmptyCommand(const char *cmd); +static int32_t shellRunSingleCommand(char *command); +static int32_t shellRunCommand(char *command); +static void shellRunSingleCommandImp(char *command); +static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision); +static void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, + int32_t precision); +static int32_t shellDumpResultToFile(const char *fname, TAOS_RES *tres); +static void shellPrintNChar(const char *str, int32_t length, int32_t width); +static void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t length, int32_t precision); +static int32_t shellVerticalPrintResult(TAOS_RES *tres); +static int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision); +static void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields); +static int32_t shellHorizontalPrintResult(TAOS_RES *tres); +static int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical); +static void shellReadHistory(); +static void shellWriteHistory(); +static void shellPrintError(TAOS_RES *tres, int64_t st); +static bool shellIsCommentLine(char *line); +static void shellSourceFile(const char *file); +static void shellGetGrantInfo(); +static void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context); +static void shellCleanup(void *arg); +static void *shellCancelHandler(void *arg); +static void *shellThreadLoop(void *arg); + +bool shellIsEmptyCommand(const char *cmd) { for (char c = *cmd++; c != 0; c = *cmd++) { if (c != ' ' && c != '\t' && c != ';') { return false; @@ -153,73 +54,67 @@ static bool isEmptyCommand(const char *cmd) { return true; } -static int32_t shellRunSingleCommand(TAOS *con, char *command) { - /* If command is empty just return */ - if (isEmptyCommand(command)) { +int32_t shellRunSingleCommand(char *command) { + if (shellIsEmptyCommand(command)) { return 0; } - // Analyse the command. - if (regex_match(command, "^[ \t]*(quit|q|exit)[ \t;]*$", REG_EXTENDED | REG_ICASE)) { - taos_close(con); - write_history(); -#ifdef WINDOWS - exit(EXIT_SUCCESS); -#endif + if (shellRegexMatch(command, "^[ \t]*(quit|q|exit)[ \t;]*$", REG_EXTENDED | REG_ICASE)) { + shellWriteHistory(); return -1; } - if (regex_match(command, "^[\t ]*clear[ \t;]*$", REG_EXTENDED | REG_ICASE)) { - // If clear the screen. + if (shellRegexMatch(command, "^[\t ]*clear[ \t;]*$", REG_EXTENDED | REG_ICASE)) { system("clear"); return 0; } - if (regex_match(command, "^[\t ]*set[ \t]+max_binary_display_width[ \t]+(default|[1-9][0-9]*)[ \t;]*$", - REG_EXTENDED | REG_ICASE)) { + if (shellRegexMatch(command, "^[\t ]*set[ \t]+max_binary_display_width[ \t]+(default|[1-9][0-9]*)[ \t;]*$", + REG_EXTENDED | REG_ICASE)) { strtok(command, " \t"); strtok(NULL, " \t"); char *p = strtok(NULL, " \t"); - if (strcasecmp(p, "default") == 0) { - tsMaxBinaryDisplayWidth = DEFAULT_MAX_BINARY_DISPLAY_WIDTH; + if (strncasecmp(p, "default", 7) == 0) { + shell.args.displayWidth = SHELL_DEFAULT_MAX_BINARY_DISPLAY_WIDTH; } else { - tsMaxBinaryDisplayWidth = atoi(p); + int32_t displayWidth = atoi(p); + displayWidth = TRANGE(displayWidth, 1, 10 * 1024); + shell.args.displayWidth = displayWidth; } return 0; } - if (regex_match(command, "^[ \t]*source[\t ]+[^ ]+[ \t;]*$", REG_EXTENDED | REG_ICASE)) { + if (shellRegexMatch(command, "^[ \t]*source[\t ]+[^ ]+[ \t;]*$", REG_EXTENDED | REG_ICASE)) { /* If source file. */ char *c_ptr = strtok(command, " ;"); assert(c_ptr != NULL); c_ptr = strtok(NULL, " ;"); assert(c_ptr != NULL); - source_file(con, c_ptr); + shellSourceFile(c_ptr); return 0; } - shellRunCommandOnServer(con, command); + shellRunSingleCommandImp(command); return 0; } -int32_t shellRunCommand(TAOS *con, char *command) { - /* If command is empty just return */ - if (isEmptyCommand(command)) { +int32_t shellRunCommand(char *command) { + if (shellIsEmptyCommand(command)) { return 0; } - /* Update the history vector. */ - if (history.hstart == history.hend || - history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE] == NULL || - strcmp(command, history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE]) != 0) { - if (history.hist[history.hend] != NULL) { - taosMemoryFreeClear(history.hist[history.hend]); + SShellHistory *pHistory = &shell.history; + if (pHistory->hstart == pHistory->hend || + pHistory->hist[(pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE] == NULL || + strcmp(command, pHistory->hist[(pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE]) != 0) { + if (pHistory->hist[pHistory->hend] != NULL) { + taosMemoryFreeClear(pHistory->hist[pHistory->hend]); } - history.hist[history.hend] = strdup(command); + pHistory->hist[pHistory->hend] = strdup(command); - history.hend = (history.hend + 1) % MAX_HISTORY_SIZE; - if (history.hend == history.hstart) { - history.hstart = (history.hstart + 1) % MAX_HISTORY_SIZE; + pHistory->hend = (pHistory->hend + 1) % SHELL_MAX_HISTORY_SIZE; + if (pHistory->hend == pHistory->hstart) { + pHistory->hstart = (pHistory->hstart + 1) % SHELL_MAX_HISTORY_SIZE; } } @@ -271,7 +166,7 @@ int32_t shellRunCommand(TAOS *con, char *command) { if (c == ';' && quote == 0) { c = *p; *p = 0; - if (shellRunSingleCommand(con, cmd) < 0) { + if (shellRunSingleCommand(cmd) < 0) { return -1; } *p = c; @@ -280,26 +175,15 @@ int32_t shellRunCommand(TAOS *con, char *command) { } *p = 0; - return shellRunSingleCommand(con, cmd); -} - -void freeResultWithRid(int64_t rid) { -#if 0 - SSqlObj* pSql = taosAcquireRef(tscObjRef, rid); - if(pSql){ - taos_free_result(pSql); - taosReleaseRef(tscObjRef, rid); - } -#endif + return shellRunSingleCommand(cmd); } -void shellRunCommandOnServer(TAOS *con, char command[]) { - int64_t st, et; - wordexp_t full_path; - char *sptr = NULL; - char *cptr = NULL; - char *fname = NULL; - bool printMode = false; +void shellRunSingleCommandImp(char *command) { + int64_t st, et; + char *sptr = NULL; + char *cptr = NULL; + char *fname = NULL; + bool printMode = false; if ((sptr = strstr(command, ">>")) != NULL) { cptr = strstr(command, ";"); @@ -307,12 +191,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { *cptr = '\0'; } - if (wordexp(sptr + 2, &full_path, 0) != 0) { - fprintf(stderr, "ERROR: invalid filename: %s\n", sptr + 2); - return; - } + fname = sptr + 2; *sptr = '\0'; - fname = full_path.we_wordv[0]; } if ((sptr = strstr(command, "\\G")) != NULL) { @@ -327,20 +207,19 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { st = taosGetTimestampUs(); - TAOS_RES *pSql = taos_query(con, command); + TAOS_RES *pSql = taos_query(shell.conn, command); if (taos_errno(pSql)) { - taos_error(pSql, st); + shellPrintError(pSql, st); return; } - int64_t oresult = atomic_load_64(&result); + int64_t oresult = atomic_load_64(&shell.result); - if (regex_match(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { + if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { fprintf(stdout, "Database changed.\n\n"); fflush(stdout); - atomic_store_64(&result, 0); - freeResultWithRid(oresult); + atomic_store_64(&shell.result, 0); taos_free_result(pSql); return; @@ -348,12 +227,11 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { TAOS_FIELD *pFields = taos_fetch_fields(pSql); if (pFields != NULL) { // select and show kinds of commands - int error_no = 0; + int32_t error_no = 0; - int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); + int32_t numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); if (numOfRows < 0) { - atomic_store_64(&result, 0); - freeResultWithRid(oresult); + atomic_store_64(&shell.result, 0); return; } @@ -365,7 +243,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { } taos_free_result(pSql); } else { - int num_rows_affacted = taos_affected_rows(pSql); + int32_t num_rows_affacted = taos_affected_rows(pSql); taos_free_result(pSql); et = taosGetTimestampUs(); printf("Query OK, %d of %d row(s) in database (%.6fs)\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); @@ -373,45 +251,11 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { printf("\n"); - if (fname != NULL) { - wordfree(&full_path); - } - - atomic_store_64(&result, 0); - freeResultWithRid(oresult); + atomic_store_64(&shell.result, 0); } -/* Function to do regular expression check */ -int regex_match(const char *s, const char *reg, int cflags) { - regex_t regex; - char msgbuf[100] = {0}; - - /* Compile regular expression */ - if (regcomp(®ex, reg, cflags) != 0) { - fprintf(stderr, "Fail to compile regex"); - exitShell(); - } - - /* Execute regular expression */ - int reti = regexec(®ex, s, 0, NULL, 0); - if (!reti) { - regfree(®ex); - return 1; - } else if (reti == REG_NOMATCH) { - regfree(®ex); - return 0; - } else { - regerror(reti, ®ex, msgbuf, sizeof(msgbuf)); - fprintf(stderr, "Regex match failed: %s\n", msgbuf); - regfree(®ex); - exitShell(); - } - - return 0; -} - -static char *formatTimestamp(char *buf, int64_t val, int precision) { - if (args.is_raw_time) { +char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { + if (shell.args.is_raw_time) { sprintf(buf, "%" PRId64, val); return buf; } @@ -429,13 +273,14 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) { ms = val % 1000; } - /* comment out as it make testcases like select_with_tags.sim fail. + /* + comment out as it make testcases like select_with_tags.sim fail. but in windows, this may cause the call to localtime crash if tt < 0, need to find a better solution. if (tt < 0) { tt = 0; } - */ + */ #ifdef WINDOWS if (tt < 0) tt = 0; @@ -465,7 +310,7 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) { return buf; } -static void dumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int precision) { +void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int32_t precision) { if (val == NULL) { taosFprintfFile(pFile, "%s", TSDB_DATA_NULL_STR); return; @@ -501,7 +346,7 @@ static void dumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, taosFprintfFile(pFile, "\'%s\'", buf); break; case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(buf, *(int64_t *)val, precision); + shellFormatTimestamp(buf, *(int64_t *)val, precision); taosFprintfFile(pFile, "'%s'", buf); break; default: @@ -509,35 +354,28 @@ static void dumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, } } -static int dumpResultToFile(const char *fname, TAOS_RES *tres) { +int32_t shellDumpResultToFile(const char *fname, TAOS_RES *tres) { + char fullname[PATH_MAX] = {0}; + if (taosExpandDir(fname, fullname, PATH_MAX) != 0) { + tstrncpy(fullname, fname, PATH_MAX); + } + TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; } - wordexp_t full_path; - - if (wordexp((char *)fname, &full_path, 0) != 0) { - fprintf(stderr, "ERROR: invalid file name: %s\n", fname); - return -1; - } - - // FILE *fp = fopen(full_path.we_wordv[0], "w"); - TdFilePtr pFile = - taosOpenFile(full_path.we_wordv[0], TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + TdFilePtr pFile = taosOpenFile(fullname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); if (pFile == NULL) { - fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]); - wordfree(&full_path); + fprintf(stderr, "failed to open file: %s\n", fullname); return -1; } - wordfree(&full_path); - - int num_fields = taos_num_fields(tres); TAOS_FIELD *fields = taos_fetch_fields(tres); - int precision = taos_result_precision(tres); + int32_t num_fields = taos_num_fields(tres); + int32_t precision = taos_result_precision(tres); - for (int col = 0; col < num_fields; col++) { + for (int32_t col = 0; col < num_fields; col++) { if (col > 0) { taosFprintfFile(pFile, ","); } @@ -545,14 +383,14 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { } taosFprintfFile(pFile, "\n"); - int numOfRows = 0; + int32_t numOfRows = 0; do { int32_t *length = taos_fetch_lengths(tres); - for (int i = 0; i < num_fields; i++) { + for (int32_t i = 0; i < num_fields; i++) { if (i > 0) { taosFprintfFile(pFile, "\n"); } - dumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision); + shellDumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision); } taosFprintfFile(pFile, "\n"); @@ -560,19 +398,19 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { row = taos_fetch_row(tres); } while (row != NULL); - result = 0; + atomic_store_64(&shell.result, 0); taosCloseFile(&pFile); return numOfRows; } -static void shellPrintNChar(const char *str, int length, int width) { +void shellPrintNChar(const char *str, int32_t length, int32_t width) { TdWchar tail[3]; - int pos = 0, cols = 0, totalCols = 0, tailLen = 0; + int32_t pos = 0, cols = 0, totalCols = 0, tailLen = 0; while (pos < length) { TdWchar wc; - int bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); + int32_t bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); if (bytes == 0) { break; } @@ -582,9 +420,9 @@ static void shellPrintNChar(const char *str, int length, int width) { } #ifdef WINDOWS - int w = bytes; + int32_t w = bytes; #else - int w = taosWcharWidth(wc); + int32_t w = taosWcharWidth(wc); #endif if (w <= 0) { continue; @@ -610,7 +448,7 @@ static void shellPrintNChar(const char *str, int length, int width) { if (totalCols > width) { // width could be 1 or 2, so printf("...") cannot be used - for (int i = 0; i < 3; i++) { + for (int32_t i = 0; i < 3; i++) { if (cols >= width) { break; } @@ -618,7 +456,7 @@ static void shellPrintNChar(const char *str, int length, int width) { ++cols; } } else { - for (int i = 0; i < tailLen; i++) { + for (int32_t i = 0; i < tailLen; i++) { printf("%lc", tail[i]); } cols = totalCols; @@ -629,9 +467,9 @@ static void shellPrintNChar(const char *str, int length, int width) { } } -static void printField(const char *val, TAOS_FIELD *field, int width, int32_t length, int precision) { +void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t length, int32_t precision) { if (val == NULL) { - int w = width; + int32_t w = width; if (field->type < TSDB_DATA_TYPE_TINYINT || field->type > TSDB_DATA_TYPE_DOUBLE) { w = 0; } @@ -682,7 +520,7 @@ static void printField(const char *val, TAOS_FIELD *field, int width, int32_t le shellPrintNChar(val, length, width); break; case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(buf, *(int64_t *)val, precision); + shellFormatTimestamp(buf, *(int64_t *)val, precision); printf("%s", buf); break; default: @@ -690,30 +528,19 @@ static void printField(const char *val, TAOS_FIELD *field, int width, int32_t le } } -bool isSelectQuery(TAOS_RES *tres) { -#if 0 - char *sql = tscGetSqlStr(tres); - - if (regex_match(sql, "^[\t ]*select[ \t]*", REG_EXTENDED | REG_ICASE)) { - return true; - } -#endif - return false; -} - -static int verticalPrintResult(TAOS_RES *tres) { +int32_t shellVerticalPrintResult(TAOS_RES *tres) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; } - int num_fields = taos_num_fields(tres); + int32_t num_fields = taos_num_fields(tres); TAOS_FIELD *fields = taos_fetch_fields(tres); - int precision = taos_result_precision(tres); + int32_t precision = taos_result_precision(tres); - int maxColNameLen = 0; - for (int col = 0; col < num_fields; col++) { - int len = (int)strlen(fields[col].name); + int32_t maxColNameLen = 0; + for (int32_t col = 0; col < num_fields; col++) { + int32_t len = (int32_t)strlen(fields[col].name); if (len > maxColNameLen) { maxColNameLen = len; } @@ -721,25 +548,25 @@ static int verticalPrintResult(TAOS_RES *tres) { uint64_t resShowMaxNum = UINT64_MAX; - if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) /*&& !tscIsQueryWithLimit(tres)*/) { - resShowMaxNum = DEFAULT_RES_SHOW_NUM; + if (shell.args.commands == NULL && shell.args.file[0] == 0) { + resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM; } - int numOfRows = 0; - int showMore = 1; + int32_t numOfRows = 0; + int32_t showMore = 1; do { if (numOfRows < resShowMaxNum) { printf("*************************** %d.row ***************************\n", numOfRows + 1); int32_t *length = taos_fetch_lengths(tres); - for (int i = 0; i < num_fields; i++) { + for (int32_t i = 0; i < num_fields; i++) { TAOS_FIELD *field = fields + i; - int padding = (int)(maxColNameLen - strlen(field->name)); + int32_t padding = (int32_t)(maxColNameLen - strlen(field->name)); printf("%*.s%s: ", padding, " ", field->name); - printField((const char *)row[i], field, 0, length[i], precision); + shellPrintField((const char *)row[i], field, 0, length[i], precision); putchar('\n'); } } else if (showMore) { @@ -755,8 +582,8 @@ static int verticalPrintResult(TAOS_RES *tres) { return numOfRows; } -static int calcColWidth(TAOS_FIELD *field, int precision) { - int width = (int)strlen(field->name); +int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { + int32_t width = (int32_t)strlen(field->name); switch (field->type) { case TSDB_DATA_TYPE_BOOL: @@ -785,23 +612,23 @@ static int calcColWidth(TAOS_FIELD *field, int precision) { return TMAX(25, width); case TSDB_DATA_TYPE_BINARY: - if (field->bytes > tsMaxBinaryDisplayWidth) { - return TMAX(tsMaxBinaryDisplayWidth, width); + if (field->bytes > shell.args.displayWidth) { + return TMAX(shell.args.displayWidth, width); } else { return TMAX(field->bytes, width); } case TSDB_DATA_TYPE_NCHAR: { int16_t bytes = field->bytes * TSDB_NCHAR_SIZE; - if (bytes > tsMaxBinaryDisplayWidth) { - return TMAX(tsMaxBinaryDisplayWidth, width); + if (bytes > shell.args.displayWidth) { + return TMAX(shell.args.displayWidth, width); } else { return TMAX(bytes, width); } } case TSDB_DATA_TYPE_TIMESTAMP: - if (args.is_raw_time) { + if (shell.args.is_raw_time) { return TMAX(14, width); } if (precision == TSDB_TIME_PRECISION_NANO) { @@ -819,55 +646,55 @@ static int calcColWidth(TAOS_FIELD *field, int precision) { return 0; } -static void printHeader(TAOS_FIELD *fields, int *width, int num_fields) { - int rowWidth = 0; - for (int col = 0; col < num_fields; col++) { +void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields) { + int32_t rowWidth = 0; + for (int32_t col = 0; col < num_fields; col++) { TAOS_FIELD *field = fields + col; - int padding = (int)(width[col] - strlen(field->name)); - int left = padding / 2; + int32_t padding = (int32_t)(width[col] - strlen(field->name)); + int32_t left = padding / 2; printf(" %*.s%s%*.s |", left, " ", field->name, padding - left, " "); rowWidth += width[col] + 3; } putchar('\n'); - for (int i = 0; i < rowWidth; i++) { + for (int32_t i = 0; i < rowWidth; i++) { putchar('='); } putchar('\n'); } -static int horizontalPrintResult(TAOS_RES *tres) { +int32_t shellHorizontalPrintResult(TAOS_RES *tres) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; } - int num_fields = taos_num_fields(tres); + int32_t num_fields = taos_num_fields(tres); TAOS_FIELD *fields = taos_fetch_fields(tres); - int precision = taos_result_precision(tres); + int32_t precision = taos_result_precision(tres); - int width[TSDB_MAX_COLUMNS]; - for (int col = 0; col < num_fields; col++) { - width[col] = calcColWidth(fields + col, precision); + int32_t width[TSDB_MAX_COLUMNS]; + for (int32_t col = 0; col < num_fields; col++) { + width[col] = shellCalcColWidth(fields + col, precision); } - printHeader(fields, width, num_fields); + shellPrintHeader(fields, width, num_fields); uint64_t resShowMaxNum = UINT64_MAX; - if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) /* && !tscIsQueryWithLimit(tres)*/) { - resShowMaxNum = DEFAULT_RES_SHOW_NUM; + if (shell.args.commands == NULL && shell.args.file[0] == 0) { + resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM; } - int numOfRows = 0; - int showMore = 1; + int32_t numOfRows = 0; + int32_t showMore = 1; do { int32_t *length = taos_fetch_lengths(tres); if (numOfRows < resShowMaxNum) { - for (int i = 0; i < num_fields; i++) { + for (int32_t i = 0; i < num_fields; i++) { putchar(' '); - printField((const char *)row[i], fields + i, width[i], length[i], precision); + shellPrintField((const char *)row[i], fields + i, width[i], length[i], precision); putchar(' '); putchar('|'); } @@ -885,50 +712,35 @@ static int horizontalPrintResult(TAOS_RES *tres) { return numOfRows; } -int shellDumpResult(TAOS_RES *tres, char *fname, int *error_no, bool vertical) { - int numOfRows = 0; +int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical) { + int32_t numOfRows = 0; if (fname != NULL) { - numOfRows = dumpResultToFile(fname, tres); + numOfRows = shellDumpResultToFile(fname, tres); } else if (vertical) { - numOfRows = verticalPrintResult(tres); + numOfRows = shellVerticalPrintResult(tres); } else { - numOfRows = horizontalPrintResult(tres); + numOfRows = shellHorizontalPrintResult(tres); } *error_no = taos_errno(tres); return numOfRows; } -void read_history() { - // Initialize history - memset(history.hist, 0, sizeof(char *) * MAX_HISTORY_SIZE); - history.hstart = 0; - history.hend = 0; - char *line = NULL; - int read_size = 0; - - char f_history[TSDB_FILENAME_LEN]; - get_history_path(f_history); - - // FILE *f = fopen(f_history, "r"); - TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) { -#ifndef WINDOWS - if (errno != ENOENT) { - fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno)); - } -#endif - return; - } +void shellReadHistory() { + SShellHistory *pHistory = &shell.history; + TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) return; + char *line = NULL; + int32_t read_size = 0; while ((read_size = taosGetLineFile(pFile, &line)) != -1) { line[read_size - 1] = '\0'; - history.hist[history.hend] = strdup(line); + pHistory->hist[pHistory->hend] = strdup(line); - history.hend = (history.hend + 1) % MAX_HISTORY_SIZE; + pHistory->hend = (pHistory->hend + 1) % SHELL_MAX_HISTORY_SIZE; - if (history.hend == history.hstart) { - history.hstart = (history.hstart + 1) % MAX_HISTORY_SIZE; + if (pHistory->hend == pHistory->hstart) { + pHistory->hstart = (pHistory->hstart + 1) % SHELL_MAX_HISTORY_SIZE; } } @@ -936,72 +748,48 @@ void read_history() { taosCloseFile(&pFile); } -void write_history() { - char f_history[TSDB_FILENAME_LEN]; - get_history_path(f_history); - - // FILE *f = fopen(f_history, "w"); - TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); - if (pFile == NULL) { -#ifndef WINDOWS - fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno)); -#endif - return; - } +void shellWriteHistory() { + SShellHistory *pHistory = &shell.history; + TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_WRITE | TD_FILE_STREAM); + if (pFile == NULL) return; - for (int i = history.hstart; i != history.hend;) { - if (history.hist[i] != NULL) { - taosFprintfFile(pFile, "%s\n", history.hist[i]); - taosMemoryFreeClear(history.hist[i]); + for (int32_t i = pHistory->hstart; i != pHistory->hend;) { + if (pHistory->hist[i] != NULL) { + taosFprintfFile(pFile, "%s\n", pHistory->hist[i]); + taosMemoryFreeClear(pHistory->hist[i]); } - i = (i + 1) % MAX_HISTORY_SIZE; + i = (i + 1) % SHELL_MAX_HISTORY_SIZE; } + taosFsyncFile(pFile); taosCloseFile(&pFile); } -void taos_error(TAOS_RES *tres, int64_t st) { +void shellPrintError(TAOS_RES *tres, int64_t st) { int64_t et = taosGetTimestampUs(); - atomic_store_ptr(&result, 0); + atomic_store_ptr(&shell.result, 0); fprintf(stderr, "\nDB error: %s (%.6fs)\n", taos_errstr(tres), (et - st) / 1E6); taos_free_result(tres); } -int isCommentLine(char *line) { - if (line == NULL) return 1; - - return regex_match(line, "^\\s*#.*", REG_EXTENDED); +bool shellIsCommentLine(char *line) { + if (line == NULL) return true; + return shellRegexMatch(line, "^\\s*#.*", REG_EXTENDED); } -void source_file(TAOS *con, char *fptr) { - wordexp_t full_path; - int read_len = 0; - char *cmd = taosMemoryCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); - size_t cmd_len = 0; - char *line = NULL; +void shellSourceFile(const char *file) { + int32_t read_len = 0; + char *cmd = taosMemoryCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); + size_t cmd_len = 0; + char *line = NULL; + char fullname[PATH_MAX] = {0}; - if (wordexp(fptr, &full_path, 0) != 0) { - fprintf(stderr, "ERROR: illegal file name\n"); - taosMemoryFree(cmd); - return; + if (taosExpandDir(file, fullname, PATH_MAX) != 0) { + tstrncpy(fullname, file, PATH_MAX); } - char *fname = full_path.we_wordv[0]; - - /* - if (access(fname, F_OK) != 0) { - fprintf(stderr, "ERROR: file %s is not exist\n", fptr); - - wordfree(&full_path); - taosMemoryFree(cmd); - return; - } - */ - - // FILE *f = fopen(fname, "r"); - TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ | TD_FILE_STREAM); + TdFilePtr pFile = taosOpenFile(fullname, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { - fprintf(stderr, "ERROR: failed to open file %s\n", fname); - wordfree(&full_path); + fprintf(stderr, "failed to open file %s\n", fullname); taosMemoryFree(cmd); return; } @@ -1010,7 +798,7 @@ void source_file(TAOS *con, char *fptr) { if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; line[--read_len] = '\0'; - if (read_len == 0 || isCommentLine(line)) { // line starts with # + if (read_len == 0 || shellIsCommentLine(line)) { // line starts with # continue; } @@ -1022,36 +810,38 @@ void source_file(TAOS *con, char *fptr) { } memcpy(cmd + cmd_len, line, read_len); - printf("%s%s\n", PROMPT_HEADER, cmd); - shellRunCommand(con, cmd); + printf("%s%s\n", shell.info.promptHeader, cmd); + shellRunCommand(cmd); memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); cmd_len = 0; } taosMemoryFree(cmd); if (line != NULL) taosMemoryFree(line); - wordfree(&full_path); taosCloseFile(&pFile); } -void shellGetGrantInfo(void *con) { - return; -#if 0 +void shellGetGrantInfo() { + char sinfo[1024] = {0}; + tstrncpy(sinfo, taos_get_server_info(shell.conn), sizeof(sinfo)); + strtok(sinfo, "\n"); + char sql[] = "show grants"; - TAOS_RES* tres = taos_query(con, sql); + TAOS_RES *tres = taos_query(shell.conn, sql); - int code = taos_errno(tres); + int32_t code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { - if (code == TSDB_CODE_COM_OPS_NOT_SUPPORT) { - fprintf(stdout, "Server is Community Edition, version is %s\n\n", taos_get_server_info(con)); + if (code == TSDB_CODE_OPS_NOT_SUPPORT) { + fprintf(stdout, "Server is Community Edition, %s\n\n", sinfo); } else { - fprintf(stderr, "Failed to check Server Edition, Reason:%d:%s\n\n", taos_errno(con), taos_errstr(con)); + fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\n\n", taos_errno(shell.conn), + taos_errstr(shell.conn)); } return; } - int num_fields = taos_field_count(tres); + int32_t num_fields = taos_field_count(tres); if (num_fields == 0) { fprintf(stderr, "\nInvalid grant information.\n"); exit(0); @@ -1062,7 +852,7 @@ void shellGetGrantInfo(void *con) { } TAOS_FIELD *fields = taos_fetch_fields(tres); - TAOS_ROW row = taos_fetch_row(tres); + TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { fprintf(stderr, "\nFailed to get grant information from server. Abort.\n"); exit(0); @@ -1077,15 +867,128 @@ void shellGetGrantInfo(void *con) { memcpy(expired, row[2], fields[2].bytes); if (strcmp(expiretime, "unlimited") == 0) { - fprintf(stdout, "Server is Enterprise %s Edition, version is %s and will never expire.\n", serverVersion, taos_get_server_info(con)); + fprintf(stdout, "Server is Enterprise %s Edition, %s and will never expire.\n", serverVersion, sinfo); } else { - fprintf(stdout, "Server is Enterprise %s Edition, version is %s and will expire at %s.\n", serverVersion, taos_get_server_info(con), expiretime); + fprintf(stdout, "Server is Enterprise %s Edition, %s and will expire at %s.\n", serverVersion, sinfo, expiretime); } - result = NULL; + atomic_store_64(&shell.result, 0); taos_free_result(tres); } fprintf(stdout, "\n"); -#endif +} + +void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { tsem_post(&shell.cancelSem); } + +void shellSigintHandler(int32_t signum, void *sigInfo, void *context) { + // do nothing +} + +void shellCleanup(void *arg) { taosResetTerminalMode(); } + +void *shellCancelHandler(void *arg) { + setThreadName("shellCancelHandler"); + while (1) { + if (tsem_wait(&shell.cancelSem) != 0) { + taosMsleep(10); + continue; + } + + taosResetTerminalMode(); + printf("\nReceive SIGTERM or other signal, quit shell.\n"); + shellWriteHistory(); + shellExit(); + } + + return NULL; +} + +void *shellThreadLoop(void *arg) { + setThreadName("shellThreadLoop"); + taosGetOldTerminalMode(); + taosThreadCleanupPush(shellCleanup, NULL); + + char *command = taosMemoryMalloc(SHELL_MAX_COMMAND_SIZE); + if (command == NULL) { + printf("failed to malloc command\n"); + return NULL; + } + + do { + memset(command, 0, SHELL_MAX_COMMAND_SIZE); + taosSetTerminalMode(); + + if (shellReadCommand(command) != 0) { + break; + } + + taosResetTerminalMode(); + } while (shellRunCommand(command) == 0); + + taosMemoryFreeClear(command); + shellWriteHistory(); + shellExit(); + + taosThreadCleanupPop(1); + return NULL; +} + +int32_t shellExecute() { + printf(shell.info.clientVersion, shell.info.osname, taos_get_client_info()); + fflush(stdout); + + SShellArgs *pArgs = &shell.args; + if (shell.args.auth == NULL) { + shell.conn = taos_connect(pArgs->host, pArgs->user, pArgs->password, pArgs->database, pArgs->port); + } else { + shell.conn = taos_connect_auth(pArgs->host, pArgs->user, pArgs->auth, pArgs->database, pArgs->port); + } + + if (shell.conn == NULL) { + fflush(stdout); + return -1; + } + + shellReadHistory(); + + if (pArgs->commands != NULL || pArgs->file[0] != 0) { + if (pArgs->commands != NULL) { + printf("%s%s\n", shell.info.promptHeader, pArgs->commands); + char *cmd = strdup(pArgs->commands); + shellRunCommand(cmd); + taosMemoryFree(cmd); + } + + if (pArgs->file[0] != 0) { + shellSourceFile(pArgs->file); + } + + taos_close(shell.conn); + shellWriteHistory(); + return 0; + } + + if (tsem_init(&shell.cancelSem, 0, 0) != 0) { + printf("failed to create cancel semphore\n"); + return -1; + } + + TdThread spid = {0}; + taosThreadCreate(&spid, NULL, shellCancelHandler, NULL); + + taosSetSignal(SIGTERM, shellQueryInterruptHandler); + taosSetSignal(SIGHUP, shellQueryInterruptHandler); + taosSetSignal(SIGABRT, shellQueryInterruptHandler); + + taosSetSignal(SIGINT, shellSigintHandler); + + shellGetGrantInfo(shell.conn); + + while (1) { + taosThreadCreate(&shell.pid, NULL, shellThreadLoop, shell.conn); + taosThreadJoin(shell.pid, NULL); + } + + return 0; } diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index db73f2fc5c24140f8392d27b9d2afdad118335f8..6672cee367b04d1a45460a87502c86480d65969b 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -14,697 +14,53 @@ */ #define __USE_XOPEN -#include "shellCommand.h" -#include "tglobal.h" -#include "tlog.h" +#include "shellInt.h" -#ifndef WINDOWS -#include -#include -#include -#endif +SShellObj shell = {0}; -#define OPT_ABORT 1 /* abort */ - -int indicator = 1; - -void insertChar(Command *cmd, char *c, int size); -void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType); -const char *argp_program_version = version; -const char *argp_program_bug_address = ""; -static char doc[] = ""; -static char args_doc[] = ""; - -TdThread pid; -static tsem_t cancelSem; -extern void taos_init(); - -#ifndef WINDOWS -static struct argp_option options[] = { - {"host", 'h', "HOST", 0, "TDengine server FQDN to connect. The default host is localhost."}, - {"password", 'p', NULL, 0, "The password to use when connecting to the server."}, - {"port", 'P', "PORT", 0, "The TCP/IP port number to use for the connection."}, - {"user", 'u', "USER", 0, "The user name to use when connecting to the server."}, - {"auth", 'A', "Auth", 0, "The auth string to use when connecting to the server."}, - {"config-dir", 'c', "CONFIG_DIR", 0, "Configuration directory."}, - {"dump-config",'C', NULL, 0, "Dump configuration."}, - {"commands", 's', "COMMANDS", 0, "Commands to run without enter the shell."}, - {"raw-time", 'r', NULL, 0, "Output time as uint64_t."}, - {"file", 'f', "FILE", 0, "Script to run without enter the shell."}, - {"directory", 'D', "DIRECTORY", 0, "Use multi-thread to import all SQL files in the directory separately."}, - {"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."}, - {"check", 'k', "CHECK", 0, "Check tables."}, - {"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."}, - {"timezone", 'z', "TIMEZONE", 0, "Time zone of the shell, default is local."}, - {"status", 't', NULL, 0, "Check the service status."}, - {"verbose", 'v', NULL, 0, "Check the details of the service status."}, - {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speed|fqdn."}, - {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, - {"pktnum", 'N', "PKTNUM", 0, "Packet numbers used for net test, default is 100."}, -// Shuduo: 3.0 does not support UDP any more -// {"pkttype", 'S', "PKTTYPE", 0, "Packet type used for net test, default is TCP."}, - {0}}; - -static error_t parse_opt(int key, char *arg, struct argp_state *state) { - /* Get the input argument from argp_parse, which we - know is a pointer to our arguments structure. */ - SShellArguments *arguments = state->input; - wordexp_t full_path; - - switch (key) { - case 'h': - arguments->host = arg; - break; - case 'p': - break; - case 'P': - if (arg) { - arguments->port = atoi(arg); - } else { - fprintf(stderr, "Invalid port\n"); - return -1; - } - - break; - case 'z': - arguments->timezone = arg; - break; - case 'u': - arguments->user = arg; - break; - case 'A': - arguments->auth = arg; - break; - case 'c': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - if (strlen(full_path.we_wordv[0]) >= TSDB_FILENAME_LEN) { - fprintf(stderr, "config file path: %s overflow max len %d\n", full_path.we_wordv[0], TSDB_FILENAME_LEN - 1); - wordfree(&full_path); - return -1; - } - tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'C': - arguments->dump_config = true; - break; - case 's': - arguments->commands = arg; - break; - case 'r': - arguments->is_raw_time = true; - break; - case 'f': - if ((0 == strlen(arg)) || (wordexp(arg, &full_path, 0) != 0)) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - tstrncpy(arguments->file, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'D': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - tstrncpy(arguments->dir, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'T': - if (arg) { - arguments->threadNum = atoi(arg); - } else { - fprintf(stderr, "Invalid number of threads\n"); - return -1; - } - break; - case 'k': - arguments->check = atoi(arg); - break; - case 't': - arguments->status = true; - break; - case 'v': - arguments->verbose = true; - break; - case 'd': - arguments->database = arg; - break; - case 'n': - arguments->netTestRole = arg; - break; - case 'l': - if (arg) { - arguments->pktLen = atoi(arg); - } else { - fprintf(stderr, "Invalid packet length\n"); - return -1; - } - break; - case 'N': - if (arg) { - arguments->pktNum = atoi(arg); - } else { - fprintf(stderr, "Invalid packet number\n"); - return -1; - } - break; - case 'S': - arguments->pktType = arg; - break; - case OPT_ABORT: - arguments->abort = 1; - break; - default: - return ARGP_ERR_UNKNOWN; - } - return 0; -} - -/* Our argp parser. */ -static struct argp argp = {options, parse_opt, args_doc, doc}; - -char LINUXCLIENT_VERSION[] = - "Welcome to the TDengine shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; -char g_password[SHELL_MAX_PASSWORD_LEN]; - -static void parse_args(int argc, char *argv[], SShellArguments *arguments) { - for (int i = 1; i < argc; i++) { - if ((strncmp(argv[i], "-p", 2) == 0) || (strncmp(argv[i], "--password", 10) == 0)) { - printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info()); - if ((strlen(argv[i]) == 2) || (strncmp(argv[i], "--password", 10) == 0)) { - printf("Enter password: "); - taosSetConsoleEcho(false); - if (scanf("%20s", g_password) > 1) { - fprintf(stderr, "password reading error\n"); - } - taosSetConsoleEcho(true); - if (EOF == getchar()) { - fprintf(stderr, "getchar() return EOF\n"); - } - } else { - tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); - strcpy(argv[i], "-p"); - } - arguments->password = g_password; - arguments->is_use_passwd = true; - } - } -} - -#endif -void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { -#ifdef WINDOWS -#else - static char verType[32] = {0}; - sprintf(verType, "version: %s\n", version); - - argp_program_version = verType; - - if (argc > 1) { - parse_args(argc, argv, arguments); - } - - argp_parse(&argp, argc, argv, 0, 0, arguments); - if (arguments->abort) { -#ifndef _ALPINE -#if 0 - error(10, 0, "ABORTED"); -#endif -#else - abort(); -#endif - } -#endif -} - -int32_t shellReadCommand(TAOS *con, char *command) { -#ifdef WINDOWS -#else - unsigned hist_counter = history.hend; - char utf8_array[10] = "\0"; - Command cmd; - memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); - showOnScreen(&cmd); - - // Read input. - char c; - while (1) { - c = (char)getchar(); // getchar() return an 'int' value - - if (c == EOF) { - return c; - } - - if (c < 0) { // For UTF-8 - int count = countPrefixOnes(c); - utf8_array[0] = c; - for (int k = 1; k < count; k++) { - c = (char)getchar(); - utf8_array[k] = c; - } - insertChar(&cmd, utf8_array, count); - } else if (c < '\033') { - // Ctrl keys. TODO: Implement ctrl combinations - switch (c) { - case 1: // ctrl A - positionCursorHome(&cmd); - break; - case 3: - printf("\n"); - resetCommand(&cmd, ""); - kill(0, SIGINT); - break; - case 4: // EOF or Ctrl+D - printf("\n"); - taos_close(con); - // write the history - write_history(); - exitShell(); - break; - case 5: // ctrl E - positionCursorEnd(&cmd); - break; - case 8: - backspaceChar(&cmd); - break; - case '\n': - case '\r': - printf("\n"); - if (isReadyGo(&cmd)) { - sprintf(command, "%s%s", cmd.buffer, cmd.command); - taosMemoryFreeClear(cmd.buffer); - taosMemoryFreeClear(cmd.command); - return 0; - } else { - updateBuffer(&cmd); - } - break; - case 11: // Ctrl + K; - clearLineAfter(&cmd); - break; - case 12: // Ctrl + L; - system("clear"); - showOnScreen(&cmd); - break; - case 21: // Ctrl + U; - clearLineBefore(&cmd); - break; - } - } else if (c == '\033') { - c = (char)getchar(); - switch (c) { - case '[': - c = (char)getchar(); - switch (c) { - case 'A': // Up arrow - if (hist_counter != history.hstart) { - hist_counter = (hist_counter + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE; - resetCommand(&cmd, (history.hist[hist_counter] == NULL) ? "" : history.hist[hist_counter]); - } - break; - case 'B': // Down arrow - if (hist_counter != history.hend) { - int next_hist = (hist_counter + 1) % MAX_HISTORY_SIZE; - - if (next_hist != history.hend) { - resetCommand(&cmd, (history.hist[next_hist] == NULL) ? "" : history.hist[next_hist]); - } else { - resetCommand(&cmd, ""); - } - hist_counter = next_hist; - } - break; - case 'C': // Right arrow - moveCursorRight(&cmd); - break; - case 'D': // Left arrow - moveCursorLeft(&cmd); - break; - case '1': - if ((c = (char)getchar()) == '~') { - // Home key - positionCursorHome(&cmd); - } - break; - case '2': - if ((c = (char)getchar()) == '~') { - // Insert key - } - break; - case '3': - if ((c = (char)getchar()) == '~') { - // Delete key - deleteChar(&cmd); - } - break; - case '4': - if ((c = (char)getchar()) == '~') { - // End key - positionCursorEnd(&cmd); - } - break; - case '5': - if ((c = (char)getchar()) == '~') { - // Page up key - } - break; - case '6': - if ((c = (char)getchar()) == '~') { - // Page down key - } - break; - case 72: - // Home key - positionCursorHome(&cmd); - break; - case 70: - // End key - positionCursorEnd(&cmd); - break; - } - break; - } - } else if (c == 0x7f) { - // press delete key - backspaceChar(&cmd); - } else { - insertChar(&cmd, &c, 1); - } - } - -#endif - return 0; -} - -void *shellLoopQuery(void *arg) { - if (indicator) { - getOldTerminalMode(); - indicator = 0; - } - - TAOS *con = (TAOS *)arg; - - setThreadName("shellLoopQuery"); - - taosThreadCleanupPush(cleanup_handler, NULL); - - char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); - if (command == NULL) { - uError("failed to malloc command"); - return NULL; - } - - int32_t err = 0; - - do { - // Read command from shell. - memset(command, 0, MAX_COMMAND_SIZE); - setTerminalMode(); - err = shellReadCommand(con, command); - if (err) { - break; - } - resetTerminalMode(); - } while (shellRunCommand(con, command) == 0); - - taosMemoryFreeClear(command); - exitShell(); - - taosThreadCleanupPop(1); - - return NULL; -} - -void get_history_path(char *_history) { snprintf(_history, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), HISTORY_FILE); } - -void clearScreen(int ecmd_pos, int cursor_pos) { -#ifdef WINDOWS -#else - struct winsize w; - if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - // fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); - w.ws_col = 120; - w.ws_row = 30; - } - - int cursor_x = cursor_pos / w.ws_col; - int cursor_y = cursor_pos % w.ws_col; - int command_x = ecmd_pos / w.ws_col; - positionCursor(cursor_y, LEFT); - positionCursor(command_x - cursor_x, DOWN); - fprintf(stdout, "\033[2K"); - for (int i = 0; i < command_x; i++) { - positionCursor(1, UP); - fprintf(stdout, "\033[2K"); - } - fflush(stdout); -#endif -} - -void showOnScreen(Command *cmd) { -#ifdef WINDOWS -#else - struct winsize w; - if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - // fprintf(stderr, "No stream device\n"); - w.ws_col = 120; - w.ws_row = 30; - } - - TdWchar wc; - int size = 0; - - // Print out the command. - char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE); - memset(total_string, '\0', MAX_COMMAND_SIZE); - if (strcmp(cmd->buffer, "") == 0) { - sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command); - } else { - sprintf(total_string, "%s%s", CONTINUE_PROMPT, cmd->command); - } - - int remain_column = w.ws_col; - /* size = cmd->commandSize + prompt_size; */ - for (char *str = total_string; size < cmd->commandSize + prompt_size;) { - int ret = taosMbToWchar(&wc, str, MB_CUR_MAX); - if (ret < 0) break; - size += ret; - /* assert(size >= 0); */ - int width = taosWcharWidth(wc); - if (remain_column > width) { - printf("%lc", wc); - remain_column -= width; - } else { - if (remain_column == width) { - printf("%lc\n\r", wc); - remain_column = w.ws_col; - } else { - printf("\n\r%lc", wc); - remain_column = w.ws_col - width; - } - } - - str = total_string + size; +int main(int argc, char *argv[]) { + if (shellCheckIntSize() != 0) { + return -1; } - taosMemoryFree(total_string); - /* for (int i = 0; i < size; i++){ */ - /* char c = total_string[i]; */ - /* if (k % w.ws_col == 0) { */ - /* printf("%c\n\r", c); */ - /* } */ - /* else { */ - /* printf("%c", c); */ - /* } */ - /* k += 1; */ - /* } */ - - // Position the cursor - int cursor_pos = cmd->screenOffset + prompt_size; - int ecmd_pos = cmd->endOffset + prompt_size; - - int cursor_x = cursor_pos / w.ws_col; - int cursor_y = cursor_pos % w.ws_col; - // int cursor_y = cursor % w.ws_col; - int command_x = ecmd_pos / w.ws_col; - int command_y = ecmd_pos % w.ws_col; - // int command_y = (command.size() + prompt_size) % w.ws_col; - positionCursor(command_y, LEFT); - positionCursor(command_x, UP); - positionCursor(cursor_x, DOWN); - positionCursor(cursor_y, RIGHT); - fflush(stdout); -#endif -} - -void cleanup_handler(void *arg) { resetTerminalMode(); } - -void exitShell() { - taos_cleanup(); - exit(EXIT_SUCCESS); -} - -void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { tsem_post(&cancelSem); } - -void *cancelHandler(void *arg) { - setThreadName("cancelHandler"); - - while (1) { - if (tsem_wait(&cancelSem) != 0) { - taosMsleep(10); - continue; - } - - resetTerminalMode(); - printf("\nReceive ctrl+c or other signal, quit shell.\n"); - exitShell(); + if (shellParseArgs(argc, argv) != 0) { + return -1; } - return NULL; -} - -int checkVersion() { - if (sizeof(int8_t) != 1) { - printf("taos int8 size is %d(!= 1)", (int)sizeof(int8_t)); - return 0; - } - if (sizeof(int16_t) != 2) { - printf("taos int16 size is %d(!= 2)", (int)sizeof(int16_t)); - return 0; - } - if (sizeof(int32_t) != 4) { - printf("taos int32 size is %d(!= 4)", (int)sizeof(int32_t)); - return 0; - } - if (sizeof(int64_t) != 8) { - printf("taos int64 size is %d(!= 8)", (int)sizeof(int64_t)); + if (shell.args.is_version) { + shellPrintVersion(); return 0; } - return 1; -} - -// Global configurations -SShellArguments args = { - .host = NULL, - .user = NULL, - .database = NULL, - .timezone = NULL, - .is_raw_time = false, - .is_use_passwd = false, - .dump_config = false, - .file = "\0", - .dir = "\0", - .threadNum = 5, - .commands = NULL, - .pktLen = 1000, - .pktNum = 100, - .pktType = "TCP", - .netTestRole = NULL, -#ifndef TD_WINDOWS - .password = NULL, -#endif -}; -void shellDumpConfig() { - if (!args.dump_config) return; - - SConfig *pCfg = taosGetCfg(); - if (NULL == pCfg) { - printf("TDengine read global config failed!\n"); - exit(EXIT_FAILURE); + if (shell.args.is_gen_auth) { + shellGenerateAuth(); + return 0; } - cfgDumpCfg(pCfg, 0, 1); - exitShell(); -} -void shellTestNetWork() { - if (args.netTestRole && args.netTestRole[0] != 0) { - taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktNum, args.pktType); - exitShell(); + if (shell.args.is_help) { + shellPrintHelp(); + return 0; } -} - -void shellCheckServerStatus() { - if (!args.status && !args.verbose) return; - TSDB_SERVER_STATUS code; - do { - char details[1024] = {0}; - code = taos_check_server_status(args.host, args.port, details, args.verbose ? 1024 : 0); - switch (code) { - case TSDB_SRV_STATUS_UNAVAILABLE: - printf("0: unavailable\n"); - break; - case TSDB_SRV_STATUS_NETWORK_OK: - printf("1: network ok\n"); - break; - case TSDB_SRV_STATUS_SERVICE_OK: - printf("2: service ok\n"); - break; - case TSDB_SRV_STATUS_SERVICE_DEGRADED: - printf("3: service degraded\n"); - break; - case TSDB_SRV_STATUS_EXTING: - printf("4: exiting\n"); - break; - } - if (strlen(details) != 0) { - printf("%s\n\n", details); - } - if (code == TSDB_SRV_STATUS_NETWORK_OK && args.verbose) { - taosMsleep(1000); - } else { - break; - } - } while (1); - - exitShell(); -} + taos_init(); -void shellExecute() { - TAOS *con = shellInit(&args); - if (con == NULL) { - exitShell(); + if (shell.args.is_dump_config) { + shellDumpConfig(); + taos_cleanup(); + return 0; } - if (tsem_init(&cancelSem, 0, 0) != 0) { - printf("failed to create cancel semphore\n"); - exitShell(); + if (shell.args.is_startup || shell.args.is_check) { + shellCheckServerStatus(); + taos_cleanup(); + return 0; } - TdThread spid; - taosThreadCreate(&spid, NULL, cancelHandler, NULL); - - taosSetSignal(SIGTERM, shellQueryInterruptHandler); - taosSetSignal(SIGINT, shellQueryInterruptHandler); - taosSetSignal(SIGHUP, shellQueryInterruptHandler); - taosSetSignal(SIGABRT, shellQueryInterruptHandler); - - shellGetGrantInfo(con); - - while (1) { - taosThreadCreate(&pid, NULL, shellLoopQuery, con); - taosThreadJoin(pid, NULL); + if (shell.args.netrole != NULL) { + shellTestNetWork(); + taos_cleanup(); + return 0; } -} - -int main(int argc, char *argv[]) { - if (!checkVersion()) exitShell(); - - shellParseArgument(argc, argv, &args); - - taos_init(); - shellDumpConfig(); - shellCheckServerStatus(); - shellTestNetWork(); - shellExecute(); - return 0; + return shellExecute(); } diff --git a/tools/shell/src/shellNettest.c b/tools/shell/src/shellNettest.c new file mode 100644 index 0000000000000000000000000000000000000000..c8ec31c48bb29c56215c1bba4090eb8f4cddc7b4 --- /dev/null +++ b/tools/shell/src/shellNettest.c @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _GNU_SOURCE +#include "shellInt.h" + +static void shellWorkAsClient() { + SShellArgs *pArgs = &shell.args; + SRpcInit rpcInit = {0}; + SEpSet epSet = {.inUse = 0, .numOfEps = 1}; + SRpcMsg rpcRsp = {0}; + void *clientRpc = NULL; + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + + taosEncryptPass_c((uint8_t *)("_pwd"), strlen("_pwd"), pass); + rpcInit.label = "CHK"; + rpcInit.numOfThreads = 1; + rpcInit.sessions = 16; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.user = "_dnd"; + rpcInit.ckey = "_key"; + rpcInit.spi = 1; + rpcInit.secret = pass; + + clientRpc = rpcOpen(&rpcInit); + if (clientRpc == NULL) { + printf("failed to init net test client since %s\n", terrstr()); + goto _OVER; + } + + if (pArgs->host == NULL) { + pArgs->host = tsFirst; + } + char fqdn[TSDB_FQDN_LEN] = {0}; + tstrncpy(fqdn, pArgs->host, TSDB_FQDN_LEN); + strtok(fqdn, ":"); + + if (pArgs->port == 0) { + pArgs->port = tsServerPort; + } + + printf("network test client is initialized, the server is %s:%u\n", fqdn, pArgs->port); + + tstrncpy(epSet.eps[0].fqdn, fqdn, TSDB_FQDN_LEN); + epSet.eps[0].port = (uint16_t)pArgs->port; + + int32_t totalSucc = 0; + uint64_t startTime = taosGetTimestampUs(); + + for (int32_t i = 0; i < pArgs->pktNum; ++i) { + SRpcMsg rpcMsg = {.ahandle = (void *)0x9525, .msgType = TDMT_DND_NET_TEST}; + rpcMsg.pCont = rpcMallocCont(pArgs->pktLen); + rpcMsg.contLen = pArgs->pktLen; + + printf("request is sent, size:%d\n", rpcMsg.contLen); + rpcSendRecv(clientRpc, &epSet, &rpcMsg, &rpcRsp); + if (rpcRsp.code == 0 && rpcRsp.contLen == rpcMsg.contLen) { + printf("response is received, size:%d\n", rpcMsg.contLen); + if (rpcRsp.code == 0) totalSucc++; + } else { + printf("response not received since %s\n", tstrerror(rpcRsp.code)); + } + + rpcFreeCont(rpcRsp.pCont); + rpcRsp.pCont = NULL; + } + + uint64_t endTime = taosGetTimestampUs(); + uint64_t elT = endTime - startTime; + + printf("\ntotal succ:%5d/%d\tcost:%8.2lf ms\tspeed:%8.2lf MB/s\n", totalSucc, pArgs->pktNum, elT / 1000.0, + pArgs->pktLen / (elT / 1000000.0) / 1024.0 / 1024.0 * totalSucc); + +_OVER: + if (clientRpc != NULL) { + rpcClose(clientRpc); + } + if (rpcRsp.pCont != NULL) { + rpcFreeCont(rpcRsp.pCont); + } +} + +static void shellProcessMsg(void *p, SRpcMsg *pRpc, SEpSet *pEpSet) { + printf("request is received, size:%d\n", pRpc->contLen); + fflush(stdout); + SRpcMsg rsp = {.handle = pRpc->handle, .refId = pRpc->refId, .ahandle = pRpc->ahandle, .code = 0}; + rsp.pCont = rpcMallocCont(pRpc->contLen); + if (rsp.pCont == NULL) { + rsp.code = TSDB_CODE_OUT_OF_MEMORY; + } else { + rsp.contLen = pRpc->contLen; + } + rpcSendResponse(&rsp); +} + +void shellNettestHandler(int32_t signum, void *sigInfo, void *context) { shellExit(); } + +static void shellWorkAsServer() { + SShellArgs *pArgs = &shell.args; + + if (pArgs->port == 0) { + pArgs->port = tsServerPort; + } + + SRpcInit rpcInit = {0}; + rpcInit.localPort = pArgs->port; + rpcInit.label = "CHK"; + rpcInit.numOfThreads = tsNumOfRpcThreads; + rpcInit.cfp = (RpcCfp)shellProcessMsg; + rpcInit.sessions = 10; + rpcInit.connType = TAOS_CONN_SERVER; + rpcInit.idleTime = tsShellActivityTimer * 1000; + + void *serverRpc = rpcOpen(&rpcInit); + if (serverRpc == NULL) { + printf("failed to init net test server since %s", terrstr()); + } else { + printf("network test server is initialized, port:%u\n", pArgs->port); + taosSetSignal(SIGTERM, shellNettestHandler); + while (1) taosMsleep(10); + } +} + +void shellTestNetWork() { + if (strcmp(shell.args.netrole, "client") == 0) { + shellWorkAsClient(); + } + + if (strcmp(shell.args.netrole, "server") == 0) { + shellWorkAsServer(); + } +} diff --git a/tools/shell/src/shellUtil.c b/tools/shell/src/shellUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..1529ac0e52b45d11c29691b491bb0284918c0575 --- /dev/null +++ b/tools/shell/src/shellUtil.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _BSD_SOURCE +#define _GNU_SOURCE +#define _XOPEN_SOURCE +#define _DEFAULT_SOURCE + +#include "shellInt.h" + +bool shellRegexMatch(const char *s, const char *reg, int32_t cflags) { + regex_t regex = {0}; + char msgbuf[100] = {0}; + + /* Compile regular expression */ + if (regcomp(®ex, reg, cflags) != 0) { + fprintf(stderr, "Fail to compile regex"); + shellExit(); + } + + /* Execute regular expression */ + int32_t reti = regexec(®ex, s, 0, NULL, 0); + if (!reti) { + regfree(®ex); + return true; + } else if (reti == REG_NOMATCH) { + regfree(®ex); + return false; + } else { + regerror(reti, ®ex, msgbuf, sizeof(msgbuf)); + fprintf(stderr, "Regex match failed: %s\n", msgbuf); + regfree(®ex); + shellExit(); + } + + return false; +} + +int32_t shellCheckIntSize() { + if (sizeof(int8_t) != 1) { + printf("taos int8 size is %d(!= 1)", (int)sizeof(int8_t)); + return -1; + } + if (sizeof(int16_t) != 2) { + printf("taos int16 size is %d(!= 2)", (int)sizeof(int16_t)); + return -1; + } + if (sizeof(int32_t) != 4) { + printf("taos int32 size is %d(!= 4)", (int)sizeof(int32_t)); + return -1; + } + if (sizeof(int64_t) != 8) { + printf("taos int64 size is %d(!= 8)", (int)sizeof(int64_t)); + return -1; + } + return 0; +} + +void shellPrintVersion() { printf("version: %s\n", version); } + +void shellGenerateAuth() { + char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)shell.args.password, strlen(shell.args.password), secretEncrypt); + printf("%s\n", secretEncrypt); + fflush(stdout); +} + +void shellDumpConfig() { + SConfig *pCfg = taosGetCfg(); + if (pCfg == NULL) { + printf("TDengine read global config failed!\n"); + } else { + cfgDumpCfg(pCfg, 1, true); + } + fflush(stdout); +} + +void shellCheckServerStatus() { + TSDB_SERVER_STATUS code; + + do { + char details[1024] = {0}; + code = taos_check_server_status(shell.args.host, shell.args.port, details, 1024); + switch (code) { + case TSDB_SRV_STATUS_UNAVAILABLE: + printf("0: unavailable\n"); + break; + case TSDB_SRV_STATUS_NETWORK_OK: + printf("1: network ok\n"); + break; + case TSDB_SRV_STATUS_SERVICE_OK: + printf("2: service ok\n"); + break; + case TSDB_SRV_STATUS_SERVICE_DEGRADED: + printf("3: service degraded\n"); + break; + case TSDB_SRV_STATUS_EXTING: + printf("4: exiting\n"); + break; + } + if (strlen(details) != 0) { + printf("%s\n\n", details); + } + fflush(stdout); + if (code == TSDB_SRV_STATUS_NETWORK_OK && shell.args.is_startup) { + taosMsleep(1000); + } else { + break; + } + } while (1); +} + +void shellExit() { + if (shell.conn != NULL) { + taos_close(shell.conn); + shell.conn = NULL; + } + taos_cleanup(); + exit(EXIT_FAILURE); +} \ No newline at end of file diff --git a/tools/shell/src/tnettest.c b/tools/shell/src/tnettest.c deleted file mode 100644 index 04b99ebc75e431bf9599965a1fb71b2bdc211d96..0000000000000000000000000000000000000000 --- a/tools/shell/src/tnettest.c +++ /dev/null @@ -1,514 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#define ALLOW_FORBID_FUNC -#include "os.h" -#include "taosdef.h" -#include "tmsg.h" -#include "taoserror.h" -#include "tlog.h" -#include "tglobal.h" -#include "trpc.h" -#include "rpcHead.h" -#include "tchecksum.h" -#include "syncMsg.h" - -#include "osSocket.h" - -#define MAX_PKG_LEN (64 * 1000) -#define MAX_SPEED_PKG_LEN (1024 * 1024 * 1024) -#define MIN_SPEED_PKG_LEN 1024 -#define MAX_SPEED_PKG_NUM 10000 -#define MIN_SPEED_PKG_NUM 1 -#define BUFFER_SIZE (MAX_PKG_LEN + 1024) - -extern int tsRpcMaxUdpSize; - -typedef struct { - char * hostFqdn; - uint32_t hostIp; - int32_t port; - int32_t pktLen; -} STestInfo; - -static void *taosNetBindUdpPort(void *sarg) { - STestInfo *pinfo = (STestInfo *)sarg; - int32_t port = pinfo->port; - SOCKET serverSocket; - char buffer[BUFFER_SIZE]; - int32_t iDataNum; - socklen_t sin_size; - int32_t bufSize = 1024000; - - struct sockaddr_in server_addr; - struct sockaddr_in clientAddr; - - setThreadName("netBindUdpPort"); - - if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { - uError("failed to create UDP socket since %s", strerror(errno)); - return NULL; - } - - bzero(&server_addr, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - server_addr.sin_addr.s_addr = htonl(INADDR_ANY); - - if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { - uError("failed to bind UDP port:%d since %s", port, strerror(errno)); - return NULL; - } - - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(serverSocket); - return NULL; - } - pSocket->fd = serverSocket; - pSocket->refId = 0; - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - uError("failed to set the send buffer size for UDP socket\n"); - taosCloseSocket(&pSocket); - return NULL; - } - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - uError("failed to set the receive buffer size for UDP socket\n"); - taosCloseSocket(&pSocket); - return NULL; - } - - uInfo("UDP server at port:%d is listening", port); - - while (1) { - memset(buffer, 0, BUFFER_SIZE); - sin_size = sizeof(*(struct sockaddr *)&server_addr); - iDataNum = recvfrom(serverSocket, buffer, BUFFER_SIZE, 0, (struct sockaddr *)&clientAddr, &sin_size); - - if (iDataNum < 0) { - uDebug("failed to perform recvfrom func at %d since %s", port, strerror(errno)); - continue; - } - - uInfo("UDP: recv:%d bytes from %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); - - if (iDataNum > 0) { - iDataNum = taosSendto(pSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size); - } - - uInfo("UDP: send:%d bytes to %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); - } - - taosCloseSocket(&pSocket); - return NULL; -} - -static void *taosNetBindTcpPort(void *sarg) { - struct sockaddr_in server_addr; - struct sockaddr_in clientAddr; - - STestInfo *pinfo = sarg; - int32_t port = pinfo->port; - SOCKET serverSocket; - int32_t addr_len = sizeof(clientAddr); - SOCKET client; - char buffer[BUFFER_SIZE]; - - setThreadName("netBindTcpPort"); - - if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { - uError("failed to create TCP socket since %s", strerror(errno)); - return NULL; - } - - bzero(&server_addr, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - server_addr.sin_addr.s_addr = htonl(INADDR_ANY); - - int32_t reuse = 1; - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(serverSocket); - return NULL; - } - pSocket->fd = serverSocket; - pSocket->refId = 0; - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { - uError("failed to bind TCP port:%d since %s", port, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - if (taosKeepTcpAlive(pSocket) < 0) { - uError("failed to set tcp server keep-alive option since %s", strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - if (listen(serverSocket, 10) < 0) { - uError("failed to listen TCP port:%d since %s", port, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - uInfo("TCP server at port:%d is listening", port); - - while (1) { - client = accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len); - if (client < 0) { - uDebug("TCP: failed to accept at port:%d since %s", port, strerror(errno)); - continue; - } - - int32_t ret = taosReadMsg(pSocket, buffer, pinfo->pktLen); - if (ret < 0 || ret != pinfo->pktLen) { - uError("TCP: failed to read %d bytes at port:%d since %s", pinfo->pktLen, port, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - uInfo("TCP: read:%d bytes from %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port); - - ret = taosWriteMsg(pSocket, buffer, pinfo->pktLen); - if (ret < 0) { - uError("TCP: failed to write %d bytes at %d since %s", pinfo->pktLen, port, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - uInfo("TCP: write:%d bytes to %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port); - } - - taosCloseSocket(&pSocket); - return NULL; -} - -static int32_t taosNetCheckTcpPort(STestInfo *info) { - SOCKET clientSocket; - char buffer[BUFFER_SIZE] = {0}; - - if ((clientSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { - uError("failed to create TCP client socket since %s", strerror(errno)); - return -1; - } - - int32_t reuse = 1; - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(clientSocket); - return -1; - } - pSocket->fd = clientSocket; - pSocket->refId = 0; - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - struct sockaddr_in serverAddr; - memset((char *)&serverAddr, 0, sizeof(serverAddr)); - serverAddr.sin_family = AF_INET; - serverAddr.sin_port = (uint16_t)htons((uint16_t)info->port); - serverAddr.sin_addr.s_addr = info->hostIp; - - if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { - uError("TCP: failed to connect port %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - taosKeepTcpAlive(pSocket); - - sprintf(buffer, "client send TCP pkg to %s:%d, content: 1122334455", taosIpStr(info->hostIp), info->port); - sprintf(buffer + info->pktLen - 16, "1122334455667788"); - - int32_t ret = taosWriteMsg(pSocket, buffer, info->pktLen); - if (ret < 0) { - uError("TCP: failed to write msg to %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - ret = taosReadMsg(pSocket, buffer, info->pktLen); - if (ret < 0) { - uError("TCP: failed to read msg from %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - taosCloseSocket(&pSocket); - return 0; -} - -static int32_t taosNetCheckUdpPort(STestInfo *info) { - SOCKET clientSocket; - char buffer[BUFFER_SIZE] = {0}; - int32_t iDataNum = 0; - int32_t bufSize = 1024000; - - struct sockaddr_in serverAddr; - - if ((clientSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { - uError("failed to create udp client socket since %s", strerror(errno)); - return -1; - } - - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(clientSocket); - return -1; - } - pSocket->fd = clientSocket; - pSocket->refId = 0; - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - uError("failed to set the send buffer size for UDP socket\n"); - taosCloseSocket(&pSocket); - return -1; - } - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - uError("failed to set the receive buffer size for UDP socket\n"); - taosCloseSocket(&pSocket); - return -1; - } - - serverAddr.sin_family = AF_INET; - serverAddr.sin_port = htons(info->port); - serverAddr.sin_addr.s_addr = info->hostIp; - - struct in_addr ipStr; - memcpy(&ipStr, &info->hostIp, 4); - sprintf(buffer, "client send udp pkg to %s:%d, content: 1122334455", taosInetNtoa(ipStr), info->port); - sprintf(buffer + info->pktLen - 16, "1122334455667788"); - - socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr); - - iDataNum = taosSendto(pSocket, buffer, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size); - if (iDataNum < 0 || iDataNum != info->pktLen) { - uError("UDP: failed to perform sendto func since %s", strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - memset(buffer, 0, BUFFER_SIZE); - sin_size = sizeof(*(struct sockaddr *)&serverAddr); - iDataNum = recvfrom(clientSocket, buffer, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size); - - if (iDataNum < 0 || iDataNum != info->pktLen) { - uError("UDP: received ack:%d bytes(expect:%d) from port:%d since %s", iDataNum, info->pktLen, info->port, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - taosCloseSocket(&pSocket); - return 0; -} - -static void taosNetCheckPort(uint32_t hostIp, int32_t startPort, int32_t endPort, int32_t pktLen) { - int32_t ret; - STestInfo info; - - memset(&info, 0, sizeof(STestInfo)); - info.hostIp = hostIp; - info.pktLen = pktLen; - - for (int32_t port = startPort; port <= endPort; port++) { - info.port = port; - ret = taosNetCheckTcpPort(&info); - if (ret != 0) { - printf("failed to test TCP port:%d\n", port); - } else { - printf("successed to test TCP port:%d\n", port); - } - - ret = taosNetCheckUdpPort(&info); - if (ret != 0) { - printf("failed to test UDP port:%d\n", port); - } else { - printf("successed to test UDP port:%d\n", port); - } - } -} - -static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) { - uInfo("work as client, host:%s Port:%d pkgLen:%d\n", host, startPort, pkgLen); - - uint32_t serverIp = taosGetIpv4FromFqdn(host); - if (serverIp == 0xFFFFFFFF) { - uError("failed to resolve fqdn:%s", host); - exit(-1); - } - - uInfo("server ip:%s is resolved from host:%s", taosIpStr(serverIp), host); - taosNetCheckPort(serverIp, startPort, startPort, pkgLen); -} - -static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { - uInfo("work as server, host:%s Port:%d pkgLen:%d\n", host, startPort, pkgLen); - - int32_t port = startPort; - int32_t num = 1; - if (num < 0) num = 1; - - TdThread *pids = taosMemoryMalloc(2 * num * sizeof(TdThread)); - STestInfo *tinfos = taosMemoryMalloc(num * sizeof(STestInfo)); - STestInfo *uinfos = taosMemoryMalloc(num * sizeof(STestInfo)); - - for (int32_t i = 0; i < num; i++) { - STestInfo *tcpInfo = tinfos + i; - tcpInfo->port = port + i; - tcpInfo->pktLen = pkgLen; - - if (taosThreadCreate(pids + i, NULL, taosNetBindTcpPort, tcpInfo) != 0) { - uInfo("failed to create TCP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); - exit(-1); - } - - STestInfo *udpInfo = uinfos + i; - udpInfo->port = port + i; - tcpInfo->pktLen = pkgLen; - if (taosThreadCreate(pids + num + i, NULL, taosNetBindUdpPort, udpInfo) != 0) { - uInfo("failed to create UDP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); - exit(-1); - } - } - - for (int32_t i = 0; i < num; i++) { - taosThreadJoin(pids[i], NULL); - taosThreadJoin(pids[(num + i)], NULL); - } -} - -static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, - int32_t pkgNum, char *pkgType) { -#if 0 - - // record config - int32_t compressTmp = tsCompressMsgSize; - int32_t maxUdpSize = tsRpcMaxUdpSize; - int32_t forceTcp = tsRpcForceTcp; - - if (0 == strcmp("tcp", pkgType)){ - tsRpcForceTcp = 1; - tsRpcMaxUdpSize = 0; // force tcp - } else { - tsRpcForceTcp = 0; - tsRpcMaxUdpSize = INT_MAX; - } - tsCompressMsgSize = -1; - - SEpSet epSet; - SRpcMsg reqMsg; - SRpcMsg rspMsg; - void * pRpcConn; - char secretEncrypt[32] = {0}; - char spi = 0; - pRpcConn = taosNetInitRpc(secretEncrypt, spi); - if (NULL == pRpcConn) { - uError("failed to init client rpc"); - return; - } - - printf("check net spend, host:%s port:%d pkgLen:%d pkgNum:%d pkgType:%s\n\n", host, port, pkgLen, pkgNum, pkgType); - int32_t totalSucc = 0; - uint64_t startT = taosGetTimestampUs(); - for (int32_t i = 1; i <= pkgNum; i++) { - uint64_t startTime = taosGetTimestampUs(); - - memset(&epSet, 0, sizeof(SEpSet)); - strcpy(epSet.eps[0].fqdn, host); - epSet.eps[0].port = port; - epSet.numOfEps = 1; - - reqMsg.msgType = TDMT_DND_NETWORK_TEST; - reqMsg.pCont = rpcMallocCont(pkgLen); - reqMsg.contLen = pkgLen; - reqMsg.code = 0; - reqMsg.handle = NULL; // rpc handle returned to app - reqMsg.ahandle = NULL; // app handle set by client - strcpy(reqMsg.pCont, "nettest speed"); - - rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg); - - int code = 0; - if ((rspMsg.code != 0) || (rspMsg.msgType != TDMT_DND_NETWORK_TEST + 1)) { - uError("ret code 0x%x %s", rspMsg.code, tstrerror(rspMsg.code)); - code = -1; - }else{ - totalSucc ++; - } - - rpcFreeCont(rspMsg.pCont); - - uint64_t endTime = taosGetTimestampUs(); - uint64_t el = endTime - startTime; - printf("progress:%5d/%d\tstatus:%d\tcost:%8.2lf ms\tspeed:%8.2lf MB/s\n", i, pkgNum, code, el/1000.0, pkgLen/(el/1000000.0)/1024.0/1024.0); - } - int64_t endT = taosGetTimestampUs(); - uint64_t elT = endT - startT; - printf("\ntotal succ:%5d/%d\tcost:%8.2lf ms\tspeed:%8.2lf MB/s\n", totalSucc, pkgNum, elT/1000.0, pkgLen/(elT/1000000.0)/1024.0/1024.0*totalSucc); - - rpcClose(pRpcConn); - - // return config - tsCompressMsgSize = compressTmp; - tsRpcMaxUdpSize = maxUdpSize; - tsRpcForceTcp = forceTcp; - return; -#endif -} - -void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType) { - tsLogEmbedded = 1; - if (host == NULL) host = tsLocalFqdn; - if (port == 0) port = tsServerPort; - if (0 == strcmp("speed", role)) { - if (pkgLen <= MIN_SPEED_PKG_LEN) pkgLen = MIN_SPEED_PKG_LEN; - if (pkgLen > MAX_SPEED_PKG_LEN) pkgLen = MAX_SPEED_PKG_LEN; - if (pkgNum <= MIN_SPEED_PKG_NUM) pkgNum = MIN_SPEED_PKG_NUM; - if (pkgNum > MAX_SPEED_PKG_NUM) pkgNum = MAX_SPEED_PKG_NUM; - } else { - if (pkgLen <= 10) pkgLen = 1000; - if (pkgLen > MAX_PKG_LEN) pkgLen = MAX_PKG_LEN; - } - - if (0 == strcmp("client", role)) { - taosNetTestClient(host, port, pkgLen); - } else if (0 == strcmp("server", role)) { - taosNetTestServer(host, port, pkgLen); - } else if (0 == strcmp("speed", role)) { - tsLogEmbedded = 0; - char type[10] = {0}; - taosNetCheckSpeed(host, port, pkgLen, pkgNum, strtolower(type, pkgType)); - } else { - TASSERT(1); - } - - tsLogEmbedded = 0; -}