diff --git a/CMakeLists.txt b/CMakeLists.txt index 500b2ff1eb0aff9d254894eb933e669d31cd6714..8dd99f0b58fbb7f43dfadac3927f80a7f41fc565 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,8 +16,8 @@ MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR}) INCLUDE(cmake/input.inc) INCLUDE(cmake/platform.inc) -INCLUDE(cmake/env.inc) INCLUDE(cmake/define.inc) +INCLUDE(cmake/env.inc) INCLUDE(cmake/install.inc) ADD_SUBDIRECTORY(deps) diff --git a/cmake/define.inc b/cmake/define.inc index d68f2230f12a2d1a36e8c1b11437f01ab651ad78..1bb1692a2b233851f8856c68f5f6a6fef5254bb6 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -28,3 +28,107 @@ ENDIF () IF (TD_RANDOM_NETWORK_FAIL) ADD_DEFINITIONS(-DTAOS_RANDOM_NETWORK_FAIL) ENDIF () + +IF (TD_ARM) + ADD_DEFINITIONS(-D_TD_ARM_) + IF (TD_ARM_64) + ADD_DEFINITIONS(-D_TD_ARM_64_) + ENDIF () + IF (TD_ARM_32) + ADD_DEFINITIONS(-D_TD_ARM_32_) + ENDIF () +ENDIF () + +IF (TD_MIPS) + ADD_DEFINITIONS(-D_TD_MIPS_) + IF (TD_MIPS_64) + ADD_DEFINITIONS(-D_TD_MIPS_64_) + ENDIF () + IF (TD_MIPS_32) + ADD_DEFINITIONS(-D_TD_MIPS_32_) + ENDIF () +ENDIF () + +IF (TD_LINUX_64) + ADD_DEFINITIONS(-D_M_X64) + ADD_DEFINITIONS(-D_TD_LINUX_64) + IF (NOT TD_ARM) + IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ELSE () + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ENDIF () + ELSE () + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ENDIF () + FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) + IF (ICONV_INCLUDE_EXIST) + ADD_DEFINITIONS(-DUSE_LIBICONV) + ENDIF () +ENDIF () + +IF (TD_LINUX_32) + ADD_DEFINITIONS(-D_TD_LINUX_32) + ADD_DEFINITIONS(-DUSE_LIBICONV) + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + IF (NOT TD_ARM) + EXIT () + ENDIF () +ENDIF () + +IF (TD_LINUX) + ADD_DEFINITIONS(-DLINUX) + ADD_DEFINITIONS(-D_TD_LINUX) + ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) + + SET(DEBUG_FLAGS "-O0 -DDEBUG") + SET(RELEASE_FLAGS "-O0") + + IF (${COVER} MATCHES "true") + MESSAGE(STATUS "Test coverage mode, add extra flags") + SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage") + SET(GCC_COVERAGE_LINK_FLAGS "-lgcov --coverage") + SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}") + ENDIF () +ENDIF () + +IF (TD_APLHINE) + SET(COMMON_FLAGS "${COMMON_FLAGS} -largp") + link_libraries(/usr/lib/libargp.a) + ADD_DEFINITIONS(-D_ALPINE) +ENDIF () + +IF (TD_DARWIN_64) + ADD_DEFINITIONS(-D_TD_DARWIN_64) + ADD_DEFINITIONS(-DDARWIN) + ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) + ADD_DEFINITIONS(-DUSE_LIBICONV) + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(DEBUG_FLAGS "-O0 -DDEBUG") + SET(RELEASE_FLAGS "-O0") +ENDIF () + +IF (TD_WINDOWS) + ADD_DEFINITIONS(-DWINDOWS) + ADD_DEFINITIONS(-D__CLEANUP_C) + ADD_DEFINITIONS(-DPTW32_STATIC_LIB) + ADD_DEFINITIONS(-DPTW32_BUILD) + ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) + SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) + IF (NOT TD_GODLL) + SET(COMMON_FLAGS "/nologo /WX /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") + SET(DEBUG_FLAGS "/Zi /W3 /GL") + SET(RELEASE_FLAGS "/W0 /GL") + ENDIF () +ENDIF () + +IF (TD_WINDOWS_64) + ADD_DEFINITIONS(-D_M_X64) + ADD_DEFINITIONS(-D_TD_WINDOWS_64) + ADD_DEFINITIONS(-DUSE_LIBICONV) +ENDIF () + +IF (TD_WINDOWS_32) + ADD_DEFINITIONS(-D_TD_WINDOWS_32) +ENDIF () + diff --git a/cmake/platform.inc b/cmake/platform.inc index 7a371df70b8b5d6bd717c33f1ef2970704831cc0..a0668de7b592f59ac817fd10ba1f33205b378aae 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -14,6 +14,7 @@ PROJECT(TDengine) # # Set macro definitions according to os platform +SET(TD_LINUX FALSE) SET(TD_LINUX_64 FALSE) SET(TD_LINUX_32 FALSE) SET(TD_ARM FALSE) @@ -22,8 +23,11 @@ SET(TD_ARM_32 FALSE) SET(TD_MIPS FALSE) SET(TD_MIPS_64 FALSE) SET(TD_MIPS_32 FALSE) -SET(TD_DARWIN_64 FALSE) +SET(TD_WINDOWS FALSE) SET(TD_WINDOWS_64 FALSE) +SET(TD_WINDOWS_32 FALSE) +SET(TD_DARWIN_64 FALSE) +SET(TD_APLHINE FALSE) # if generate ARM version: # cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64 @@ -31,27 +35,21 @@ IF (${CPUTYPE} MATCHES "aarch32") SET(TD_ARM TRUE) SET(TD_ARM_32 TRUE) SET(TD_PAGMODE_LITE TRUE) - ADD_DEFINITIONS(-D_TD_ARM_) - ADD_DEFINITIONS(-D_TD_ARM_32_) MESSAGE(STATUS "input cpuType: aarch32") ELSEIF (${CPUTYPE} MATCHES "aarch64") SET(TD_ARM TRUE) SET(TD_ARM_64 TRUE) - ADD_DEFINITIONS(-D_TD_ARM_) - ADD_DEFINITIONS(-D_TD_ARM_64_) - MESSAGE(STATUS "input cpuType: aarch64") + MESSAGE(STATUS "input cpuType: aarch64") ELSEIF (${CPUTYPE} MATCHES "mips64") SET(TD_MIPS TRUE) SET(TD_MIPS_64 TRUE) - ADD_DEFINITIONS(-D_TD_MIPS_) - ADD_DEFINITIONS(-D_TD_MIPS_64_) MESSAGE(STATUS "input cpuType: mips64") ELSEIF (${CPUTYPE} MATCHES "x64") MESSAGE(STATUS "input cpuType: x64") ELSEIF (${CPUTYPE} MATCHES "x86") MESSAGE(STATUS "input cpuType: x86") ELSE () - MESSAGE(STATUS "input cpuType: " ${CPUTYPE}) + MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE}) ENDIF () # @@ -62,16 +60,13 @@ execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh "" OUTPUT_ MESSAGE(STATUS "The current os is " ${TD_OS_INFO}) IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + SET(TD_LINUX TRUE) IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) SET(TD_LINUX_64 TRUE) - ADD_DEFINITIONS(-D_M_X64) - ADD_DEFINITIONS(-D_TD_LINUX_64) MESSAGE(STATUS "The current platform is Linux 64-bit") ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4) SET(TD_LINUX_32 TRUE) - ADD_DEFINITIONS(-D_TD_LINUX_32) IF (TD_ARM) - #ADD_DEFINITIONS(-D_M_IX86) MESSAGE(STATUS "The current platform is Linux 32-bit") ELSE () MESSAGE(FATAL_ERROR "The current platform is Linux 32-bit, but no ARM not supported yet") @@ -81,24 +76,26 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") MESSAGE(FATAL_ERROR "The current platform is Linux neither 32-bit nor 64-bit, not supported yet") EXIT () ENDIF () + + IF (${TD_OS_INFO} MATCHES "Alpine") + SET(TD_APLHINE TRUE) + MESSAGE(STATUS "The current OS is Alpine, append extra flags") + ENDIF() ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) SET(TD_DARWIN_64 TRUE) - ADD_DEFINITIONS(-D_TD_DARWIN_64) MESSAGE(STATUS "The current platform is Darwin 64-bit") ELSE () MESSAGE(FATAL_ERROR "The current platform is Darwin 32-bit, not supported yet") EXIT () ENDIF () ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows") + SET(TD_WINDOWS TRUE) IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) SET(TD_WINDOWS_64 TRUE) - ADD_DEFINITIONS(-D_M_X64) - ADD_DEFINITIONS(-D_TD_WINDOWS_64) MESSAGE(STATUS "The current platform is Windows 64-bit") ELSE () SET(TD_WINDOWS_32 TRUE) - ADD_DEFINITIONS(-D_TD_WINDOWS_32) MESSAGE(FATAL_ERROR "The current platform is Windows 32-bit, not supported yet") EXIT () ENDIF () @@ -106,74 +103,3 @@ ELSE() MESSAGE(FATAL_ERROR "The current platform is not Linux/Darwin/Windows, stop compile") EXIT () ENDIF () - -IF (TD_LINUX_64) - SET(DEBUG_FLAGS "-O0 -DDEBUG") - SET(RELEASE_FLAGS "-O0") - IF (NOT TD_ARM) - IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - ELSE () - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - ENDIF () - ELSE () - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - ENDIF () - ADD_DEFINITIONS(-DLINUX) - ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) - IF (${TD_OS_INFO} MATCHES "Alpine") - MESSAGE(STATUS "The current OS is Alpine, append extra flags") - SET(COMMON_FLAGS "${COMMON_FLAGS} -largp") - link_libraries(/usr/lib/libargp.a) - ADD_DEFINITIONS(-D_ALPINE) - ENDIF () - IF (${COVER} MATCHES "true") - MESSAGE(STATUS "Test coverage mode, add extra flags") - SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage") - SET(GCC_COVERAGE_LINK_FLAGS "-lgcov --coverage") - SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}") - ENDIF () - ELSEIF (TD_LINUX_32) - IF (NOT TD_ARM) - EXIT () - ENDIF () - SET(DEBUG_FLAGS "-O0 -DDEBUG") - SET(RELEASE_FLAGS "-O0") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - ADD_DEFINITIONS(-DLINUX) - ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) - ADD_DEFINITIONS(-DUSE_LIBICONV) - IF (${TD_OS_INFO} MATCHES "Alpine") - MESSAGE(STATUS "The current OS is Alpine, add extra flags") - SET(COMMON_FLAGS "${COMMON_FLAGS} -largp") - link_library(/usr/lib/libargp.a) - ADD_DEFINITIONS(-D_ALPINE) - ENDIF () - IF (${COVER} MATCHES "true") - MESSAGE(STATUS "Test coverage mode, add extra flags") - SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage") - SET(GCC_COVERAGE_LINK_FLAGS "-lgcov --coverage") - SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}") - ENDIF () - ELSEIF (TD_WINDOWS_64) - SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) - IF (NOT TD_GODLL) - SET(COMMON_FLAGS "/nologo /WX /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") - SET(DEBUG_FLAGS "/Zi /W3 /GL") - SET(RELEASE_FLAGS "/W0 /GL") - ENDIF () - ADD_DEFINITIONS(-DWINDOWS) - ADD_DEFINITIONS(-D__CLEANUP_C) - ADD_DEFINITIONS(-DPTW32_STATIC_LIB) - ADD_DEFINITIONS(-DPTW32_BUILD) - ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) - ELSEIF (TD_DARWIN_64) - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - SET(DEBUG_FLAGS "-O0 -DDEBUG") - SET(RELEASE_FLAGS "-O0") - ADD_DEFINITIONS(-DDARWIN) - ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) - ELSE () - MESSAGE(FATAL_ERROR "The current platform is not support yet, stop compile") - EXIT () - ENDIF () diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 17840df4a4062002b54f4aea5293427a5a7a1c3b..7efcd54cfda619d24e3457f1834ff77a8139828d 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -31,8 +31,8 @@ extern "C" { #include "tutil.h" #include "qExecutor.h" +#include "qSqlparser.h" #include "qTsbuf.h" -#include "qsqlparser.h" #include "tcmdtype.h" // forward declaration diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 9dd33e03cb21d8dfc807a8d8e1f91042621c7633..85cff4ba17859fe0ca48ce1e5b9a280765e0c79a 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -430,7 +430,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { pRes->code = code; if (code != TSDB_CODE_SUCCESS) { - tscError("%p ge tableMeta failed, code:%s", pSql, tstrerror(code)); + tscError("%p get tableMeta failed, code:%s", pSql, tstrerror(code)); goto _error; } else { tscDebug("%p get tableMeta successfully", pSql); diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 72ccd5adc6ee835e9e25f9b9b8b2511cad797910..17f6c97ea16223d692d4b8d06c00856ab477d09b 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2131,6 +2131,11 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { } bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval) { + SResultInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo == NULL) { + return true; + } + STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx); // required number of results are not reached, continue load data block diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 80fc82d90b1856753c4c88268c650147fed9e124..bf76b8cbe8a9fe31ef139db1a8b6f266a4b13c3b 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -691,9 +691,15 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pModel = createColumnModel(pSchema, size, capacity); + int32_t pg = DEFAULT_PAGE_SIZE; + int32_t overhead = sizeof(tFilePage); + while((pg - overhead) < pModel->rowSize * 2) { + pg *= 2; + } + size_t numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups; for (int32_t i = 0; i < numOfSubs; ++i) { - (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel); + (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pg, pModel); (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; } diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 8e554d62ffa1da2cdcefb9eaabf8ea400aef283f..19886177e69f4af28e3235b222c1b35fc1fe5cdf 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -334,7 +334,7 @@ void tscSaveSubscriptionProgress(void* sub) { char path[256]; sprintf(path, "%s/subscribe", tsDataDir); - if (tmkdir(path, 0777) != 0) { + if (taosMkDir(path, 0777) != 0) { tscError("failed to create subscribe dir: %s", path); } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 5d26d09fae1dcc6c947375004703a9e6621ec629..95d65f4aff325acdf05a4ec30c867e60e53b8fb4 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1505,12 +1505,11 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSql, trsupport, pSql); - // todo add to async res or not?? if (pNew == NULL) { - tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vgId:%d, orderOfSub:%d", - trsupport->pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex); + tscError("%p sub:%p failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d", + trsupport->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, trsupport->subqueryIndex); - pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; + pParentSql->res.code = terrno; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; return pParentSql->res.code; diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index bf1b67426089a91244f6e4d05863a6dfa5c90ee9..42356108b18dcbe6c78a619fd6f100ea493600c5 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -148,7 +148,7 @@ void taos_init_imp() { refreshTime = refreshTime < 10 ? 10 : refreshTime; if (tscCacheHandle == NULL) { - tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "client"); + tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta"); } tscDebug("client is initialized successfully"); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 17adc0c03d804280c43df0326a955cd29fd1aa60..e7fa2a84a9d23ca2e94a25679955c8536f6d483d 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -356,9 +356,9 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { // pSql->sqlstr will be used by tscBuildQueryStreamDesc if (pObj->signature == pObj) { - pthread_mutex_lock(&pObj->mutex); + //pthread_mutex_lock(&pObj->mutex); tfree(pSql->sqlstr); - pthread_mutex_unlock(&pObj->mutex); + //pthread_mutex_unlock(&pObj->mutex); } tscFreeSqlResult(pSql); @@ -1675,6 +1675,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { tscError("%p new subquery failed, tableIndex:%d", pSql, tableIndex); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } @@ -1688,6 +1689,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); free(pNew); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } @@ -1706,6 +1708,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { tscFreeSqlObj(pNew); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } @@ -1743,6 +1746,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); tscFreeSqlObj(pNew); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } @@ -1827,8 +1831,16 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } if (pFinalInfo->pTableMeta == NULL) { - tscError("%p new subquery failed for get tableMeta is NULL from cache", pSql); + tscError("%p new subquery failed since no tableMeta in cache, name:%s", pSql, name); tscFreeSqlObj(pNew); + + if (pPrevSql != NULL) { + assert(pPrevSql->res.code != TSDB_CODE_SUCCESS); + terrno = pPrevSql->res.code; + } else { + terrno = TSDB_CODE_TSC_APP_ERROR; + } + return NULL; } diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index bbea1a5e0bb52a56cfbf5a23dd29a17e094586a9..cb53bb5e60b1d4f0316c776644e7582af0579ebe 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -49,7 +49,7 @@ static taos_qset readQset; int32_t dnodeInitVnodeRead() { readQset = taosOpenQset(); - readPool.min = 2; + readPool.min = tsNumOfCores; readPool.max = tsNumOfCores * tsNumOfThreadsPerCore; if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min; readPool.readWorker = (SReadWorker *)calloc(sizeof(SReadWorker), readPool.max); @@ -206,10 +206,14 @@ static void *dnodeProcessReadQueue(void *param) { taosMsg[pReadMsg->rpcMsg.msgType], type); int32_t code = vnodeProcessRead(pVnode, pReadMsg); - if (type == TAOS_QTYPE_RPC) { + if (type == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) { dnodeSendRpcReadRsp(pVnode, pReadMsg, code); } else { - dnodeDispatchNonRspMsg(pVnode, pReadMsg, code); + if (code == TSDB_CODE_QRY_HAS_RSP) { + dnodeSendRpcReadRsp(pVnode, pReadMsg, TSDB_CODE_SUCCESS); + } else { + dnodeDispatchNonRspMsg(pVnode, pReadMsg, code); + } } taosFreeQitem(pReadMsg); diff --git a/src/inc/query.h b/src/inc/query.h index d201b649f9de6aa55e0b9f1e0aa8c9aff11a6092..ec1e458b628a60f0db86a48e9da9af7257454ed2 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -28,7 +28,7 @@ typedef void* qinfo_t; * @param qinfo * @return */ -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, void* param, qinfo_t* qinfo); +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo); /** @@ -38,7 +38,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMs * @param qinfo * @return */ -void qTableQuery(qinfo_t qinfo); +bool qTableQuery(qinfo_t qinfo); /** * Retrieve the produced results information, if current query is not paused or completed, @@ -48,7 +48,7 @@ void qTableQuery(qinfo_t qinfo); * @param qinfo * @return */ -int32_t qRetrieveQueryResultInfo(qinfo_t qinfo); +int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext); /** * @@ -60,16 +60,14 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo); * @param contLen payload length * @return */ -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen); +int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); /** - * Decide if more results will be produced or not, NOTE: this function will increase the ref count of QInfo, - * so it can be only called once for each retrieve * * @param qinfo * @return */ -bool qHasMoreResultsToRetrieve(qinfo_t qinfo); +void* qGetResultRetrieveMsg(qinfo_t qinfo); /** * kill current ongoing query and free query handle automatically @@ -89,7 +87,7 @@ void qQueryMgmtNotifyClosed(void* pExecutor); void qCleanupQueryMgmt(void* pExecutor); void** qRegisterQInfo(void* pMgmt, uint64_t qInfo); void** qAcquireQInfo(void* pMgmt, uint64_t key); -void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool needFree); +void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle); #ifdef __cplusplus } diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 1f03f1fe097930cf2fff8b30a0bfbb81515e2fb2..bb3a6ee78e197f1bddbb661cd9d7522b79d54a58 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -216,6 +216,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY, 0, 0x0703, "query out TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR, 0, 0x0704, "query app error") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, 0, 0x0705, "query duplicated join key") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "query tag conditon too many") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "query not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "query should response") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "grant expired") diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 9121f31131bc0cafa85830409bf9d40390da6790..30a292f522cb0fa380a30b2c990b2858f9242adf 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -68,7 +68,7 @@ int32_t mnodeInitProfile() { mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg); mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_CONN, mnodeProcessKillConnectionMsg); - tsMnodeConnCache = taosCacheInit(TSDB_DATA_TYPE_INT, CONN_CHECK_TIME, false, mnodeFreeConn, "conn"); + tsMnodeConnCache = taosCacheInit(TSDB_DATA_TYPE_INT, CONN_CHECK_TIME, true, mnodeFreeConn, "conn"); return 0; } @@ -119,7 +119,7 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po return NULL; } - if (pConn->ip != ip || pConn->port != port /* || strcmp(pConn->user, user) != 0 */) { + if (/* pConn->ip != ip || */ pConn->port != port /* || strcmp(pConn->user, user) != 0 */) { mError("connId:%d, incoming conn user:%s ip:%s:%u, not match exist conn user:%s ip:%s:%u", connId, user, taosIpStr(ip), port, pConn->user, taosIpStr(pConn->ip), pConn->port); taosCacheRelease(tsMnodeConnCache, (void **)&pConn, false); diff --git a/src/os/inc/osDarwin64.h b/src/os/inc/osDarwin64.h index 2bb0777ab81352f0d3f6a29fc8b9b12ea0366793..3ceb7ea8f59849f6fd73782c200f1facff90d62b 100644 --- a/src/os/inc/osDarwin64.h +++ b/src/os/inc/osDarwin64.h @@ -205,7 +205,6 @@ ssize_t tread(int fd, void *buf, size_t count); ssize_t twrite(int fd, void *buf, size_t n); -char *taosCharsetReplace(char *charsetstr); bool taosCheckPthreadValid(pthread_t thread); @@ -219,8 +218,6 @@ int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optle void taosPrintOsInfo(); -char *taosCharsetReplace(char *charsetstr); - void taosPrintOsInfo(); void taosGetSystemInfo(); diff --git a/src/os/inc/osSpec.h b/src/os/inc/osSpec.h index b7b7bd9ff457c0fd8b9f28ca6330b03cd96bf21f..48ec6ae04f189184af1ef84ada3678b60ea48c37 100644 --- a/src/os/inc/osSpec.h +++ b/src/os/inc/osSpec.h @@ -20,6 +20,89 @@ extern "C" { #endif +#define tclose(x) taosCloseSocket(x) +#define tfree(x) \ + do { \ + if (x) { \ + free((void *)(x)); \ + x = 0; \ + } \ + } while (0); + +#define tstrncpy(dst, src, size) \ + do { \ + strncpy((dst), (src), (size)); \ + (dst)[(size)-1] = 0; \ + } while (0); + +#ifndef STDERR_FILENO +#define STDERR_FILENO (2) +#endif + +#define FD_VALID(x) ((x) > STDERR_FILENO) +#define FD_INITIALIZER ((int32_t)-1) + +#define WCHAR wchar_t + +#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) +#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) + +#ifndef NDEBUG +#define ASSERT(x) assert(x) +#else +#define ASSERT(x) +#endif + +#ifdef UNUSED +#undefine UNUSED +#endif +#define UNUSED(x) ((void)(x)) + +#ifdef UNUSED_FUNC +#undefine UNUSED_FUNC +#endif + +#ifdef UNUSED_PARAM +#undef UNUSED_PARAM +#endif + +#if defined(__GNUC__) +#define UNUSED_PARAM(x) _UNUSED##x __attribute__((unused)) +#define UNUSED_FUNC __attribute__((unused)) +#else +#define UNUSED_PARAM(x) x +#define UNUSED_FUNC +#endif + +#ifdef tListLen +#undefine tListLen +#endif +#define tListLen(x) (sizeof(x) / sizeof((x)[0])) + +#if defined(__GNUC__) +#define FORCE_INLINE inline __attribute__((always_inline)) +#else +#define FORCE_INLINE +#endif + +#define DEFAULT_UNICODE_ENCODEC "UCS-4LE" + +#define DEFAULT_COMP(x, y) \ + do { \ + if ((x) == (y)) { \ + return 0; \ + } else { \ + return (x) < (y) ? -1 : 1; \ + } \ + } while (0) + +#define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1))) + +// align to 8bytes +#define ALIGN8(n) ALIGN_NUM(n, 8) + +#define POW2(x) ((x) * (x)) + #ifndef TAOS_OS_FUNC_MATH #define SWAP(a, b, c) \ do { \ @@ -46,6 +129,12 @@ extern "C" { #ifndef TAOS_OS_DEF_TIME #define MILLISECOND_PER_SECOND ((int64_t)1000L) #endif +#define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60) +#define MILLISECOND_PER_HOUR (MILLISECOND_PER_MINUTE * 60) +#define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24) +#define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7) +#define MILLISECOND_PER_MONTH (MILLISECOND_PER_DAY * 30) +#define MILLISECOND_PER_YEAR (MILLISECOND_PER_DAY * 365) #ifndef TAOS_OS_FUNC_SEMPHONE #define tsem_t sem_t @@ -144,24 +233,13 @@ extern "C" { ssize_t taosTReadImp(int fd, void *buf, size_t count); ssize_t taosTWriteImp(int fd, void *buf, size_t count); ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size); -#ifndef TAOS_OS_FUNC_FILE +#ifndef TAOS_OS_FUNC_FILE_OP #define taosTRead(fd, buf, count) taosTReadImp(fd, buf, count) #define taosTWrite(fd, buf, count) taosTWriteImp(fd, buf, count) #define taosLSeek(fd, offset, whence) lseek(fd, offset, whence) #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size) #endif -#ifdef TAOS_RANDOM_FILE_FAIL - void taosSetRandomFileFailFactor(int factor); - void taosSetRandomFileFailOutput(const char *path); - ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); - ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); - off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line); - #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__) - #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__) - #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__) -#endif - #ifndef TAOS_OS_FUNC_NETWORK #define taosSend(sockfd, buf, len, flags) send(sockfd, buf, len, flags) #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) sendto(sockfd, buf, len, flags, dest_addr, addrlen) @@ -176,17 +254,6 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size); } #endif -#ifdef TAOS_RANDOM_NETWORK_FAIL - ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags); - ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen); - ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count); - ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count); - #define taosSend(sockfd, buf, len, flags) taosSendRandomFail(sockfd, buf, len, flags) - #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) taosSendToRandomFail(sockfd, buf, len, flags, dest_addr, addrlen) - #define taosReadSocket(fd, buf, len) taosReadSocketRandomFail(fd, buf, len) - #define taosWriteSocket(fd, buf, len) taosWriteSocketRandomFail(fd, buf, len) -#endif - #ifndef TAOS_OS_FUNC_LZ4 #define BUILDIN_CLZL(val) __builtin_clzl(val) #define BUILDIN_CTZL(val) __builtin_ctzl(val) @@ -194,6 +261,10 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size); #define BUILDIN_CTZ(val) __builtin_ctz(val) #endif +#ifndef TAOS_OS_FUNC_WCHAR + #define twcslen wcslen +#endif + #undef threadlocal #ifdef _ISOC11_SOURCE #define threadlocal _Thread_local @@ -233,6 +304,81 @@ void taosMsleep(int mseconds); int taosInitTimer(void (*callback)(int), int ms); void taosUninitTimer(); +// TAOS_OS_FUNC_RAND +uint32_t taosRand(void); +void taosRandStr(char* str, int32_t size); +uint32_t trand(void); + +// TAOS_OS_FUNC_FILE +void getTmpfilePath(const char *fileNamePrefix, char *dstPath); +int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath); + +// USE_LIBICONV +int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); +bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t *len); +int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes); +bool taosValidateEncodec(const char *encodec); +char * taosCharsetReplace(char *charsetstr); + +// TAOS_OS_FUNC_MALLOC +#define TAOS_ALLOC_MODE_DEFAULT 0 +#define TAOS_ALLOC_MODE_RANDOM_FAIL 1 +#define TAOS_ALLOC_MODE_DETECT_LEAK 2 +void taosSetAllocMode(int mode, const char *path, bool autoDump); +void taosDumpMemoryLeak(); +void * tmalloc(size_t size); +void * tcalloc(size_t nmemb, size_t size); +size_t tsizeof(void *ptr); +void tmemset(void *ptr, int c); +void * trealloc(void *ptr, size_t size); +void tzfree(void *ptr); + +// TAOS_OS_FUNC_DIR +void taosRemoveDir(char *rootDir); +int taosMkDir(const char *pathname, mode_t mode); +void taosMvDir(char* destDir, char *srcDir); + + + +#ifdef TAOS_RANDOM_FILE_FAIL + void taosSetRandomFileFailFactor(int factor); + void taosSetRandomFileFailOutput(const char *path); + ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); + ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); + off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line); + #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__) + #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__) + #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__) +#endif + +#ifdef TAOS_RANDOM_NETWORK_FAIL + ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags); + ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen); + ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count); + ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count); + #define taosSend(sockfd, buf, len, flags) taosSendRandomFail(sockfd, buf, len, flags) + #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) taosSendToRandomFail(sockfd, buf, len, flags, dest_addr, addrlen) + #define taosReadSocket(fd, buf, len) taosReadSocketRandomFail(fd, buf, len) + #define taosWriteSocket(fd, buf, len) taosWriteSocketRandomFail(fd, buf, len) +#endif + +#ifdef TAOS_MEM_CHECK + void * taos_malloc(size_t size, const char *file, uint32_t line); + void * taos_calloc(size_t num, size_t size, const char *file, uint32_t line); + void * taos_realloc(void *ptr, size_t size, const char *file, uint32_t line); + void taos_free(void *ptr, const char *file, uint32_t line); + char * taos_strdup(const char *str, const char *file, uint32_t line); + char * taos_strndup(const char *str, size_t size, const char *file, uint32_t line); + ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line); + #define malloc(size) taos_malloc(size, __FILE__, __LINE__) + #define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__) + #define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__) + #define free(ptr) taos_free(ptr, __FILE__, __LINE__) + #define strdup(str) taos_strdup(str, __FILE__, __LINE__) + #define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__) + #define getline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__) +#endif // TAOS_MEM_CHECK + #ifdef __cplusplus } #endif diff --git a/src/os/inc/osWindows64.h b/src/os/inc/osWindows64.h index 8fd41fb2fdf51dfd070a91b62142d72c5e971d31..b4687afb82fd05999b60bd50ba2eb797e6a1d11c 100644 --- a/src/os/inc/osWindows64.h +++ b/src/os/inc/osWindows64.h @@ -48,6 +48,8 @@ extern "C" { #endif +#define TAOS_OS_FUNC_WCHAR + // for function open in stat.h #define S_IRWXU _S_IREAD #define S_IRWXG _S_IWRITE @@ -359,7 +361,6 @@ int taosSetNonblocking(int sock, int on); int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen); -char *taosCharsetReplace(char *charsetstr); void taosPrintOsInfo(); diff --git a/src/os/src/detail/osCoredump.c b/src/os/src/detail/osCoredump.c index fe97000c7d987487b9a465b8bde331c0683a66b5..1ead88ab9ec42401e3ddb55f563486352e10d66e 100644 --- a/src/os/src/detail/osCoredump.c +++ b/src/os/src/detail/osCoredump.c @@ -18,7 +18,6 @@ #include "tconfig.h" #include "tglobal.h" #include "tulog.h" -#include "tutil.h" #include "tsystem.h" #ifndef TAOS_OS_FUNC_CORE diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c new file mode 100644 index 0000000000000000000000000000000000000000..27555938c2bcd9855def598c9fbded0f6024c5fe --- /dev/null +++ b/src/os/src/detail/osDir.c @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tglobal.h" +#include "tulog.h" + +#ifndef TAOS_OS_FUNC_DIR + +void taosRemoveDir(char *rootDir) { + DIR *dir = opendir(rootDir); + if (dir == NULL) return; + + struct dirent *de = NULL; + while ((de = readdir(dir)) != NULL) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; + + char filename[1024]; + snprintf(filename, 1023, "%s/%s", rootDir, de->d_name); + if (de->d_type & DT_DIR) { + taosRemoveDir(filename); + } else { + (void)remove(filename); + uInfo("file:%s is removed", filename); + } + } + + closedir(dir); + rmdir(rootDir); + + uInfo("dir:%s is removed", rootDir); +} + +int taosMkDir(const char *path, mode_t mode) { + int code = mkdir(path, 0755); + if (code < 0 && errno == EEXIST) code = 0; + return code; +} + +void taosMvDir(char* destDir, char *srcDir) { + if (0 == tsEnableVnodeBak) { + uInfo("vnode backup not enabled"); + return; + } + + char shellCmd[1024+1] = {0}; + + //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir); + (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir); + tSystem(shellCmd); + uInfo("shell cmd:%s is executed", shellCmd); +} + + +#endif \ No newline at end of file diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 897b6c3f034b0a4f79ed606e85dc534d746ee075..5502dfb4a6c20bc593c81892ebb0d863c2de258d 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -15,74 +15,54 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "ttime.h" -ssize_t taosTReadImp(int fd, void *buf, size_t count) { - size_t leftbytes = count; - ssize_t readbytes; - char * tbuf = (char *)buf; +#ifndef TAOS_OS_FUNC_FILE - while (leftbytes > 0) { - readbytes = read(fd, (void *)tbuf, leftbytes); - if (readbytes < 0) { - if (errno == EINTR) { - continue; - } else { - return -1; - } - } else if (readbytes == 0) { - return (ssize_t)(count - leftbytes); - } - - leftbytes -= readbytes; - tbuf += readbytes; +void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { + const char* tdengineTmpFileNamePrefix = "tdengine-"; + + char tmpPath[PATH_MAX]; + char *tmpDir = "/tmp/"; + + strcpy(tmpPath, tmpDir); + strcat(tmpPath, tdengineTmpFileNamePrefix); + if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { + strcat(tmpPath, fileNamePrefix); + strcat(tmpPath, "-%d-%s"); } - - return (ssize_t)count; + + char rand[8] = {0}; + taosRandStr(rand, tListLen(rand) - 1); + snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); } -ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { - size_t leftbytes = size; - ssize_t sentbytes; +// rename file name +int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) { + int32_t ts = taosGetTimestampSec(); - while (leftbytes > 0) { - /* - * TODO : Think to check if file is larger than 1GB - */ - //if (leftbytes > 1000000000) leftbytes = 1000000000; - sentbytes = sendfile(dfd, sfd, offset, leftbytes); - if (sentbytes == -1) { - if (errno == EINTR) { - continue; - } - else { - return -1; - } - } else if (sentbytes == 0) { - return (ssize_t)(size - leftbytes); - } + char fname[PATH_MAX] = {0}; // max file name length must be less than 255 - leftbytes -= sentbytes; - } + char *delimiterPos = strrchr(fullPath, delimiter); + if (delimiterPos == NULL) return -1; - return size; -} + int32_t fileNameLen = 0; + if (suffix) + fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix); + else + fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts); -ssize_t taosTWriteImp(int fd, void *buf, size_t n) { - size_t nleft = n; - ssize_t nwritten = 0; - char *tbuf = (char *)buf; - - while (nleft > 0) { - nwritten = write(fd, (void *)tbuf, nleft); - if (nwritten < 0) { - if (errno == EINTR) { - continue; - } - return -1; - } - nleft -= nwritten; - tbuf += nwritten; + size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1); + if (*dstPath == NULL) { + *dstPath = calloc(1, len + 1); + if (*dstPath == NULL) return -1; } - return n; + strncpy(*dstPath, fullPath, (size_t)(delimiterPos - fullPath + 1)); + strncat(*dstPath, fname, (size_t)fileNameLen); + (*dstPath)[len] = 0; + + return rename(fullPath, *dstPath); } + +#endif \ No newline at end of file diff --git a/src/os/src/detail/osFileOp.c b/src/os/src/detail/osFileOp.c new file mode 100644 index 0000000000000000000000000000000000000000..897b6c3f034b0a4f79ed606e85dc534d746ee075 --- /dev/null +++ b/src/os/src/detail/osFileOp.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +ssize_t taosTReadImp(int fd, void *buf, size_t count) { + size_t leftbytes = count; + ssize_t readbytes; + char * tbuf = (char *)buf; + + while (leftbytes > 0) { + readbytes = read(fd, (void *)tbuf, leftbytes); + if (readbytes < 0) { + if (errno == EINTR) { + continue; + } else { + return -1; + } + } else if (readbytes == 0) { + return (ssize_t)(count - leftbytes); + } + + leftbytes -= readbytes; + tbuf += readbytes; + } + + return (ssize_t)count; +} + +ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { + size_t leftbytes = size; + ssize_t sentbytes; + + while (leftbytes > 0) { + /* + * TODO : Think to check if file is larger than 1GB + */ + //if (leftbytes > 1000000000) leftbytes = 1000000000; + sentbytes = sendfile(dfd, sfd, offset, leftbytes); + if (sentbytes == -1) { + if (errno == EINTR) { + continue; + } + else { + return -1; + } + } else if (sentbytes == 0) { + return (ssize_t)(size - leftbytes); + } + + leftbytes -= sentbytes; + } + + return size; +} + +ssize_t taosTWriteImp(int fd, void *buf, size_t n) { + size_t nleft = n; + ssize_t nwritten = 0; + char *tbuf = (char *)buf; + + while (nleft > 0) { + nwritten = write(fd, (void *)tbuf, nleft); + if (nwritten < 0) { + if (errno == EINTR) { + continue; + } + return -1; + } + nleft -= nwritten; + tbuf += nwritten; + } + + return n; +} diff --git a/src/os/src/detail/osMalloc.c b/src/os/src/detail/osMalloc.c new file mode 100644 index 0000000000000000000000000000000000000000..66377827daff8d748d0d8017539f51b589f9f961 --- /dev/null +++ b/src/os/src/detail/osMalloc.c @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#ifndef TAOS_OS_FUNC_MALLOC + +void *tmalloc(size_t size) { + if (size <= 0) return NULL; + + void *ret = malloc(size + sizeof(size_t)); + if (ret == NULL) return NULL; + + *(size_t *)ret = size; + + return (void *)((char *)ret + sizeof(size_t)); +} + +void *tcalloc(size_t nmemb, size_t size) { + size_t tsize = nmemb * size; + void * ret = tmalloc(tsize); + if (ret == NULL) return NULL; + + tmemset(ret, 0); + return ret; +} + +size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; } + +void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); } + +void * trealloc(void *ptr, size_t size) { + if (ptr == NULL) return tmalloc(size); + + if (size <= tsizeof(ptr)) return ptr; + + void * tptr = (void *)((char *)ptr - sizeof(size_t)); + size_t tsize = size + sizeof(size_t); + tptr = realloc(tptr, tsize); + if (tptr == NULL) return NULL; + + *(size_t *)tptr = size; + + return (void *)((char *)tptr + sizeof(size_t)); +} + +void tzfree(void *ptr) { + if (ptr) { + free((void *)((char *)ptr - sizeof(size_t))); + } +} + +#endif \ No newline at end of file diff --git a/src/util/src/tmem.c b/src/os/src/detail/osMem.c similarity index 99% rename from src/util/src/tmem.c rename to src/os/src/detail/osMem.c index 9c512ad8dc5938626d983d8263271f060abdc78c..93943c6543c9b24ff58b566e12ef8358d61e3a63 100644 --- a/src/util/src/tmem.c +++ b/src/os/src/detail/osMem.c @@ -13,13 +13,10 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "tulog.h" -#define TAOS_MEM_CHECK_IMPL -#include "tutil.h" - - #ifdef TAOS_MEM_CHECK static int allocMode = TAOS_ALLOC_MODE_DEFAULT; diff --git a/src/os/src/detail/osRand.c b/src/os/src/detail/osRand.c new file mode 100644 index 0000000000000000000000000000000000000000..7e5f58563455ce9271a1bac0f41fe722f0896a1f --- /dev/null +++ b/src/os/src/detail/osRand.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#ifndef TAOS_OS_FUNC_RAND + +uint32_t taosRand(void) { return rand(); } + +uint32_t trand(void) { + int fd; + int seed; + + fd = open("/dev/urandom", 0); + if (fd < 0) { + seed = time(0); + } else { + int len = read(fd, &seed, sizeof(seed)); + if (len < 0) { + seed = time(0); + } + close(fd); + } + + return (uint32_t)seed; +} + +#endif + +void taosRandStr(char* str, int32_t size) { + const char* set = "abcdefghijklmnopqrstuvwxyz0123456789-_."; + int32_t len = 39; + + for (int32_t i = 0; i < size; ++i) { + str[i] = set[taosRand() % len]; + } +} \ No newline at end of file diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index ac9b44155506f8f670699d745b15cf9720a2d72f..7a395285af5ac7bc883c4dae0130f3e838e4ab7c 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -18,8 +18,8 @@ #include "tconfig.h" #include "tglobal.h" #include "tulog.h" -#include "tutil.h" #include "tsystem.h" +#include "taosdef.h" #ifndef TAOS_OS_FUNC_SYSINFO diff --git a/src/os/src/detail/osTimer.c b/src/os/src/detail/osTimer.c index 843c29fcb0838c3fd88baae8a1117de8cfd71893..e0a2e903146927402ae9bda22eaf85990e899e2e 100644 --- a/src/os/src/detail/osTimer.c +++ b/src/os/src/detail/osTimer.c @@ -13,13 +13,12 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "taosdef.h" #include "tglobal.h" #include "ttimer.h" #include "tulog.h" -#include "tutil.h" -#include #ifndef TAOS_OS_FUNC_TIMER diff --git a/src/os/src/detail/osWchar.c b/src/os/src/detail/osWchar.c new file mode 100644 index 0000000000000000000000000000000000000000..4e02973e500076780f0a745c10eecb5fe824bb82 --- /dev/null +++ b/src/os/src/detail/osWchar.c @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tglobal.h" +#include "taosdef.h" + +#ifndef TAOS_OS_FUNC_WCHAR + +int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes) { + return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / TSDB_NCHAR_SIZE); +} + +#endif + +#ifdef USE_LIBICONV +#include "iconv.h" + +int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { + iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); + size_t ucs4_input_len = ucs4_max_len; + size_t outLen = ucs4_max_len; + if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) { + iconv_close(cd); + return -1; + } + + iconv_close(cd); + return (int32_t)(ucs4_max_len - outLen); +} + +bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) { + memset(ucs4, 0, ucs4_max_len); + iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); + size_t ucs4_input_len = mbsLength; + size_t outLeft = ucs4_max_len; + if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) { + iconv_close(cd); + return false; + } + + iconv_close(cd); + if (len != NULL) { + *len = ucs4_max_len - outLeft; + } + + return true; +} + +bool taosValidateEncodec(const char *encodec) { + iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC); + if (cd == (iconv_t)(-1)) { + return false; + } + + iconv_close(cd); + return true; +} + +#else + +int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { + mbstate_t state = {0}; + int32_t len = (int32_t)wcsnrtombs(NULL, (const wchar_t **)&ucs4, ucs4_max_len / 4, 0, &state); + if (len < 0) { + return -1; + } + + memset(&state, 0, sizeof(state)); + len = wcsnrtombs(mbs, (const wchar_t **)&ucs4, ucs4_max_len / 4, (size_t)len, &state); + if (len < 0) { + return -1; + } + + return len; +} + +bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) { + memset(ucs4, 0, ucs4_max_len); + mbstate_t state = {0}; + int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state); + *len = retlen; + + return retlen >= 0; +} + +bool taosValidateEncodec(const char *encodec) { + return true; +} + +#endif + +typedef struct CharsetPair { + char *oldCharset; + char *newCharset; +} CharsetPair; + +char *taosCharsetReplace(char *charsetstr) { + CharsetPair charsetRep[] = { + { "utf8", "UTF-8" }, { "936", "CP936" }, + }; + + for (int32_t i = 0; i < tListLen(charsetRep); ++i) { + if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { + return strdup(charsetRep[i].newCharset); + } + } + + return strdup(charsetstr); +} \ No newline at end of file diff --git a/src/os/src/windows64/w64File.c b/src/os/src/windows64/w64File.c new file mode 100644 index 0000000000000000000000000000000000000000..140277f990293f186c198fe752b9620a0f044305 --- /dev/null +++ b/src/os/src/windows64/w64File.c @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { + const char* tdengineTmpFileNamePrefix = "tdengine-"; + char tmpPath[PATH_MAX]; + + char *tmpDir = getenv("tmp"); + if (tmpDir == NULL) { + tmpDir = ""; + } + + strcpy(tmpPath, tmpDir); + strcat(tmpPath, tdengineTmpFileNamePrefix); + if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { + strcat(tmpPath, fileNamePrefix); + strcat(tmpPath, "-%d-%s"); + } + + char rand[8] = {0}; + taosRandStr(rand, tListLen(rand) - 1); + snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); +} diff --git a/src/os/src/windows64/w64Wchar.c b/src/os/src/windows64/w64Wchar.c new file mode 100644 index 0000000000000000000000000000000000000000..d5930d1de9856860af154776f78bd6f1d347f366 --- /dev/null +++ b/src/os/src/windows64/w64Wchar.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tglobal.h" +#include "tulog.h" + +size_t twcslen(const wchar_t *wcs) { + int *wstr = (int *)wcs; + if (NULL == wstr) { + return 0; + } + + size_t n = 0; + while (1) { + if (0 == *wstr++) { + break; + } + n++; + } + + return n; +} + +int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes) { + for (int i = 0; i < bytes; ++i) { + int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i * 4); + int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i * 4); + + if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) { + return f1 - f2; + } else if (f1 == 0 && f2 == 0) { + return 0; + } + + if (f1 != f2) { + return f1 - f2; + } + } + + return 0; + +#if 0 + int32_t ucs4_max_len = bytes + 4; + char *f1_mbs = calloc(bytes, 1); + char *f2_mbs = calloc(bytes, 1); + if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) { + return -1; + } + if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) { + return -1; + } + int32_t ret = strcmp(f1_mbs, f2_mbs); + free(f1_mbs); + free(f2_mbs); + return ret; +#endif +} diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 225977abae698ccfeb12c36562d5588a3f166027..ca65f6560816935ae3e453d97093402b90d166e9 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -58,7 +58,7 @@ static void httpDestroyContext(void *data) { } bool httpInitContexts() { - tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, false, httpDestroyContext, "restc"); + tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, true, httpDestroyContext, "restc"); if (tsHttpServer.contextCache == NULL) { httpError("failed to init context cache"); return false; diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h index 00049b486d765f341a97ce45250ec764f67502db..01b4c16ac1649e090e9acbe267dd83f83f141fe0 100644 --- a/src/query/inc/qAst.h +++ b/src/query/inc/qAst.h @@ -42,7 +42,6 @@ typedef void (*__do_filter_suppl_fn_t)(void *, void *); /** * this structure is used to filter data in tags, so the offset of filtered tag column in tagdata string is required - * */ typedef struct tQueryInfo { uint8_t optr; // expression operator diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 42dfce42a6f893af32c5175bc52a094658c9a3ab..bd2e0a4470a4c067a4f1da1af9642351e6391d9a 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -20,8 +20,8 @@ #include "hash.h" #include "qFill.h" #include "qResultbuf.h" +#include "qSqlparser.h" #include "qTsbuf.h" -#include "qsqlparser.h" #include "query.h" #include "taosdef.h" #include "tarray.h" @@ -43,7 +43,7 @@ typedef struct SSqlGroupbyExpr { typedef struct SPosInfo { int32_t pageId; - int16_t rowId; + int32_t rowId; } SPosInfo; typedef struct SWindowStatus { @@ -177,13 +177,18 @@ typedef struct SQueryRuntimeEnv { SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file } SQueryRuntimeEnv; +enum { + QUERY_RESULT_NOT_READY = 1, + QUERY_RESULT_READY = 2, +}; + typedef struct SQInfo { void* signature; int32_t pointsInterpo; int32_t code; // error code to returned to client - sem_t dataReady; +// sem_t dataReady; + void* tsdb; - void* param; int32_t vgId; STableGroupInfo tableGroupInfo; // table id list < only includes the STable list> STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure @@ -200,8 +205,11 @@ typedef struct SQInfo { */ int32_t tableIndex; int32_t numOfGroupResultPages; - void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; + void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; + pthread_mutex_t lock; // used to synchronize the rsp/query threads + int32_t dataReady; // denote if query result is ready or not + void* rspContext; // response context } SQInfo; #endif // TDENGINE_QUERYEXECUTOR_H diff --git a/src/query/inc/qExtbuffer.h b/src/query/inc/qExtbuffer.h index b57c48933f0878ca86384219b1c7f66446db28eb..36fc0c9820a8338973e718c473e7619da1990ad8 100644 --- a/src/query/inc/qExtbuffer.h +++ b/src/query/inc/qExtbuffer.h @@ -19,7 +19,6 @@ extern "C" { #endif - #include "os.h" #include "taosmsg.h" @@ -28,9 +27,9 @@ extern "C" { #include "tdataformat.h" #include "talgo.h" -#define DEFAULT_PAGE_SIZE (1024L*4) // 16k larger than the SHistoInfo -#define MAX_TMPFILE_PATH_LENGTH PATH_MAX +#define MAX_TMPFILE_PATH_LENGTH PATH_MAX #define INITIAL_ALLOCATION_BUFFER_SIZE 64 +#define DEFAULT_PAGE_SIZE (4096L) // 16k larger than the SHistoInfo typedef enum EXT_BUFFER_FLUSH_MODEL { /* @@ -126,7 +125,7 @@ typedef struct tExtMemBuffer { * @param pModel * @return */ -tExtMemBuffer *createExtMemBuffer(int32_t inMemSize, int32_t elemSize, SColumnModel *pModel); +tExtMemBuffer *createExtMemBuffer(int32_t inMemSize, int32_t elemSize, int32_t pagesize, SColumnModel *pModel); /** * diff --git a/src/query/inc/qResultbuf.h b/src/query/inc/qResultbuf.h index 8c8afb0957c862042e1da99e211ed06e091dee4a..5303251d981f9e12bb92088b65cf6231d6e45f88 100644 --- a/src/query/inc/qResultbuf.h +++ b/src/query/inc/qResultbuf.h @@ -13,50 +13,85 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_VNODEQUERYUTIL_H -#define TDENGINE_VNODEQUERYUTIL_H +#ifndef TDENGINE_QRESULTBUF_H +#define TDENGINE_QRESULTBUF_H #ifdef __cplusplus extern "C" { #endif +#include #include "hash.h" #include "os.h" #include "qExtbuffer.h" +#include "tlockfree.h" typedef struct SArray* SIDList; +typedef struct SPageDiskInfo { + int32_t offset; + int32_t length; +} SPageDiskInfo; + +typedef struct SPageInfo { + SListNode* pn; // point to list node + int32_t pageId; + SPageDiskInfo info; + void* pData; + bool used; // set current page is in used +} SPageInfo; + +typedef struct SFreeListItem { + int32_t offset; + int32_t len; +} SFreeListItem; + +typedef struct SResultBufStatis { + int32_t flushBytes; + int32_t loadBytes; + int32_t getPages; + int32_t releasePages; + int32_t flushPages; +} SResultBufStatis; + typedef struct SDiskbasedResultBuf { int32_t numOfRowsPerPage; int32_t numOfPages; int64_t totalBufSize; - int32_t fd; // data file fd + int64_t fileSize; // disk file size + FILE* file; int32_t allocateId; // allocated page id - int32_t incStep; // minimum allocated pages - void* pBuf; // mmap buffer pointer char* path; // file path int32_t pageSize; // current used page size int32_t inMemPages; // numOfPages that are allocated in memory - SHashObj* idsTable; // id hash table - SIDList list; // for each id, there is a page id list - - void* iBuf; // inmemory buf - void* handle; // for debug purpose + SHashObj* groupSet; // id hash table + SHashObj* all; + SList* lruList; void* emptyDummyIdList; // dummy id list + void* assistBuf; // assistant buffer for compress/decompress data + SArray* pFree; // free area in file + bool comp; // compressed before flushed to disk + int32_t nextPos; // next page flush position + + const void* handle; // for debug purpose + SResultBufStatis statis; } SDiskbasedResultBuf; -#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) +#define DEFAULT_INTERN_BUF_PAGE_SIZE (4096L) #define DEFAULT_INMEM_BUF_PAGES 10 +#define PAGE_INFO_INITIALIZER (SPageDiskInfo){-1, -1} /** * create disk-based result buffer * @param pResultBuf - * @param size * @param rowSize + * @param pagesize + * @param inMemPages + * @param handle * @return */ -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize, int32_t pagesize, - int32_t inMemPages, void* handle); +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize, + int32_t inMemBufSize, const void* handle); /** * @@ -72,7 +107,7 @@ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32 * @param pResultBuf * @return */ -int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf); +size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf); /** * @@ -88,42 +123,52 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); * @param id * @return */ -static FORCE_INLINE tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { - if (id < pResultBuf->inMemPages) { - return (tFilePage*) ((char*) pResultBuf->iBuf + id * pResultBuf->pageSize); - } else { - return (tFilePage*) ((char*) pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize); - } -} +tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id); + +/** + * release the referenced buf pages + * @param pResultBuf + * @param page + */ +void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page); + +/** + * + * @param pResultBuf + * @param pi + */ +void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi); + + /** * get the total buffer size in the format of disk file * @param pResultBuf * @return */ -int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf); +size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf); /** * get the number of groups in the result buffer * @param pResultBuf * @return */ -int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf); +size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf); /** * destroy result buffer * @param pResultBuf */ -void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle); +void destroyResultBuf(SDiskbasedResultBuf* pResultBuf); /** * * @param pList * @return */ -int32_t getLastPageId(SIDList pList); +SPageInfo* getLastPageInfo(SIDList pList); #ifdef __cplusplus } #endif -#endif // TDENGINE_VNODEQUERYUTIL_H +#endif // TDENGINE_QRESULTBUF_H diff --git a/src/query/inc/qsqlparser.h b/src/query/inc/qSqlparser.h similarity index 100% rename from src/query/inc/qsqlparser.h rename to src/query/inc/qSqlparser.h diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 7119cb75fe9f6604e42a96406ff631840477d5f7..ed7c7e8845317ac754758b8553e3f65ba8da4544 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -45,13 +45,14 @@ bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize); -static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult) { +static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult, + tFilePage* page) { assert(pResult != NULL && pRuntimeEnv != NULL); SQuery *pQuery = pRuntimeEnv->pQuery; - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId); - int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery); +// tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery); return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage + pQuery->pSelectExpr[columnIndex].bytes * realRowId; } diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 5ce9121cf1c4f5e88f3b68823431ad61923e70ec..58d2ff260ec79077fc83e1fa4a444041d8e338dc 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -22,7 +22,7 @@ extern "C" { #include "os.h" -#include "../../common/inc/tname.h" +#include "tname.h" #include "taosdef.h" #include "trpc.h" #include "tvariant.h" diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index c2578c15c0536ea222f376e833a8fe11fa269229..44051c7e3b26f8dad68ae7af17eaf9748dbf26ce 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -18,8 +18,8 @@ #include "exception.h" #include "qAst.h" +#include "qSqlparser.h" #include "qSyntaxtreefunction.h" -#include "qsqlparser.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" @@ -878,28 +878,6 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, free(pRightOutput); } -//void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res) { -// if (pExprs == NULL) { -// return; -// } -// -// tExprNode *pLeft = pExprs->_node.pLeft; -// tExprNode *pRight = pExprs->_node.pRight; -// -// // recursive traverse left child branch -// if (pLeft->nodeType == TSQL_NODE_EXPR) { -// tSQLBinaryExprTrv(pLeft, res); -// } else if (pLeft->nodeType == TSQL_NODE_COL) { -// taosArrayPush(res, &pLeft->pSchema->colId); -// } -// -// if (pRight->nodeType == TSQL_NODE_EXPR) { -// tSQLBinaryExprTrv(pRight, res); -// } else if (pRight->nodeType == TSQL_NODE_COL) { -// taosArrayPush(res, &pRight->pSchema->colId); -// } -//} - static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) { tbufWriteUint8(bw, expr->nodeType); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 8afc2fd87a988d32f4311af876bb57e627eecf30..704a383be2da8e054f0a124ff82c05ec975f9e18 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -221,7 +221,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { } static int32_t getGroupResultId(int32_t groupIndex) { - int32_t base = 200000; + int32_t base = 20000000; return base + (groupIndex * 10000); } @@ -478,10 +478,14 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult if (taosArrayGetSize(list) == 0) { pData = getNewDataBuf(pResultBuf, sid, &pageId); } else { - pageId = getLastPageId(list); - pData = getResBufPage(pResultBuf, pageId); + SPageInfo* pi = getLastPageInfo(list); + pData = getResBufPage(pResultBuf, pi->pageId); + pageId = pi->pageId; if (pData->num >= numOfRowsPerPage) { + // release current page first, and prepare the next one + releaseResBufPageInfo(pResultBuf, pi); + pData = getNewDataBuf(pResultBuf, sid, &pageId); if (pData != NULL) { assert(pData->num == 0); // number of elements must be 0 for new allocated buffer @@ -497,6 +501,8 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult if (pWindowRes->pos.pageId == -1) { // not allocated yet, allocate new buffer pWindowRes->pos.pageId = pageId; pWindowRes->pos.rowId = pData->num++; + + assert(pWindowRes->pos.pageId >= 0); } return 0; @@ -1490,8 +1496,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order goto _clean; } - qDebug("QInfo:%p setup runtime env1", GET_QINFO_ADDR(pRuntimeEnv)); - pRuntimeEnv->offset[0] = 0; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; @@ -1536,8 +1540,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order } } - qDebug("QInfo:%p setup runtime env2", GET_QINFO_ADDR(pRuntimeEnv)); - // set the order information for top/bottom query int32_t functionId = pCtx->functionId; @@ -1558,25 +1560,19 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order } } - qDebug("QInfo:%p setup runtime env3", GET_QINFO_ADDR(pRuntimeEnv)); - char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultInfo) * pQuery->numOfOutput; // set the intermediate result output buffer setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf); - qDebug("QInfo:%p setup runtime env4", GET_QINFO_ADDR(pRuntimeEnv)); - // if it is group by normal column, do not set output buffer, the output buffer is pResult if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery) { resetCtxOutputBuf(pRuntimeEnv); } - qDebug("QInfo:%p setup runtime env5", GET_QINFO_ADDR(pRuntimeEnv)); - setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx); - qDebug("QInfo:%p init completed", GET_QINFO_ADDR(pRuntimeEnv)); + qDebug("QInfo:%p init runtime completed", GET_QINFO_ADDR(pRuntimeEnv)); return TSDB_CODE_SUCCESS; _clean: @@ -1615,7 +1611,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { pRuntimeEnv->pFillInfo = taosDestoryFillInfo(pRuntimeEnv->pFillInfo); - destroyResultBuf(pRuntimeEnv->pResultBuf, pQInfo); + destroyResultBuf(pRuntimeEnv->pResultBuf); tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); @@ -2111,9 +2107,6 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, } if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { -#if defined(_DEBUG_VIEW) - qDebug("QInfo:%p block discarded by per-filter", GET_QINFO_ADDR(pRuntimeEnv)); -#endif // current block has been discard due to filter applied pRuntimeEnv->summary.discardBlocks += 1; qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), @@ -2446,6 +2439,8 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (!mergeFlag) { @@ -2458,7 +2453,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes pCtx[i].hasNull = true; pCtx[i].nStartQueryTimestamp = timestamp; - pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes); + pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); // in case of tag column, the tag information should be extracted from input buffer if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) { @@ -2615,14 +2610,16 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; SWindowResult * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos); + tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pos.pageId); - char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1); + char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1); TSKEY leftTimestamp = GET_INT64_VAL(b1); SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; SWindowResult * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos); + tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pos.pageId); - char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2); + char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2); TSKEY rightTimestamp = GET_INT64_VAL(b2); if (leftTimestamp == rightTimestamp) { @@ -2685,35 +2682,26 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { int32_t id = getGroupResultId(pQInfo->groupIndex - 1); SIDList list = getDataBufPagesIdList(pResultBuf, pQInfo->offset + id); - int32_t total = 0; int32_t size = taosArrayGetSize(list); - for (int32_t i = 0; i < size; ++i) { - int32_t* pgId = taosArrayGet(list, i); - tFilePage *pData = getResBufPage(pResultBuf, *pgId); - total += pData->num; - } - - int32_t rows = total; int32_t offset = 0; for (int32_t j = 0; j < size; ++j) { - int32_t* pgId = taosArrayGet(list, j); - tFilePage *pData = getResBufPage(pResultBuf, *pgId); + SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j); + tFilePage *pData = getResBufPage(pResultBuf, pi->pageId); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; char * pDest = pQuery->sdata[i]->data; - - memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, - bytes * pData->num); + memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, bytes * pData->num); } +// rows += pData->num; offset += pData->num; } assert(pQuery->rec.rows == 0); - pQuery->rec.rows += rows; + pQuery->rec.rows += offset; pQInfo->offset += 1; } @@ -2777,7 +2765,6 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { assert(pQInfo->numOfGroupResultPages == 0); return 0; } else if (numOfTables == 1) { // no need to merge results since only one table in each group - } SCompSupporter cs = {pTableList, posList, pQInfo}; @@ -2802,8 +2789,9 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; SWindowResult * pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId); - char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes); + char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); TSKEY ts = GET_INT64_VAL(b); assert(ts == pWindowRes->window.skey); @@ -3517,9 +3505,11 @@ void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult); + pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult, page); int32_t functionId = pQuery->pSelectExpr[i].base.functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { @@ -3542,6 +3532,8 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group + tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; @@ -3550,7 +3542,7 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * continue; } - pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult); + pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult, bufPage); pCtx->currentStage = 0; int32_t functionId = pCtx->functionId; @@ -3713,11 +3705,13 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ pQInfo->groupIndex += 1; } + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i].pos.pageId); + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; char *out = pQuery->sdata[j]->data + numOfResult * size; - char *in = getPosInResultPage(pRuntimeEnv, j, &result[i]); + char *in = getPosInResultPage(pRuntimeEnv, j, &result[i], page); memcpy(out, in + oldOffset * size, size * numOfRowsToCopy); } @@ -4238,10 +4232,10 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo int32_t ps = DEFAULT_PAGE_SIZE; int32_t rowsize = 0; getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); + int32_t TWOMB = 1024*1024*2; if (isSTableQuery && !onlyQueryTags(pRuntimeEnv->pQuery)) { - int32_t numOfPages = getInitialPageNum(pQInfo); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, numOfPages, rowsize, ps, numOfPages, pQInfo); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TWOMB, pQInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4269,8 +4263,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { int32_t numOfResultRows = getInitialPageNum(pQInfo); getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); - - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, numOfResultRows, rowsize, ps, numOfResultRows, pQInfo); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TWOMB, pQInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -5894,16 +5887,11 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, } pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo)); + pQInfo->dataReady = QUERY_RESULT_NOT_READY; + pthread_mutex_init(&pQInfo->lock, NULL); pQuery->pos = -1; pQuery->window = pQueryMsg->window; - - if (sem_init(&pQInfo->dataReady, 0, 0) != 0) { - int32_t code = TAOS_SYSTEM_ERROR(errno); - qError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, tstrerror(code)); - goto _cleanup; - } - colIdCheck(pQuery); qDebug("qmsg:%p QInfo:%p created", pQueryMsg, pQInfo); @@ -5945,7 +5933,7 @@ static bool isValidQInfo(void *param) { return (sig == (uint64_t)pQInfo); } -static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable, void* param) { +static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable) { int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -5965,18 +5953,12 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ pQuery->window.ekey, pQuery->order.order); setQueryStatus(pQuery, QUERY_COMPLETED); pQInfo->tableqinfoGroupInfo.numOfTables = 0; - - sem_post(&pQInfo->dataReady); return TSDB_CODE_SUCCESS; } - pQInfo->param = param; - if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo); setQueryStatus(pQuery, QUERY_COMPLETED); - - sem_post(&pQInfo->dataReady); return TSDB_CODE_SUCCESS; } @@ -6018,7 +6000,6 @@ static void freeQInfo(SQInfo *pQInfo) { tfree(pQuery->sdata[col]); } - sem_destroy(&(pQInfo->dataReady)); teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { @@ -6170,7 +6151,7 @@ typedef struct SQueryMgmt { pthread_mutex_t lock; } SQueryMgmt; -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, void* param, qinfo_t* pQInfo) { +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo) { assert(pQueryMsg != NULL && tsdb != NULL); int32_t code = TSDB_CODE_SUCCESS; @@ -6266,7 +6247,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo goto _over; } - code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery, param); + code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery); _over: free(tagCond); @@ -6306,24 +6287,32 @@ void qDestroyQueryInfo(qinfo_t qHandle) { freeQInfo(pQInfo); } -void qTableQuery(qinfo_t qinfo) { +static void setQueryResultReady(SQInfo* pQInfo) { + pthread_mutex_lock(&pQInfo->lock); + pQInfo->dataReady = QUERY_RESULT_READY; + pthread_mutex_unlock(&pQInfo->lock); +} + +bool qTableQuery(qinfo_t qinfo) { SQInfo *pQInfo = (SQInfo *)qinfo; if (pQInfo == NULL || pQInfo->signature != pQInfo) { qDebug("QInfo:%p has been freed, no need to execute", pQInfo); - return; + return false; } if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); - sem_post(&pQInfo->dataReady); - return; + setQueryResultReady(pQInfo); + return false; } if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { + setQueryStatus(pQInfo->runtimeEnv.pQuery, QUERY_COMPLETED); + setQueryResultReady(pQInfo); + qDebug("QInfo:%p no table exists for query, abort", pQInfo); - sem_post(&pQInfo->dataReady); - return; + return false; } // error occurs, record the error code and return to client @@ -6331,8 +6320,9 @@ void qTableQuery(qinfo_t qinfo) { if (ret != TSDB_CODE_SUCCESS) { pQInfo->code = ret; qDebug("QInfo:%p query abort due to error/cancel occurs, code:%s", pQInfo, tstrerror(pQInfo->code)); - sem_post(&pQInfo->dataReady); - return; + + setQueryResultReady(pQInfo); + return false; } qDebug("QInfo:%p query task is launched", pQInfo); @@ -6357,10 +6347,20 @@ void qTableQuery(qinfo_t qinfo) { pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } - sem_post(&pQInfo->dataReady); + bool buildRes = false; + pthread_mutex_lock(&pQInfo->lock); + pQInfo->dataReady = QUERY_RESULT_READY; + + if (pQInfo->rspContext != NULL) { + buildRes = true; + } + + + pthread_mutex_unlock(&pQInfo->lock); + return buildRes; } -int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) { +int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext) { SQInfo *pQInfo = (SQInfo *)qinfo; if (pQInfo == NULL || !isValidQInfo(pQInfo)) { @@ -6373,11 +6373,21 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) { return pQInfo->code; } - sem_wait(&pQInfo->dataReady); - qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows, - pQInfo->code); + int32_t code = TSDB_CODE_SUCCESS; + pthread_mutex_lock(&pQInfo->lock); + if (pQInfo->dataReady == QUERY_RESULT_READY) { + *buildRes = true; + qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows, + pQInfo->code); + } else { + *buildRes = false; + qDebug("QInfo:%p retrieve req set query return result after paused", pQInfo); + pQInfo->rspContext = pRspContext; + } - return pQInfo->code; + code = pQInfo->code; + pthread_mutex_unlock(&pQInfo->lock); + return code; } bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { @@ -6389,6 +6399,7 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { } SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + bool ret = false; if (Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { ret = false; @@ -6407,7 +6418,7 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { return ret; } -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen) { +int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen, bool* continueExec) { SQInfo *pQInfo = (SQInfo *)qinfo; if (pQInfo == NULL || !isValidQInfo(pQInfo)) { @@ -6417,8 +6428,10 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; size_t size = getResultSize(pQInfo, &pQuery->rec.rows); + size += sizeof(int32_t); size += sizeof(STableIdInfo) * taosArrayGetSize(pQInfo->arrTableIdInfo); + *contLen = size + sizeof(SRetrieveTableRsp); // todo proper handle failed to allocate memory, @@ -6427,6 +6440,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co if (*pRsp == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } + (*pRsp)->numOfRows = htonl(pQuery->rec.rows); int32_t code = pQInfo->code; @@ -6434,8 +6448,8 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co (*pRsp)->offset = htobe64(pQuery->limit.offset); (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); } else { - (*pRsp)->offset = 0; (*pRsp)->useconds = 0; + (*pRsp)->offset = 0; } (*pRsp)->precision = htons(pQuery->precision); @@ -6446,10 +6460,20 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co code = pQInfo->code; } + pQInfo->rspContext = NULL; + pQInfo->dataReady = QUERY_RESULT_NOT_READY; + if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { (*pRsp)->completed = 1; // notify no more result to client } + if (qHasMoreResultsToRetrieve(pQInfo)) { + *continueExec = true; + } else { // failed to dump result, free qhandle immediately + *continueExec = false; + qKillQuery(pQInfo); + } + return code; } @@ -6460,7 +6484,7 @@ int32_t qKillQuery(qinfo_t qinfo) { return TSDB_CODE_QRY_INVALID_QHANDLE; } - sem_post(&pQInfo->dataReady); +// sem_post(&pQInfo->dataReady); setQueryKilled(pQInfo); return TSDB_CODE_SUCCESS; } @@ -6607,6 +6631,13 @@ static void buildTagQueryResult(SQInfo* pQInfo) { setQueryStatus(pQuery, QUERY_COMPLETED); } +void* qGetResultRetrieveMsg(qinfo_t qinfo) { + SQInfo* pQInfo = (SQInfo*) qinfo; + assert(pQInfo != NULL); + + return pQInfo->rspContext; +} + void freeqinfoFn(void *qhandle) { void** handle = qhandle; if (handle == NULL || *handle == NULL) { @@ -6618,19 +6649,21 @@ void freeqinfoFn(void *qhandle) { } void* qOpenQueryMgmt(int32_t vgId) { - const int32_t REFRESH_HANDLE_INTERVAL = 2; // every 2 seconds, refresh handle pool + const int32_t REFRESH_HANDLE_INTERVAL = 30; // every 30 seconds, refresh handle pool char cacheName[128] = {0}; sprintf(cacheName, "qhandle_%d", vgId); - SQueryMgmt* pQueryHandle = calloc(1, sizeof(SQueryMgmt)); + SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt)); + + pQueryMgmt->qinfoPool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName); + pQueryMgmt->closed = false; + pQueryMgmt->vgId = vgId; - pQueryHandle->qinfoPool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName); - pQueryHandle->closed = false; - pthread_mutex_init(&pQueryHandle->lock, NULL); + pthread_mutex_init(&pQueryMgmt->lock, NULL); qDebug("vgId:%d, open querymgmt success", vgId); - return pQueryHandle; + return pQueryMgmt; } static void queryMgmtKillQueryFn(void* handle) { @@ -6670,7 +6703,7 @@ void qCleanupQueryMgmt(void* pQMgmt) { pthread_mutex_destroy(&pQueryMgmt->lock); tfree(pQueryMgmt); - qDebug("vgId:%d querymgmt cleanup completed", vgId); + qDebug("vgId:%d queryMgmt cleanup completed", vgId); } void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { @@ -6716,14 +6749,15 @@ void** qAcquireQInfo(void* pMgmt, uint64_t key) { } } -void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool needFree) { +void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { SQueryMgmt *pQueryMgmt = pMgmt; if (pQueryMgmt->qinfoPool == NULL) { return NULL; } - taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, needFree); + taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); return 0; } + diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index 69c5f0e24fe6361d41953c35fce1380b97d4e752..fb57f71199d8688fa8503790dab5551e7105ca3a 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -28,10 +28,10 @@ /* * SColumnModel is deeply copy */ -tExtMemBuffer* createExtMemBuffer(int32_t inMemSize, int32_t elemSize, SColumnModel *pModel) { +tExtMemBuffer* createExtMemBuffer(int32_t inMemSize, int32_t elemSize, int32_t pagesize, SColumnModel *pModel) { tExtMemBuffer* pMemBuffer = (tExtMemBuffer *)calloc(1, sizeof(tExtMemBuffer)); - pMemBuffer->pageSize = DEFAULT_PAGE_SIZE; + pMemBuffer->pageSize = pagesize; pMemBuffer->inMemCapacity = ALIGN8(inMemSize) / pMemBuffer->pageSize; pMemBuffer->nElemSize = elemSize; diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index ecc11f8f4d76769d0d52b03f44813349aed29adc..1e58dbbe0b75ee5ed2ddc0627b1590e314456b94 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "qsqlparser.h" +#include "qSqlparser.h" #include "queryLog.h" #include "taosdef.h" #include "taosmsg.h" diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index c4490a01e79408ec2bf09527049082f9c05566c8..85e45e46b3eee0beef2babc893234aa0a5a0818f 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -535,7 +535,7 @@ void tMemBucketPut(tMemBucket *pBucket, void *data, int32_t numOfRows) { if (pSeg->pBuffer[slotIdx] == NULL) { pSeg->pBuffer[slotIdx] = createExtMemBuffer(pBucket->numOfTotalPages * pBucket->pageSize, pBucket->nElemSize, - pBucket->pOrderDesc->pColumnModel); + pBucket->pageSize, pBucket->pOrderDesc->pColumnModel); pSeg->pBuffer[slotIdx]->flushModel = SINGLE_APPEND_MODEL; pBucket->pOrderDesc->pColumnModel->capacity = pSeg->pBuffer[slotIdx]->numOfElemsPerPage; } diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index a3d3386a878cf69c9dae7ea089f36c28df57ed4f..93ccad8e27b8ec3fc16af70f3015f429c70a8af2 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -1,221 +1,439 @@ #include "qResultbuf.h" +#include "stddef.h" +#include "tscompression.h" #include "hash.h" #include "qExtbuffer.h" #include "queryLog.h" #include "taoserror.h" -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize, - int32_t pagesize, int32_t inMemPages, void* handle) { +#define GET_DATA_PAYLOAD(_p) ((_p)->pData + POINTER_BYTES) +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize, + int32_t inMemBufSize, const void* handle) { *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf)); + SDiskbasedResultBuf* pResBuf = *pResultBuf; if (pResBuf == NULL) { return TSDB_CODE_COM_OUT_OF_MEMORY; } - pResBuf->pageSize = pagesize; - pResBuf->numOfPages = inMemPages; // all pages are in buffer in the first place - pResBuf->inMemPages = inMemPages; - assert(inMemPages <= numOfPages); - - pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize; + pResBuf->pageSize = pagesize; + pResBuf->numOfPages = 0; // all pages are in buffer in the first place + pResBuf->totalBufSize = 0; + pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit. + pResBuf->allocateId = -1; + pResBuf->comp = true; + pResBuf->file = NULL; + pResBuf->handle = handle; + pResBuf->fileSize = 0; - pResBuf->totalBufSize = pResBuf->numOfPages * pagesize; - pResBuf->incStep = 4; - pResBuf->allocateId = -1; + // at least more than 2 pages must be in memory + assert(inMemBufSize >= pagesize * 2); - pResBuf->iBuf = calloc(pResBuf->inMemPages, pResBuf->pageSize); + pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize; + pResBuf->lruList = tdListNew(POINTER_BYTES); // init id hash table - pResBuf->idsTable = taosHashInit(numOfPages, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); - pResBuf->list = taosArrayInit(numOfPages, POINTER_BYTES); + pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES + pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); char path[PATH_MAX] = {0}; - getTmpfilePath("tsdb_qbuf", path); + getTmpfilePath("qbuf", path); pResBuf->path = strdup(path); - pResBuf->fd = FD_INITIALIZER; - pResBuf->pBuf = NULL; pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); - qDebug("QInfo:%p create resBuf for output, page size:%d, initial pages:%d, %" PRId64 "bytes", handle, - pResBuf->pageSize, pResBuf->numOfPages, pResBuf->totalBufSize); - + qDebug("QInfo:%p create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", handle, pResBuf->pageSize, + pResBuf->inMemPages, pResBuf->path); + return TSDB_CODE_SUCCESS; } -int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); } - -int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } - -#define NUM_OF_PAGES_ON_DISK(_r) ((_r)->numOfPages - (_r)->inMemPages) -#define FILE_SIZE_ON_DISK(_r) (NUM_OF_PAGES_ON_DISK(_r) * (_r)->pageSize) - -static int32_t createDiskResidesBuf(SDiskbasedResultBuf* pResultBuf) { - pResultBuf->fd = open(pResultBuf->path, O_CREAT | O_RDWR | O_TRUNC, 0666); - if (!FD_VALID(pResultBuf->fd)) { +static int32_t createDiskFile(SDiskbasedResultBuf* pResultBuf) { + pResultBuf->file = fopen(pResultBuf->path, "wb+"); + if (pResultBuf->file == NULL) { qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } - assert(pResultBuf->numOfPages == pResultBuf->inMemPages); - pResultBuf->numOfPages += pResultBuf->incStep; + return TSDB_CODE_SUCCESS; +} - int32_t ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize); - if (ret != TSDB_CODE_SUCCESS) { - qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); +static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing + if (!pResultBuf->comp) { + *dst = srcSize; + return data; } - pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0); - if (pResultBuf->pBuf == MAP_FAILED) { - qError("QInfo:%p failed to map temp file: %s. %s", pResultBuf->handle, pResultBuf->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); + *dst = tsCompressString(data, srcSize, 1, pResultBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0); + + memcpy(data, pResultBuf->assistBuf, *dst); + return data; +} + +static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing + if (!pResultBuf->comp) { + *dst = srcSize; + return data; } - pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize; - return TSDB_CODE_SUCCESS; + *dst = tsDecompressString(data, srcSize, 1, pResultBuf->assistBuf, pResultBuf->pageSize, ONE_STAGE_COMP, NULL, 0); + + memcpy(data, pResultBuf->assistBuf, *dst); + return data; } -static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t incNumOfPages) { - assert(pResultBuf->numOfPages * pResultBuf->pageSize == pResultBuf->totalBufSize); - int32_t ret = TSDB_CODE_SUCCESS; +static int32_t allocatePositionInFile(SDiskbasedResultBuf* pResultBuf, size_t size) { + if (pResultBuf->pFree == NULL) { + return pResultBuf->nextPos; + } else { + int32_t offset = -1; + + size_t num = taosArrayGetSize(pResultBuf->pFree); + for(int32_t i = 0; i < num; ++i) { + SFreeListItem* pi = taosArrayGet(pResultBuf->pFree, i); + if (pi->len >= size) { + offset = pi->offset; + pi->offset += size; + pi->len -= size; + + return offset; + } + } + + // no available recycle space, allocate new area in file + return pResultBuf->nextPos; + } +} + +static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { + assert(!pg->used && pg->pData != NULL); + + int32_t size = -1; + char* t = doCompressData(GET_DATA_PAYLOAD(pg), pResultBuf->pageSize, &size, pResultBuf); + + // this page is flushed to disk for the first time + if (pg->info.offset == -1) { + pg->info.offset = allocatePositionInFile(pResultBuf, size); + pResultBuf->nextPos += size; - if (pResultBuf->pBuf == NULL) { - assert(pResultBuf->fd == FD_INITIALIZER); + fseek(pResultBuf->file, pg->info.offset, SEEK_SET); + /*int32_t ret =*/ fwrite(t, 1, size, pResultBuf->file); - if ((ret = createDiskResidesBuf(pResultBuf)) != TSDB_CODE_SUCCESS) { - return ret; + if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { + pResultBuf->fileSize = pg->info.offset + pg->info.length; } } else { - ret = munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf)); - pResultBuf->numOfPages += incNumOfPages; - - /* - * disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may - * be insufficient - */ - ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize); - if (ret != TSDB_CODE_SUCCESS) { - // dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile, - // strerror(errno)); - return TSDB_CODE_QRY_NO_DISKSPACE; + // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing + if (pg->info.length < size) { + // 1. add current space to free list + taosArrayPush(pResultBuf->pFree, &pg->info); + + // 2. allocate new position, and update the info + pg->info.offset = allocatePositionInFile(pResultBuf, size); + pResultBuf->nextPos += size; } - pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize; - pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0); + //3. write to disk. + fseek(pResultBuf->file, pg->info.offset, SEEK_SET); + fwrite(t, size, 1, pResultBuf->file); - if (pResultBuf->pBuf == MAP_FAILED) { - // dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno)); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { + pResultBuf->fileSize = pg->info.offset + pg->info.length; } } - return TSDB_CODE_SUCCESS; + char* ret = pg->pData; + memset(ret, 0, pResultBuf->pageSize); + + pg->pData = NULL; + pg->info.length = size; + + pResultBuf->statis.flushBytes += pg->info.length; + + return ret; } -#define NO_AVAILABLE_PAGES(_b) ((_b)->allocateId == (_b)->numOfPages - 1) +static char* flushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { + int32_t ret = TSDB_CODE_SUCCESS; + assert(pResultBuf->numOfPages * pResultBuf->pageSize == pResultBuf->totalBufSize && pResultBuf->numOfPages >= pResultBuf->inMemPages); -static FORCE_INLINE int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - assert(pResultBuf != NULL); + if (pResultBuf->file == NULL) { + if ((ret = createDiskFile(pResultBuf)) != TSDB_CODE_SUCCESS) { + terrno = ret; + return NULL; + } + } + + return doFlushPageToDisk(pResultBuf, pg); +} + +// load file block data in disk +static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { + int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); + ret = fread(GET_DATA_PAYLOAD(pg), 1, pg->info.length, pResultBuf->file); + if (ret != pg->info.length) { + terrno = errno; + return NULL; + } + + pResultBuf->statis.loadBytes += pg->info.length; - char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t)); + int32_t fullSize = 0; + doDecompressData(GET_DATA_PAYLOAD(pg), pg->info.length, &fullSize, pResultBuf); + + return GET_DATA_PAYLOAD(pg); +} + +#define NO_AVAILABLE_PAGES(_b) ((_b)->numOfPages >= (_b)->inMemPages) + +static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { + assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL); + + SArray* pa = taosArrayInit(1, POINTER_BYTES); + int32_t ret = taosHashPut(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES); + assert(ret == 0); + + return pa; +} + +static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { + SIDList list = NULL; + + char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t)); if (p == NULL) { // it is a new group id - return -1; + list = addNewGroup(pResultBuf, groupId); + } else { + list = (SIDList) (*p); } - int32_t slot = GET_INT32_VAL(p); - assert(slot >= 0 && slot < taosHashGetSize(pResultBuf->idsTable)); + pResultBuf->numOfPages += 1; - return slot; + SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL}; + ppi->info = PAGE_INFO_INITIALIZER; + ppi->pageId = pageId; + ppi->pData = NULL; + ppi->pn = NULL; + ppi->used = true; + + return *(SPageInfo**) taosArrayPush(list, &ppi); } -static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot - taosHashPut(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t), &num, sizeof(int32_t)); +static SListNode* getEldestUnrefedPage(SDiskbasedResultBuf* pResultBuf) { + SListIter iter = {0}; + tdListInitIter(pResultBuf->lruList, &iter, TD_LIST_BACKWARD); + + SListNode* pn = NULL; + while((pn = tdListNext(&iter)) != NULL) { + assert(pn != NULL); + + SPageInfo* pageInfo = *(SPageInfo**) pn->data; + assert(pageInfo->pageId >= 0 && pageInfo->pn == pn); - SArray* pa = taosArrayInit(1, sizeof(int32_t)); - taosArrayPush(pResultBuf->list, &pa); + if (!pageInfo->used) { + break; + } + } - assert(taosArrayGetSize(pResultBuf->list) == taosHashGetSize(pResultBuf->idsTable)); - return num; + return pn; } -static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { - int32_t slot = getGroupIndex(pResultBuf, groupId); - if (slot < 0) { - slot = addNewGroupId(pResultBuf, groupId); +static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) { + char* bufPage = NULL; + SListNode* pn = getEldestUnrefedPage(pResultBuf); + + // all pages are referenced by user, try to allocate new space + if (pn == NULL) { + int32_t prev = pResultBuf->inMemPages; + pResultBuf->inMemPages = pResultBuf->inMemPages * 1.5; + + qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev, + pResultBuf->inMemPages, pResultBuf->pageSize); + } else { + pResultBuf->statis.flushPages += 1; + tdListPopNode(pResultBuf->lruList, pn); + + SPageInfo* d = *(SPageInfo**) pn->data; + assert(d->pn == pn); + + d->pn = NULL; + tfree(pn); + + bufPage = flushPageToDisk(pResultBuf, d); } - SIDList pList = taosArrayGetP(pResultBuf->list, slot); - taosArrayPush(pList, &pageId); + return bufPage; +} + +static void lruListPushFront(SList *pList, SPageInfo* pi) { + tdListPrepend(pList, &pi); + SListNode* front = tdListGetHead(pList); + pi->pn = front; +} + +static void lruListMoveToFront(SList *pList, SPageInfo* pi) { + tdListPopNode(pList, pi->pn); + tdListPrependNode(pList, pi->pn); } tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { + pResultBuf->statis.getPages += 1; + + char* availablePage = NULL; if (NO_AVAILABLE_PAGES(pResultBuf)) { - if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) { - return NULL; - } + availablePage = evicOneDataPage(pResultBuf); } // register new id in this group *pageId = (++pResultBuf->allocateId); - registerPageId(pResultBuf, groupId, *pageId); - // clear memory for the new page - tFilePage* page = getResBufPage(pResultBuf, *pageId); - memset(page, 0, pResultBuf->pageSize); - - return page; + // register page id info + SPageInfo* pi = registerPage(pResultBuf, groupId, *pageId); + + // add to LRU list + assert(listNEles(pResultBuf->lruList) < pResultBuf->inMemPages && pResultBuf->inMemPages > 0); + + lruListPushFront(pResultBuf->lruList, pi); + + // add to hash map + taosHashPut(pResultBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES); + + // allocate buf + if (availablePage == NULL) { + pi->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES); + } else { + pi->pData = availablePage; + } + + pResultBuf->totalBufSize += pResultBuf->pageSize; + + ((void**)pi->pData)[0] = pi; + pi->used = true; + + return GET_DATA_PAYLOAD(pi); +} + +tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { + assert(pResultBuf != NULL && id >= 0); + pResultBuf->statis.getPages += 1; + + SPageInfo** pi = taosHashGet(pResultBuf->all, &id, sizeof(int32_t)); + assert(pi != NULL && *pi != NULL); + + if ((*pi)->pData != NULL) { // it is in memory + // no need to update the LRU list if only one page exists + if (pResultBuf->numOfPages == 1) { + (*pi)->used = true; + return GET_DATA_PAYLOAD(*pi); + } + + SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data); + assert(*pInfo == *pi); + + lruListMoveToFront(pResultBuf->lruList, (*pi)); + (*pi)->used = true; + + return GET_DATA_PAYLOAD(*pi); + + } else { // not in memory + assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->info.length >= 0 && (*pi)->info.offset >= 0); + + char* availablePage = NULL; + if (NO_AVAILABLE_PAGES(pResultBuf)) { + availablePage = evicOneDataPage(pResultBuf); + } + + if (availablePage == NULL) { + (*pi)->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES); + } else { + (*pi)->pData = availablePage; + } + + ((void**)((*pi)->pData))[0] = (*pi); + + lruListPushFront(pResultBuf->lruList, *pi); + loadPageFromDisk(pResultBuf, *pi); + return GET_DATA_PAYLOAD(*pi); + } } -int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } +void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page) { + assert(pResultBuf != NULL && page != NULL); + char* p = (char*) page - POINTER_BYTES; + + SPageInfo* ppi = ((SPageInfo**) p)[0]; + releaseResBufPageInfo(pResultBuf, ppi); +} + +void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) { + assert(pi->pData != NULL && pi->used); + + pi->used = false; + pResultBuf->statis.releasePages += 1; +} + +size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } + +size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); } + +size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - int32_t slot = getGroupIndex(pResultBuf, groupId); - if (slot < 0) { + assert(pResultBuf != NULL); + + char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t)); + if (p == NULL) { // it is a new group id return pResultBuf->emptyDummyIdList; } else { - return taosArrayGetP(pResultBuf->list, slot); + return (SArray*) (*p); } } -void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) { +void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { if (pResultBuf == NULL) { return; } - if (FD_VALID(pResultBuf->fd)) { - qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file created:%s, file size:%d", handle, - pResultBuf->totalBufSize, pResultBuf->path, FILE_SIZE_ON_DISK(pResultBuf)); + if (pResultBuf->file != NULL) { + qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file size:%"PRId64" bytes", + pResultBuf->handle, pResultBuf->totalBufSize, pResultBuf->fileSize); - close(pResultBuf->fd); - munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf)); - pResultBuf->pBuf = NULL; + fclose(pResultBuf->file); } else { - qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", handle, + qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", pResultBuf->handle, pResultBuf->totalBufSize); } unlink(pResultBuf->path); tfree(pResultBuf->path); - size_t size = taosArrayGetSize(pResultBuf->list); - for (int32_t i = 0; i < size; ++i) { - SArray* pa = taosArrayGetP(pResultBuf->list, i); - taosArrayDestroy(pa); + SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet); + while(taosHashIterNext(iter)) { + SArray** p = (SArray**) taosHashIterGet(iter); + size_t n = taosArrayGetSize(*p); + for(int32_t i = 0; i < n; ++i) { + SPageInfo* pi = taosArrayGetP(*p, i); + tfree(pi->pData); + tfree(pi); + } + + taosArrayDestroy(*p); } - taosArrayDestroy(pResultBuf->list); + taosHashDestroyIter(iter); + + tdListFree(pResultBuf->lruList); taosArrayDestroy(pResultBuf->emptyDummyIdList); - taosHashCleanup(pResultBuf->idsTable); + taosHashCleanup(pResultBuf->groupSet); + taosHashCleanup(pResultBuf->all); - tfree(pResultBuf->iBuf); + tfree(pResultBuf->assistBuf); tfree(pResultBuf); } -int32_t getLastPageId(SIDList pList) { +SPageInfo* getLastPageInfo(SIDList pList) { size_t size = taosArrayGetSize(pList); - return *(int32_t*) taosArrayGet(pList, size - 1); + return (SPageInfo*) taosArrayGetP(pList, size - 1); } diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 7b03dd9d941668119b93d3fb19dd1be3af9f3289..c9143b3a5323fa2419e3a543cc3d4d94d404cffd 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -236,11 +236,13 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow if (pWindowRes == NULL) { return; } - + + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId); + for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { SResultInfo *pResultInfo = &pWindowRes->resultInfo[i]; - char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes); + char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; memset(s, 0, size); @@ -277,8 +279,11 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con memcpy(pDst->interResultBuf, pSrc->interResultBuf, pDst->bufLen); // copy the output buffer data from src to dst, the position info keep unchanged - char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst); - char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src); + tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pos.pageId); + char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst, dstpage); + + tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pos.pageId); + char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src, srcpage); size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; memcpy(dstBuf, srcBuf, s); diff --git a/src/query/src/sql.c b/src/query/src/sql.c index ac9952bb97d9c333a5e4209318cf1b26d3fe4b9c..307d5203b34161c024a8dd259d610a7170098438 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -30,7 +30,7 @@ #include #include #include -#include "qsqlparser.h" +#include "qSqlparser.h" #include "tcmdtype.h" #include "tstoken.h" #include "ttokendef.h" diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index 63ed89ab9fe1de2222c6f5ea8ae05b331437f76d..e9611a3232de4a8c442ca3985a2c447b091bc981 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -10,7 +10,7 @@ namespace { // simple test void simpleTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64, 1024, 4, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4096, NULL); int32_t pageId = 0; int32_t groupId = 0; @@ -18,17 +18,144 @@ void simpleTest() { tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); ASSERT_TRUE(pBufPage != NULL); - ASSERT_EQ(getNumOfRowsPerPage(pResultBuf), (16384L - sizeof(int64_t))/64); - ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L); + ASSERT_EQ(getResBufSize(pResultBuf), 1024); SIDList list = getDataBufPagesIdList(pResultBuf, groupId); ASSERT_EQ(taosArrayGetSize(list), 1); ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); - - destroyResultBuf(pResultBuf, NULL); + + releaseResBufPage(pResultBuf, pBufPage); + + tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); + + tFilePage* t = getResBufPage(pResultBuf, pageId); + ASSERT_TRUE(t == pBufPage1); + + tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); + tFilePage* t1 = getResBufPage(pResultBuf, pageId); + ASSERT_TRUE(t1 == pBufPage2); + + tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); + tFilePage* t2 = getResBufPage(pResultBuf, pageId); + ASSERT_TRUE(t2 == pBufPage3); + + tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); + tFilePage* t3 = getResBufPage(pResultBuf, pageId); + ASSERT_TRUE(t3 == pBufPage4); + + tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); + tFilePage* t4 = getResBufPage(pResultBuf, pageId); + ASSERT_TRUE(t4 == pBufPage5); + + destroyResultBuf(pResultBuf); +} + +void writeDownTest() { + SDiskbasedResultBuf* pResultBuf = NULL; + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL); + + int32_t pageId = 0; + int32_t writePageId = 0; + int32_t groupId = 0; + int32_t nx = 12345; + + tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + ASSERT_TRUE(pBufPage != NULL); + + *(int32_t*)(pBufPage->data) = nx; + writePageId = pageId; + releaseResBufPage(pResultBuf, pBufPage); + + tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); + tFilePage* t1 = getResBufPage(pResultBuf, pageId); + ASSERT_TRUE(t1 == pBufPage1); + ASSERT_TRUE(pageId == 1); + + tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); + tFilePage* t2 = getResBufPage(pResultBuf, pageId); + ASSERT_TRUE(t2 == pBufPage2); + ASSERT_TRUE(pageId == 2); + + tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); + tFilePage* t3 = getResBufPage(pResultBuf, pageId); + ASSERT_TRUE(t3 == pBufPage3); + ASSERT_TRUE(pageId == 3); + + tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); + tFilePage* t4 = getResBufPage(pResultBuf, pageId); + ASSERT_TRUE(t4 == pBufPage4); + ASSERT_TRUE(pageId == 4); + releaseResBufPage(pResultBuf, t4); + + // flush the written page to disk, and read it out again + tFilePage* pBufPagex = getResBufPage(pResultBuf, writePageId); + ASSERT_EQ(*(int32_t*)pBufPagex->data, nx); + + SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); + ASSERT_EQ(taosArrayGetSize(pa), 5); + + destroyResultBuf(pResultBuf); +} + +void recyclePageTest() { + SDiskbasedResultBuf* pResultBuf = NULL; + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL); + + int32_t pageId = 0; + int32_t writePageId = 0; + int32_t groupId = 0; + int32_t nx = 12345; + + tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + ASSERT_TRUE(pBufPage != NULL); + releaseResBufPage(pResultBuf, pBufPage); + + tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); + tFilePage* t1 = getResBufPage(pResultBuf, pageId); + ASSERT_TRUE(t1 == pBufPage1); + ASSERT_TRUE(pageId == 1); + + tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); + tFilePage* t2 = getResBufPage(pResultBuf, pageId); + ASSERT_TRUE(t2 == pBufPage2); + ASSERT_TRUE(pageId == 2); + + tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); + tFilePage* t3 = getResBufPage(pResultBuf, pageId); + ASSERT_TRUE(t3 == pBufPage3); + ASSERT_TRUE(pageId == 3); + + tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); + tFilePage* t4 = getResBufPage(pResultBuf, pageId); + ASSERT_TRUE(t4 == pBufPage4); + ASSERT_TRUE(pageId == 4); + releaseResBufPage(pResultBuf, t4); + releaseResBufPage(pResultBuf, t4); + + tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); + tFilePage* t5 = getResBufPage(pResultBuf, pageId); + ASSERT_TRUE(t5 == pBufPage5); + ASSERT_TRUE(pageId == 5); + + // flush the written page to disk, and read it out again + tFilePage* pBufPagex = getResBufPage(pResultBuf, writePageId); + *(int32_t*)(pBufPagex->data) = nx; + writePageId = pageId; // update the data + releaseResBufPage(pResultBuf, pBufPagex); + + tFilePage* pBufPagex1 = getResBufPage(pResultBuf, 1); + + SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); + ASSERT_EQ(taosArrayGetSize(pa), 6); + + destroyResultBuf(pResultBuf); } } // namespace + TEST(testCase, resultBufferTest) { + srand(time(NULL)); simpleTest(); + writeDownTest(); + recyclePageTest(); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index c0f6c82b99728b02d276c1b3328e806212756b67..0e3a657fde547f186623734e9dcf24b9087e9f27 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -30,11 +30,6 @@ #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) -enum { - QUERY_RANGE_LESS_EQUAL = 0, - QUERY_RANGE_GREATER_EQUAL = 1, -}; - enum { TSDB_QUERY_TYPE_ALL = 1, TSDB_QUERY_TYPE_LAST = 2, @@ -210,7 +205,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab if (pQueryHandle->pColumns == NULL) { goto out_of_memory; } - + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {{0}, 0}; @@ -222,29 +217,29 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab taosArrayPush(pQueryHandle->pColumns, &colInfo); pQueryHandle->statis[i].colId = colInfo.info.colId; } - + pQueryHandle->pTableCheckInfo = taosArrayInit(groupList->numOfTables, sizeof(STableCheckInfo)); if (pQueryHandle->pTableCheckInfo == NULL) { goto out_of_memory; } STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); - + for (int32_t i = 0; i < sizeOfGroup; ++i) { SArray* group = *(SArray**) taosArrayGet(groupList->pGroupList, i); - + size_t gsize = taosArrayGetSize(group); assert(gsize > 0); - + for (int32_t j = 0; j < gsize; ++j) { STable* pTable = (STable*) taosArrayGetP(group, j); - + STableCheckInfo info = { .lastKey = pQueryHandle->window.skey, .tableId = pTable->tableId, .pTableObj = pTable, }; - + assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); @@ -280,17 +275,17 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) { assert(pHandle != NULL); - + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) pHandle; - + size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); SArray* res = taosArrayInit(size, POINTER_BYTES); - + for(int32_t i = 0; i < size; ++i) { STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); taosArrayPush(res, &pCheckInfo->pTableObj); } - + return res; } @@ -306,11 +301,11 @@ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) { STable* pTable = pCheckInfo->pTableObj; assert(pTable != NULL); - + if (pCheckInfo->initBuf) { return true; } - + pCheckInfo->initBuf = true; int32_t order = pHandle->order; @@ -318,7 +313,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh if (pHandle->mem == NULL && pHandle->imem == NULL) { return false; } - + assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); // TODO: add uid check @@ -338,17 +333,17 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh if (pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL) { return false; } - + bool memEmpty = (pCheckInfo->iter == NULL) || (pCheckInfo->iter != NULL && !tSkipListIterNext(pCheckInfo->iter)); bool imemEmpty = (pCheckInfo->iiter == NULL) || (pCheckInfo->iiter != NULL && !tSkipListIterNext(pCheckInfo->iiter)); if (memEmpty && imemEmpty) { // buffer is empty return false; } - + if (!memEmpty) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); assert(node != NULL); - + SDataRow row = SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64", tid:%d check data in mem from skey:%" PRId64 ", order:%d, %p", pHandle, @@ -357,11 +352,11 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pHandle->qinfo); } - + if (!imemEmpty) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); assert(node != NULL); - + SDataRow row = SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64", tid:%d check data in imem from skey:%" PRId64 ", order:%d, %p", pHandle, @@ -370,7 +365,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pHandle->qinfo); } - + return true; } @@ -473,7 +468,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); pHandle->cur.fid = -1; - + STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); STable* pTable = pCheckInfo->pTableObj; @@ -491,17 +486,17 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { pCheckInfo->lastKey = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qinfo); - + // all data in mem are checked already. if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_TRAVERSE(pHandle->order)) || (pCheckInfo->lastKey < pHandle->window.ekey && !ASCENDING_TRAVERSE(pHandle->order))) { return false; } - + int32_t step = ASCENDING_TRAVERSE(pHandle->order)? 1:-1; STimeWindow* win = &pHandle->cur.win; pHandle->cur.rows = tsdbReadRowsFromCache(pCheckInfo, pHandle->window.ekey, pHandle->outputCapacity, win, pHandle); - + // update the last key value pCheckInfo->lastKey = win->ekey + step; pHandle->cur.lastKey = win->ekey + step; @@ -510,7 +505,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { if (!ASCENDING_TRAVERSE(pHandle->order)) { SWAP(win->skey, win->ekey, TSKEY); } - + return true; } @@ -519,31 +514,31 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio if (key == TSKEY_INITIAL_VAL) { return INT32_MIN; } - + int64_t fid = (int64_t)(key / (daysPerFile * tsMsPerDay[precision])); // set the starting fileId if (fid < 0L && llabs(fid) > INT32_MAX) { // data value overflow for INT32 fid = INT32_MIN; } - + if (fid > 0L && fid > INT32_MAX) { fid = INT32_MAX; } - + return fid; } static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { int32_t firstSlot = 0; int32_t lastSlot = numOfBlocks - 1; - + int32_t midSlot = firstSlot; - + while (1) { numOfBlocks = lastSlot - firstSlot + 1; midSlot = (firstSlot + (numOfBlocks >> 1)); - + if (numOfBlocks == 1) break; - + if (skey > pBlock[midSlot].keyLast) { if (numOfBlocks == 2) break; if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break; @@ -555,7 +550,7 @@ static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSK break; // got the slot } } - + return midSlot; } @@ -644,7 +639,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo .uid = (_checkInfo)->tableId.uid}) -static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) { +static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { STsdbRepo *pRepo = pQueryHandle->pTsdb; bool blockLoaded = false; int64_t st = taosGetTimestampUs(); @@ -678,8 +673,9 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo int64_t elapsedTime = (taosGetTimestampUs() - st); pQueryHandle->cost.blockLoadTime += elapsedTime; - tsdbDebug("%p load file block into buffer, elapsed time:%"PRId64 " us", pQueryHandle, elapsedTime); + tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64" , rows:%d, elapsed time:%"PRId64 " us, %p", + pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pQueryHandle->qinfo); return blockLoaded; } @@ -692,18 +688,17 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL; cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1); - + if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) { - + if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) { // do not load file block into buffer int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; - cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step, - pQueryHandle->outputCapacity, &cur->win, pQueryHandle); + cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); pQueryHandle->realNumOfRows = cur->rows; // update the last key value @@ -711,13 +706,13 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { SWAP(cur->win.skey, cur->win.ekey, TSKEY); } - + cur->mixBlock = true; cur->blockCompleted = false; return; } - - doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); + + doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot); doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); } else { /* @@ -742,39 +737,41 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock SQueryFilePos* cur = &pQueryHandle->cur; if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - // query ended in current block + // query ended in/started from current block if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { - if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { + if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) { return false; } SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0]; assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows); - + if (pCheckInfo->lastKey > pBlock->keyFirst) { cur->pos = binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); } else { cur->pos = 0; } - + + assert(pCheckInfo->lastKey <= pBlock->keyLast); doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); } else { // the whole block is loaded in to buffer handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); } } else { //desc order, query ended in current block if (pQueryHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) { - if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) { + if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) { return false; } - + SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0]; if (pCheckInfo->lastKey < pBlock->keyLast) { cur->pos = binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); } else { cur->pos = pBlock->numOfRows - 1; } - + + assert(pCheckInfo->lastKey >= pBlock->keyFirst); doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); } else { handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); @@ -790,7 +787,7 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) { TSKEY* keyList; assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - + if (num <= 0) return -1; keyList = (TSKEY*)pValue; @@ -849,13 +846,19 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) { static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { char* pData = NULL; int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; - + SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; TSKEY* tsArray = pCols->cols[0].pData; - + int32_t num = end - start + 1; + assert(num >= 0); + + if (num == 0) { + return numOfRows; + } + int32_t requiredNumOfCols = taosArrayGetSize(pQueryHandle->pColumns); - + //data in buffer has greater timestamp, copy data in file block int32_t i = 0, j = 0; while(i < requiredNumOfCols && j < pCols->numOfCols) { @@ -928,7 +931,7 @@ static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t cap i++; } - + pQueryHandle->cur.win.ekey = tsArray[end]; pQueryHandle->cur.lastKey = tsArray[end] + step; @@ -995,24 +998,99 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, } } +static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols) { + if (numOfRows == 0 || ASCENDING_TRAVERSE(pQueryHandle->order)) { + return; + } + + // if the buffer is not full in case of descending order query, move the data in the front of the buffer + if (numOfRows < pQueryHandle->outputCapacity) { + int32_t emptySize = pQueryHandle->outputCapacity - numOfRows; + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); + } + } +} + +static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, + int32_t numOfExisted, int32_t *start, int32_t *end) { + *start = -1; + + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + int32_t remain = endPos - startPos + 1; + if (remain + numOfExisted > pQueryHandle->outputCapacity) { + *end = (pQueryHandle->outputCapacity - numOfExisted) + startPos - 1; + } else { + *end = endPos; + } + + *start = startPos; + } else { + int32_t remain = (startPos - endPos) + 1; + if (remain + numOfExisted > pQueryHandle->outputCapacity) { + *end = startPos + 1 - (pQueryHandle->outputCapacity - numOfExisted); + } else { + *end = endPos; + } + + *start = *end; + *end = startPos; + } +} + +static void updateInfoAfterMerge(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, int32_t endPos) { + SQueryFilePos* cur = &pQueryHandle->cur; + + pCheckInfo->lastKey = cur->lastKey; + pQueryHandle->realNumOfRows = numOfRows; + cur->rows = numOfRows; + cur->pos = endPos; +} + +static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle) { + SQueryFilePos* cur = &pQueryHandle->cur; + + if (cur->rows > 0) { + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + assert(cur->win.skey >= pQueryHandle->window.skey && cur->win.ekey <= pQueryHandle->window.ekey); + } else { + assert(cur->win.skey >= pQueryHandle->window.ekey && cur->win.ekey <= pQueryHandle->window.skey); + } + + SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, 0); + assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows-1]); + } else { + cur->win = pQueryHandle->window; + + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; + cur->lastKey = pQueryHandle->window.ekey + step; + } +} + // only return the qualified data to client in terms of query time window, data rows in the same block but do not // be included in the query time window will be discarded static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { SQueryFilePos* cur = &pQueryHandle->cur; SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); - + initTableMemIterator(pQueryHandle, pCheckInfo); + SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && + cur->pos >= 0 && cur->pos < pBlock->numOfRows); + + TSKEY* tsArray = pCols->cols[0].pData; // for search the endPos, so the order needs to reverse int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns); + int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); STable* pTable = pCheckInfo->pTableObj; - int32_t endPos = cur->pos; + if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) { endPos = blockInfo.rows - 1; cur->mixBlock = (cur->pos != 0); @@ -1024,48 +1102,36 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order); cur->mixBlock = true; } - + // compared with the data from in-memory buffer, to generate the correct timestamp array list - int32_t pos = cur->pos; - - assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == 0); - TSKEY* tsArray = pCols->cols[0].pData; - int32_t numOfRows = 0; - pQueryHandle->cur.win = TSWINDOW_INITIALIZER; + int32_t pos = cur->pos; + cur->win = TSWINDOW_INITIALIZER; // no data in buffer, load data from file directly if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) { int32_t start = cur->pos; int32_t end = endPos; + if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { - end = cur->pos; - start = endPos; - } - - cur->win.skey = tsArray[start]; - cur->win.ekey = tsArray[end]; - - // todo opt in case of no data in buffer + SWAP(start, end, int32_t); + } + numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end); - - // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < pQueryHandle->outputCapacity) { - int32_t emptySize = pQueryHandle->outputCapacity - numOfRows; - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); - } - } - + // the time window should always be right order: skey <= ekey + cur->win = (STimeWindow) {.skey = tsArray[start], .ekey = tsArray[end]}; + cur->lastKey = tsArray[endPos]; pos += (end - start + 1) * step; - cur->blockCompleted = (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))); - - pCheckInfo->lastKey = cur->lastKey; - pQueryHandle->realNumOfRows = numOfRows; - cur->rows = numOfRows; + + cur->blockCompleted = + (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || + ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))); + + // if the buffer is not full in case of descending order query, move the data in the front of the buffer + moveDataToFront(pQueryHandle, numOfRows, numOfCols); + updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); + doCheckGeneratedBlockRange(pQueryHandle); return; } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; @@ -1111,30 +1177,18 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it moveToNextRowInMem(pCheckInfo); } - - int32_t start = -1; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - int32_t remain = end - pos + 1; - if (remain + numOfRows > pQueryHandle->outputCapacity) { - end = (pQueryHandle->outputCapacity - numOfRows) + pos - 1; - } - start = pos; - } else { - int32_t remain = (pos - end) + 1; - if (remain + numOfRows > pQueryHandle->outputCapacity) { - end = pos + 1 - (pQueryHandle->outputCapacity - numOfRows); - } + int32_t qstart = 0, qend = 0; + getQualifiedRowsPos(pQueryHandle, pos, end, numOfRows, &qstart, &qend); - start = end; - end = pos; - } + numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, qstart, qend); + pos += (qend - qstart + 1) * step; - numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end); - pos += (end - start + 1) * step; + cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[qend]:tsArray[qstart]; + cur->lastKey = cur->win.ekey + step; } } while (numOfRows < pQueryHandle->outputCapacity); - + if (numOfRows < pQueryHandle->outputCapacity) { /** * if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT @@ -1148,54 +1202,29 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->win.skey = tsArray[pos]; } - int32_t start = -1; - int32_t end = -1; - - // all remain data are qualified, but check the remain capacity in the first place. - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - int32_t remain = endPos - pos + 1; - if (remain + numOfRows > pQueryHandle->outputCapacity) { - endPos = (pQueryHandle->outputCapacity - numOfRows) + pos - 1; - } - - start = pos; - end = endPos; - } else { - int32_t remain = pos + 1; - if (remain + numOfRows > pQueryHandle->outputCapacity) { - endPos = pos + 1 - (pQueryHandle->outputCapacity - numOfRows); - } - - start = endPos; - end = pos; - } + int32_t start = -1, end = -1; + getQualifiedRowsPos(pQueryHandle, pos, endPos, numOfRows, &start, &end); numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end); pos += (end - start + 1) * step; + + cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start]; + cur->lastKey = cur->win.ekey + step; } } } - - cur->blockCompleted = (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))); + + cur->blockCompleted = + (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || + ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))); if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { SWAP(cur->win.skey, cur->win.ekey, TSKEY); - - // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (numOfRows < pQueryHandle->outputCapacity) { - int32_t emptySize = pQueryHandle->outputCapacity - numOfRows; - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); - } - } } - - pCheckInfo->lastKey = cur->lastKey; - pQueryHandle->realNumOfRows = numOfRows; - cur->rows = numOfRows; - cur->pos = pos; + + moveDataToFront(pQueryHandle, numOfRows, numOfCols); + updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); + doCheckGeneratedBlockRange(pQueryHandle); tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->win.skey, cur->win.ekey, cur->rows, pQueryHandle->qinfo); @@ -1332,16 +1361,16 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO cleanBlockOrderSupporter(&sup, 0); return TSDB_CODE_TDB_OUT_OF_MEMORY; } - + int32_t cnt = 0; int32_t numOfQualTables = 0; - + for (int32_t j = 0; j < numOfTables; ++j) { STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pQueryHandle->pTableCheckInfo, j); if (pTableCheck->numOfBlocks <= 0) { continue; } - + SCompBlock* pBlock = pTableCheck->pCompInfo->blocks; sup.numOfBlocksPerTable[numOfQualTables] = pTableCheck->numOfBlocks; @@ -1437,37 +1466,39 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex // current file are not overlapped with query time window, ignore remain files if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { - tsdbDebug("%p remain files are not qualified for qrange:%"PRId64"-%"PRId64", ignore, %p", pQueryHandle, pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo) + (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %p", pQueryHandle, + pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo); pQueryHandle->pFileGroup = NULL; + assert(pQueryHandle->numOfBlocks == 0); break; } if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } - - tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks, - numOfTables, pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo); - + + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks, numOfTables, + pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo); + assert(numOfBlocks >= 0); if (numOfBlocks == 0) { continue; } - + // todo return error code to query engine - if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) { + if ((code = createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } - + assert(numOfBlocks >= pQueryHandle->numOfBlocks); if (pQueryHandle->numOfBlocks > 0) { break; } } - + // no data in file anymore - if (pQueryHandle->numOfBlocks <= 0) { + if (pQueryHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) { assert(pQueryHandle->pFileGroup == NULL); } @@ -1476,10 +1507,11 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex *exists = false; return code; } - + + assert(pQueryHandle->pFileGroup != NULL && pQueryHandle->numOfBlocks > 0); cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; cur->fid = pQueryHandle->pFileGroup->fileId; - + STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; *exists = loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo); @@ -1495,7 +1527,7 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists pQueryHandle->locateStart = true; STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); - + tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order); tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); @@ -1504,7 +1536,7 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists // check if current file block is all consumed STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; - + // current block is done, try next if (!cur->mixBlock || cur->blockCompleted) { if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || @@ -1515,10 +1547,10 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists // next block of the same file int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; cur->slot += step; - + cur->mixBlock = false; cur->blockCompleted = false; - + STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; *exists = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo); @@ -1540,10 +1572,10 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { if (hasMoreDataInCache(pQueryHandle)) { return true; } - + pQueryHandle->activeIndex += 1; } - + return false; } @@ -1561,14 +1593,14 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) { pQueryHandle->type = TSDB_QUERY_TYPE_ALL; pQueryHandle->order = TSDB_ORDER_DESC; - + if (!tsdbNextDataBlock(pHandle)) { return false; } - + /*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo(pHandle, &blockInfo); /*SArray *pDataBlock = */tsdbRetrieveDataBlock(pHandle, pQueryHandle->defaultLoadColumn); - + if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) { // data already retrieve, discard other data rows and return int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); @@ -1576,7 +1608,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); memcpy(pCol->pData, pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows-1), pCol->info.bytes); } - + pQueryHandle->cur.win = (STimeWindow){pQueryHandle->window.skey, pQueryHandle->window.skey}; pQueryHandle->window = pQueryHandle->cur.win; pQueryHandle->cur.rows = 1; @@ -1593,7 +1625,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pSecQueryHandle->checkFiles = true; pSecQueryHandle->activeIndex = 0; pSecQueryHandle->outputCapacity = ((STsdbRepo*)pSecQueryHandle->pTsdb)->config.maxRowsPerFileBlock; - + if (tsdbInitReadHelper(&pSecQueryHandle->rhelper, (STsdbRepo*) pSecQueryHandle->pTsdb) != 0) { free(pSecQueryHandle); return false; @@ -1603,24 +1635,24 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { // allocate buffer in order to load data blocks from file int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); - + pSecQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis)); pSecQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {{0}, 0}; SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); - + colInfo.info = pCol->info; colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCol->info.bytes); taosArrayPush(pSecQueryHandle->pColumns, &colInfo); } - + size_t si = taosArrayGetSize(pQueryHandle->pTableCheckInfo); pSecQueryHandle->pTableCheckInfo = taosArrayInit(si, sizeof(STableCheckInfo)); STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb); assert(pMeta != NULL); - + for (int32_t j = 0; j < si; ++j) { STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j); STableCheckInfo info = { @@ -1628,10 +1660,10 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { .tableId = pCheckInfo->tableId, .pTableObj = pCheckInfo->pTableObj, }; - + taosArrayPush(pSecQueryHandle->pTableCheckInfo, &info); } - + tsdbInitDataBlockLoadInfo(&pSecQueryHandle->dataBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pSecQueryHandle->compBlockLoadInfo); pSecQueryHandle->defaultLoadColumn = taosArrayClone(pQueryHandle->defaultLoadColumn); @@ -1641,17 +1673,17 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo); tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); - + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); memcpy(pCol->pData, pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows-1), pCol->info.bytes); - + SColumnInfoData* pCol1 = taosArrayGet(pSecQueryHandle->pColumns, i); assert(pCol->info.colId == pCol1->info.colId); - + memcpy(pCol->pData + pCol->info.bytes, pCol1->pData, pCol1->info.bytes); } - + SColumnInfoData* pTSCol = taosArrayGet(pQueryHandle->pColumns, 0); // it is ascending order @@ -1675,7 +1707,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pQueryHandle->checkFiles = false; return true; } - + if (pQueryHandle->checkFiles) { bool exists = true; int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); @@ -1688,11 +1720,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pQueryHandle->cost.checkForNextTime += elapsedTime; return exists; } - + pQueryHandle->activeIndex = 0; pQueryHandle->checkFiles = false; } - + // TODO: opt by consider the scan order bool ret = doHasDataInBuffer(pQueryHandle); terrno = TSDB_CODE_SUCCESS; @@ -1705,15 +1737,15 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle; assert(!ASCENDING_TRAVERSE(pQueryHandle->order)); - + // starts from the buffer in case of descending timestamp order check data blocks - + // todo consider the query time window, current last_row does not apply the query time window size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - + TSKEY key = TSKEY_INITIAL_VAL; int32_t index = -1; - + for(int32_t i = 0; i < numOfTables; ++i) { STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); if (pCheckInfo->pTableObj->lastKey > key) { @@ -1721,36 +1753,36 @@ void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) { index = i; } } - + if (index == -1) { // todo add failure test cases return; } - + // erase all other elements in array list size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); for (int32_t i = 0; i < size; ++i) { if (i == index) { continue; } - + STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); tSkipListDestroyIter(pTableCheckInfo->iter); - + if (pTableCheckInfo->pDataCols != NULL) { tfree(pTableCheckInfo->pDataCols->buf); } - + tfree(pTableCheckInfo->pDataCols); tfree(pTableCheckInfo->pCompInfo); } - + STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, index); taosArrayClear(pQueryHandle->pTableCheckInfo); - + info.lastKey = key; taosArrayPush(pQueryHandle->pTableCheckInfo, &info); - + // update the query time window according to the chosen last timestamp pQueryHandle->window = (STimeWindow) {key, key}; } @@ -1759,13 +1791,13 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { // filter the queried time stamp in the first place STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; pQueryHandle->order = TSDB_ORDER_DESC; - + assert(pQueryHandle->window.skey == pQueryHandle->window.ekey); - + // starts from the buffer in case of descending timestamp order check data blocks // todo consider the query time window, current last_row does not apply the query time window size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - + int32_t i = 0; while(i < numOfTables) { STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); @@ -1773,21 +1805,21 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) { break; } - + i++; } - + // there are no data in all the tables if (i == numOfTables) { return; } - + STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); taosArrayClear(pQueryHandle->pTableCheckInfo); - + info.lastKey = pQueryHandle->window.skey; taosArrayPush(pQueryHandle->pTableCheckInfo, &info); - + // update the query time window according to the chosen last timestamp pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; } @@ -1811,7 +1843,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, pQueryHandle->window.ekey); - + break; } @@ -1826,24 +1858,24 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int moveToNextRowInMem(pCheckInfo); break; } - + } while(moveToNextRowInMem(pCheckInfo)); assert(numOfRows <= maxRowsToRead); - + // if the buffer is not full in case of descending order query, move the data in the front of the buffer if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { int32_t emptySize = maxRowsToRead - numOfRows; - + for(int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); } } - + int64_t elapsedTime = taosGetTimestampUs() - st; - tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d", pQueryHandle, - elapsedTime, numOfRows, numOfCols); + tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle, + elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo); return numOfRows; } @@ -1852,7 +1884,7 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; SQueryFilePos* cur = &pHandle->cur; STable* pTable = NULL; - + // there are data in file if (pHandle->cur.fid >= 0) { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot]; @@ -1874,13 +1906,13 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p */ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) { STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; - + SQueryFilePos* c = &pHandle->cur; if (c->mixBlock) { *pBlockStatis = NULL; return TSDB_CODE_SUCCESS; } - + STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[c->slot]; assert((c->slot >= 0 && c->slot < pHandle->numOfBlocks) || ((c->slot == pHandle->numOfBlocks) && (c->slot == 0))); @@ -1900,7 +1932,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta for(int32_t i = 0; i < numOfCols; ++i) { pHandle->statis[i].colId = colIds[i]; } - + tsdbGetDataStatis(&pHandle->rhelper, pHandle->statis, numOfCols); // always load the first primary timestamp column data @@ -1949,31 +1981,31 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { } else { SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock); assert(pHandle->realNumOfRows <= binfo.rows); - + // data block has been loaded, todo extract method SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo; - + if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fileId == pHandle->cur.fid && pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) { return pHandle->pColumns; } else { // only load the file block SCompBlock* pBlock = pBlockInfo->compBlock; - doLoadFileDataBlock(pHandle, pBlock, pCheckInfo); - + doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot); + // todo refactor int32_t numOfRows = copyDataFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1); - + // if the buffer is not full in case of descending order query, move the data in the front of the buffer if (!ASCENDING_TRAVERSE(pHandle->order) && numOfRows < pHandle->outputCapacity) { int32_t emptySize = pHandle->outputCapacity - numOfRows; int32_t reqNumOfCols = taosArrayGetSize(pHandle->pColumns); - + for(int32_t i = 0; i < reqNumOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pHandle->pColumns, i); memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); } } - + return pHandle->pColumns; } } @@ -1984,11 +2016,11 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) { SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); while (tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); - + STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); taosArrayPush(list, pTable); } - + tSkipListDestroyIter(iter); return TSDB_CODE_SUCCESS; } @@ -1998,12 +2030,12 @@ static void destroyHelper(void* param) { return; } - + tQueryInfo* pInfo = (tQueryInfo*)param; if (pInfo->optr != TSDB_RELATION_IN) { tfree(pInfo->q); } - + // tVariantDestroy(&(pInfo->q)); free(param); } @@ -2015,7 +2047,7 @@ void filterPrepare(void* expr, void* param) { } pExpr->_node.info = calloc(1, sizeof(tQueryInfo)); - + STSchema* pTSSchema = (STSchema*) param; tQueryInfo* pInfo = pExpr->_node.info; tVariant* pCond = pExpr->_node.pRight->pVal; @@ -2025,7 +2057,7 @@ void filterPrepare(void* expr, void* param) { pInfo->optr = pExpr->_node.optr; pInfo->compare = getComparFunc(pSchema->type, pInfo->optr); pInfo->param = pTSSchema; - + if (pInfo->optr == TSDB_RELATION_IN) { pInfo->q = (char*) pCond->arr; } else { @@ -2038,25 +2070,24 @@ typedef struct STableGroupSupporter { int32_t numOfCols; SColIndex* pCols; STSchema* pTagSchema; -// void* tsdbMeta; } STableGroupSupporter; int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param; STable* pTable1 = *(STable**) p1; STable* pTable2 = *(STable**) p2; - + for (int32_t i = 0; i < pTableGroupSupp->numOfCols; ++i) { SColIndex* pColIndex = &pTableGroupSupp->pCols[i]; int32_t colIndex = pColIndex->colIndex; - + assert(colIndex >= TSDB_TBNAME_COLUMN_INDEX); - + char * f1 = NULL; char * f2 = NULL; int32_t type = 0; int32_t bytes = 0; - + if (colIndex == TSDB_TBNAME_COLUMN_INDEX) { f1 = (char*) TABLE_NAME(pTable1); f2 = (char*) TABLE_NAME(pTable2); @@ -2090,14 +2121,14 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { return ret; } } - + return 0; } void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, STableGroupSupporter* pSupp, __ext_compar_fn_t compareFn) { STable* pTable = taosArrayGetP(pTableList, 0); - + SArray* g = taosArrayInit(16, POINTER_BYTES); taosArrayPush(g, &pTable); tsdbRefTable(pTable); @@ -2105,10 +2136,10 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable for (int32_t i = 1; i < numOfTables; ++i) { STable** prev = taosArrayGet(pTableList, i - 1); STable** p = taosArrayGet(pTableList, i); - + int32_t ret = compareFn(prev, p, pSupp); assert(ret == 0 || ret == -1); - + tsdbRefTable(*p); assert((*p)->type == TSDB_CHILD_TABLE); @@ -2120,20 +2151,20 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable taosArrayPush(g, p); } } - + taosArrayPush(pGroups, &g); } SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols) { assert(pTableList != NULL); SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); - + size_t size = taosArrayGetSize(pTableList); if (size == 0) { tsdbDebug("no qualified tables"); return pTableGroup; } - + if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table SArray* sa = taosArrayInit(size, POINTER_BYTES); for(int32_t i = 0; i < size; ++i) { @@ -2143,7 +2174,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC tsdbRefTable(*pTable); taosArrayPush(sa, pTable); } - + taosArrayPush(pTableGroup, &sa); tsdbDebug("all %zu tables belong to one group", size); } else { @@ -2151,18 +2182,18 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC pSupp->numOfCols = numOfOrderCols; pSupp->pTagSchema = pTagSchema; pSupp->pCols = pCols; - + taosqsort(pTableList->pData, size, POINTER_BYTES, pSupp, tableGroupComparFn); createTableGroupImpl(pTableGroup, pTableList, size, pSupp, tableGroupComparFn); tfree(pSupp); } - + return pTableGroup; } bool indexedNodeFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; - + STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); char* val = NULL; @@ -2172,7 +2203,7 @@ bool indexedNodeFilterFp(const void* pNode, void* param) { } else { val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId); } - + int32_t ret = 0; if (val == NULL) { //the val is possible to be null, so check it out carefully ret = -1; // val is missing in table tags value pairs @@ -2209,7 +2240,7 @@ bool indexedNodeFilterFp(const void* pNode, void* param) { default: assert(false); } - + return true; } @@ -2239,7 +2270,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT goto _error; } - + if (pTable->type != TSDB_SUPER_TABLE) { tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId.tid, pTable->name->data); @@ -2252,7 +2283,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT //NOTE: not add ref count for super table SArray* res = taosArrayInit(8, POINTER_BYTES); STSchema* pTagSchema = tsdbGetTableTagSchema(pTable); - + // no tags and tbname condition, all child tables of this stable are involved if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { int32_t ret = getAllTableList(pTable, res); @@ -2263,7 +2294,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT pGroupInfo->numOfTables = taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); - + tsdbDebug("%p no table name/tag condition, all tables belong to one group, numOfTables:%zu", tsdb, pGroupInfo->numOfTables); taosArrayDestroy(res); @@ -2299,7 +2330,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT } CATCH( code ) { CLEANUP_EXECUTE(); terrno = code; - goto _error; + goto _error; // TODO: more error handling } END_TRY @@ -2335,12 +2366,12 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, STableGroupInfo* p pGroupInfo->numOfTables = 1; pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES); - + SArray* group = taosArrayInit(1, POINTER_BYTES); - + taosArrayPush(group, &pTable); taosArrayPush(pGroupInfo->pGroupList, &group); - + return TSDB_CODE_SUCCESS; _error: @@ -2428,8 +2459,8 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { tsdbDestroyHelper(&pQueryHandle->rhelper); SIOCostSummary* pCost = &pQueryHandle->cost; - tsdbDebug(":io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p", - pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo); + tsdbDebug("%p :io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p", + pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo); tfree(pQueryHandle); } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 1969b40443361ab38ff47234ee2cb1e5a307642f..2521c582d10d1d90aa91ef210892be2283316240 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -12,9 +12,10 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil pthread osdetail m rt lz4) + ADD_SUBDIRECTORY(tests) + FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) IF (ICONV_INCLUDE_EXIST) - ADD_DEFINITIONS(-DUSE_LIBICONV) FIND_PATH(ICONV_LIBRARY_A_EXIST libiconv.a /usr/lib/ /usr/local/lib/ /lib64) FIND_PATH(ICONV_LIBRARY_SO_EXIST libiconv.so /usr/lib/ /usr/local/lib/ /lib64) IF (ICONV_LIBRARY_A_EXIST OR ICONV_LIBRARY_SO_EXIST) @@ -28,9 +29,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) MESSAGE(STATUS "Failed to find iconv, use default encoding method") ENDIF () - ADD_SUBDIRECTORY(tests) ELSEIF (TD_WINDOWS_64) - ADD_DEFINITIONS(-DUSE_LIBICONV) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/iconv) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex) @@ -70,7 +69,6 @@ ELSEIF (TD_WINDOWS_64) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil iconv regex pthread osdetail winmm IPHLPAPI ws2_32 lz4) ELSEIF(TD_DARWIN_64) - ADD_DEFINITIONS(-DUSE_LIBICONV) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil iconv pthread osdetail lz4) diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 5a3545fd8f20169f7dc5a7a57f2994db40337ef2..1e2aeae394a4c49c104fe24de4a4a643884bdc37 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -68,8 +68,6 @@ typedef struct { int64_t refreshTime; STrashElem * pTrash; char* name; -// void * tmrCtrl; -// void * pTimer; SCacheStatis statistics; SHashObj * pHashTable; __cache_free_fn_t freeFp; diff --git a/src/util/inc/tlist.h b/src/util/inc/tlist.h index a4ed9311e24bf5098f9cca8c05edd0998b8d6253..e8380294da47de1c9eb65059263ac820f3d816be 100644 --- a/src/util/inc/tlist.h +++ b/src/util/inc/tlist.h @@ -55,6 +55,8 @@ int tdListPrepend(SList *list, void *data); int tdListAppend(SList *list, void *data); SListNode *tdListPopHead(SList *list); SListNode *tdListPopTail(SList *list); +SListNode *tdListGetHead(SList *list); +SListNode *tsListGetTail(SList *list); SListNode *tdListPopNode(SList *list, SListNode *node); void tdListMove(SList *src, SList *dst); void tdListDiscard(SList *list); diff --git a/src/util/inc/ttime.h b/src/util/inc/ttime.h index 576c9a51f667604c1a8c5b89ddc47858eb0f1cb4..25d1bdb23e853eeefa20976eea23cfb0a4909827 100644 --- a/src/util/inc/ttime.h +++ b/src/util/inc/ttime.h @@ -20,9 +20,8 @@ extern "C" { #endif -#include -#include -#include "tutil.h" +#include "os.h" +#include "taosdef.h" //@return timestamp in second int32_t taosGetTimestampSec(); diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index 4c003eeb2c3f644cab9617f3ef80a41bb367a4ea..e312513ff3f1f2294360ab2fce25310bcd9bf6d7 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -25,148 +25,24 @@ extern "C" { #include "tcrc32c.h" #include "taosdef.h" -#ifndef STDERR_FILENO -#define STDERR_FILENO (2) -#endif - -#define FD_VALID(x) ((x) > STDERR_FILENO) -#define FD_INITIALIZER ((int32_t)-1) - -#define WCHAR wchar_t - -#define tfree(x) \ - do { \ - if (x) { \ - free((void *)(x)); \ - x = 0; \ - } \ - } while(0); - -#define tstrncpy(dst, src, size) \ - do { \ - strncpy((dst), (src), (size)); \ - (dst)[(size)-1] = 0; \ - } while (0); - -#define tclose(x) taosCloseSocket(x) - -// Pointer p drift right by b bytes -#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) -#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) - -#ifndef NDEBUG -#define ASSERT(x) assert(x) -#else -#define ASSERT(x) -#endif - -#ifdef UNUSED -#undefine UNUSED -#endif -#define UNUSED(x) ((void)(x)) - -#ifdef UNUSED_FUNC -#undefine UNUSED_FUNC -#endif - -#ifdef UNUSED_PARAM -#undef UNUSED_PARAM -#endif - -#if defined(__GNUC__) -#define UNUSED_PARAM(x) _UNUSED##x __attribute__((unused)) -#define UNUSED_FUNC __attribute__((unused)) -#else -#define UNUSED_PARAM(x) x -#define UNUSED_FUNC -#endif - -#ifdef tListLen -#undefine tListLen -#endif -#define tListLen(x) (sizeof(x) / sizeof((x)[0])) - -#if defined(__GNUC__) -#define FORCE_INLINE inline __attribute__((always_inline)) -#else -#define FORCE_INLINE -#endif - -#define DEFAULT_UNICODE_ENCODEC "UCS-4LE" - -#define DEFAULT_COMP(x, y) \ - do { \ - if ((x) == (y)) { \ - return 0; \ - } else { \ - return (x) < (y) ? -1 : 1; \ - } \ - } while (0) - -#define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1))) - -// align to 8bytes -#define ALIGN8(n) ALIGN_NUM(n, 8) - -#define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60) -#define MILLISECOND_PER_HOUR (MILLISECOND_PER_MINUTE * 60) -#define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24) -#define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7) -#define MILLISECOND_PER_MONTH (MILLISECOND_PER_DAY * 30) -#define MILLISECOND_PER_YEAR (MILLISECOND_PER_DAY * 365) - -#define POW2(x) ((x) * (x)) - -uint32_t taosRand(void); - -uint32_t trand(void); - -size_t twcslen(const wchar_t *wcs); - int32_t strdequote(char *src); - -size_t strtrim(char *src); - -char *strnchr(char *haystack, char needle, int32_t len, bool skipquote); - -char **strsplit(char *src, const char *delim, int32_t *num); - -char* strtolower(char *dst, const char *src); - -char* strntolower(char *dst, const char *src, int32_t n); - +size_t strtrim(char *src); +char * strnchr(char *haystack, char needle, int32_t len, bool skipquote); +char ** strsplit(char *src, const char *delim, int32_t *num); +char * strtolower(char *dst, const char *src); +char * strntolower(char *dst, const char *src, int32_t n); int64_t strnatoi(char *num, int32_t len); - -//char* strreplace(const char* str, const char* pattern, const char* rep); - -char *strbetween(char *string, char *begin, char *end); - -char *paGetToken(char *src, char **token, int32_t *tokenLen); +char * strbetween(char *string, char *begin, char *end); +char * paGetToken(char *src, char **token, int32_t *tokenLen); int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]); - int32_t taosHexStrToByteArray(char hexstr[], char bytes[]); -int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath); - -/** - * - * @param fileNamePattern - * @param dstPath - */ -void getTmpfilePath(const char *fileNamePattern, char *dstPath); - -bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t* len); - -int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes); - -void taosRandStr(char* str, int32_t size); - -int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); - -bool taosValidateEncodec(const char *encodec); - bool taosGetVersionNumber(char *versionStr, int *versionNubmer); +int taosCheckVersion(char *input_client_version, char *input_server_version, int compared_segments); + +char * taosIpStr(uint32_t ipInt); +uint32_t ip2uint(const char *const ip_addr); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, char *target) { MD5_CTX context; @@ -176,56 +52,6 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, cha memcpy(target, context.digest, TSDB_KEY_LEN); } -int taosCheckVersion(char *input_client_version, char *input_server_version, int compared_segments); - -char *taosIpStr(uint32_t ipInt); - -uint32_t ip2uint(const char *const ip_addr); - -void taosRemoveDir(char *rootDir); -int tmkdir(const char *pathname, mode_t mode); -void taosMvDir(char* destDir, char *srcDir); - -#define TAOS_ALLOC_MODE_DEFAULT 0 -#define TAOS_ALLOC_MODE_RANDOM_FAIL 1 -#define TAOS_ALLOC_MODE_DETECT_LEAK 2 -void taosSetAllocMode(int mode, const char* path, bool autoDump); -void taosDumpMemoryLeak(); - -void * tmalloc(size_t size); -void * tcalloc(size_t nmemb, size_t size); -size_t tsizeof(void *ptr); -void tmemset(void *ptr, int c); -void * trealloc(void *ptr, size_t size); -void tzfree(void *ptr); - -#ifdef TAOS_MEM_CHECK - -void * taos_malloc(size_t size, const char *file, uint32_t line); -void * taos_calloc(size_t num, size_t size, const char *file, uint32_t line); -void * taos_realloc(void *ptr, size_t size, const char *file, uint32_t line); -void taos_free(void *ptr, const char *file, uint32_t line); -char * taos_strdup(const char *str, const char *file, uint32_t line); -char * taos_strndup(const char *str, size_t size, const char *file, uint32_t line); -ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line); - -#ifndef TAOS_MEM_CHECK_IMPL - -#define malloc(size) taos_malloc(size, __FILE__, __LINE__) -#define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__) -#define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__) -#define free(ptr) taos_free(ptr, __FILE__, __LINE__) -#define strdup(str) taos_strdup(str, __FILE__, __LINE__) -#define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__) -#define getline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__) - -#endif // TAOS_MEM_CHECK_IMPL - -#endif // TAOS_MEM_CHECK - - -char *taosCharsetReplace(char *charsetstr); - #ifdef __cplusplus } #endif diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 5dff6286f076408d2fd5d693452eb70fcb3551f7..695529d02f6b51b46cab5af43b69f2b1928c9925 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -14,10 +14,9 @@ */ #include "os.h" - #include "hash.h" #include "tulog.h" -#include "tutil.h" +#include "taosdef.h" static FORCE_INLINE void __wr_lock(void *lock) { if (lock == NULL) { diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 3d2df5f2146b3aa3e46b715d7c761a70239e7b74..dc9128d4a95d210e82aec4c01ef679540c9945d8 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -343,7 +343,7 @@ void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t ke SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); if (ptNode != NULL) { T_REF_INC(*ptNode); - (*ptNode)->expireTime = taosGetTimestampMs() + (*ptNode)->lifespan; + (*ptNode)->expireTime = expireTime; // taosGetTimestampMs() + (*ptNode)->lifespan; } __cache_unlock(pCacheObj); @@ -381,7 +381,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { } void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) { - if (pCacheObj == NULL || data == NULL) return NULL; + if (pCacheObj == NULL || data == NULL || (*data) == NULL) return NULL; size_t offset = offsetof(SCacheDataNode, data); SCacheDataNode *ptNode = (SCacheDataNode *)((char *)(*data) - offset); @@ -419,7 +419,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { // note: extend lifespan before dec ref count bool inTrashCan = pNode->inTrashCan; - if (pCacheObj->extendLifespan && (!inTrashCan)) { + if (pCacheObj->extendLifespan && (!inTrashCan) && (!_remove)) { atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs()); uDebug("cache:%s data:%p extend life time to %"PRId64 " before release", pCacheObj->name, pNode->data, pNode->expireTime); } @@ -457,8 +457,9 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } else { // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. int32_t ref = T_REF_DEC(pNode); - uDebug("cache:%s, key:%p, %p is released, refcnt:%d, in trashcan:%d", pCacheObj->name, pNode->key, pNode->data, ref, - inTrashCan); + + uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trancan:%d", pCacheObj->name, pNode->key, pNode->data, + ref, inTrashCan); } } @@ -572,6 +573,7 @@ void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) { free(pElem); } +// TODO add another lock when scanning trashcan void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { __cache_wr_lock(pCacheObj); @@ -643,6 +645,7 @@ static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_free_fn_t __cache_wr_lock(pCacheObj); while (taosHashIterNext(pIter)) { SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); + if (pNode->expireTime < time && T_REF_VAL_GET(pNode) <= 0) { taosCacheReleaseNode(pCacheObj, pNode); continue; @@ -674,6 +677,7 @@ void* taosCacheTimedRefresh(void *handle) { // check if current cache object will be deleted every 500ms. if (pCacheObj->deleting) { + uDebug("%s refresh threads quit", pCacheObj->name); break; } diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 889d38ff207315ed489f7877e3ee9f566a5cde41..95645882547110cb36fdc16eafe8cc7d1fe09bcf 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -1,6 +1,6 @@ #include "taosdef.h" #include "tcompare.h" -#include +#include "tarray.h" #include "tutil.h" int32_t compareInt32Val(const void *pLeft, const void *pRight) { diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index d561e8ba5f79a5de8faf9feeb8a857442e36d05e..59bc09e87783c25f48c18acc7d170a929ea4b231 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -118,7 +118,7 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { wordfree(&full_path); - int code = tmkdir(option, 0755); + int code = taosMkDir(option, 0755); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("config option:%s, input value:%s, directory not exist, create fail:%s", diff --git a/src/util/src/tlist.c b/src/util/src/tlist.c index f402c2307e68cee6100a28e53dce1935af44fb3f..8c2ad83de117aaf528565a711a2aa3732984e0a9 100644 --- a/src/util/src/tlist.c +++ b/src/util/src/tlist.c @@ -76,6 +76,7 @@ int tdListPrepend(SList *list, void *data) { SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize); if (node == NULL) return -1; + node->next = node->prev = NULL; memcpy((void *)(node->data), data, list->eleSize); tdListPrependNode(list, node); @@ -121,6 +122,22 @@ SListNode *tdListPopTail(SList *list) { return node; } +SListNode *tdListGetHead(SList *list) { + if (list == NULL || list->numOfEles == 0) { + return NULL; + } + + return list->head; +} + +SListNode *tsListGetTail(SList *list) { + if (list == NULL || list->numOfEles == 0) { + return NULL; + } + + return list->tail; +} + SListNode *tdListPopNode(SList *list, SListNode *node) { if (list->head == node) { list->head = node->next; diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 7b73b1e17c54a2278ebd15420eea96bb84bfe547..eaa60d739b10ce90646cd439e3dbe589e06e80bb 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -14,11 +14,6 @@ */ #include "os.h" - -#ifdef USE_LIBICONV -#include "iconv.h" -#endif - #include "tcrc32c.h" #include "tglobal.h" #include "ttime.h" @@ -27,76 +22,6 @@ #include "tulog.h" #include "taoserror.h" - -#ifdef WINDOWS -uint32_t taosRand(void) -{ - return rand(); -} -#else -uint32_t taosRand(void) -{ -/* - int fd; - int seed; - - fd = open("/dev/urandom", 0); - if (fd < 0) { - seed = time(0); - } else { - int len = read(fd, &seed, sizeof(seed)); - if (len < 0) { - seed = time(0); - } - close(fd); - } - - return (uint32_t)seed; -*/ - return rand(); -} - -uint32_t trand(void) -{ - int fd; - int seed; - - fd = open("/dev/urandom", 0); - if (fd < 0) { - seed = time(0); - } else { - int len = read(fd, &seed, sizeof(seed)); - if (len < 0) { - seed = time(0); - } - close(fd); - } - - return (uint32_t)seed; -} -#endif - -size_t twcslen(const wchar_t *wcs) { -#ifdef WINDOWS - int *wstr = (int *)wcs; - if (NULL == wstr) { - return 0; - } - - size_t n = 0; - while (1) { - if (0 == *wstr++) { - break; - } - n++; - } - - return n; -#else - return wcslen(wcs); -#endif -} - int32_t strdequote(char *z) { if (z == NULL) { return 0; @@ -234,6 +159,7 @@ char* strtolower(char *dst, const char *src) { *p = 0; return dst; } + char* strntolower(char *dst, const char *src, int32_t n) { int esc = 0; char quote = 0, *p = dst, c; @@ -343,96 +269,6 @@ int64_t strnatoi(char *num, int32_t len) { return ret; } -#if 0 -FORCE_INLINE size_t getLen(size_t old, size_t size) { - if (old == 1) { - old = 2; - } - - while (old < size) { - old = (old * 1.5); - } - - return old; -} - -static char *ensureSpace(char *dest, size_t *curSize, size_t size) { - if (*curSize < size) { - *curSize = getLen(*curSize, size); - - char *tmp = realloc(dest, *curSize); - if (tmp == NULL) { - free(dest); - return NULL; - } - - return tmp; - } - - return dest; -} - -char *strreplace(const char *str, const char *pattern, const char *rep) { - if (str == NULL || pattern == NULL || rep == NULL) { - return NULL; - } - - const char *s = str; - - size_t oldLen = strlen(str); - size_t newLen = oldLen; - - size_t repLen = strlen(rep); - size_t patternLen = strlen(pattern); - - char *dest = calloc(1, oldLen + 1); - if (dest == NULL) { - return NULL; - } - - if (patternLen == 0) { - return strcpy(dest, str); - } - - int32_t start = 0; - - while (1) { - char *p = strstr(str, pattern); - if (p == NULL) { // remain does not contain pattern - size_t remain = (oldLen - (str - s)); - size_t size = remain + start + 1; - - dest = ensureSpace(dest, &newLen, size); - if (dest == NULL) { - return NULL; - } - - strcpy(dest + start, str); - dest[start + remain] = 0; - break; - } - - size_t len = p - str; - size_t size = start + len + repLen + 1; - - dest = ensureSpace(dest, &newLen, size); - if (dest == NULL) { - return NULL; - } - - memcpy(dest + start, str, len); - - str += (len + patternLen); - start += len; - - memcpy(dest + start, rep, repLen); - start += repLen; - } - - return dest; -} -#endif - char *strbetween(char *string, char *begin, char *end) { char *result = NULL; char *_begin = strstr(string, begin); @@ -491,170 +327,6 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) { return 0; } -// rename file name -int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) { - int32_t ts = taosGetTimestampSec(); - - char fname[PATH_MAX] = {0}; // max file name length must be less than 255 - - char *delimiterPos = strrchr(fullPath, delimiter); - if (delimiterPos == NULL) return -1; - - int32_t fileNameLen = 0; - if (suffix) - fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix); - else - fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts); - - size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1); - if (*dstPath == NULL) { - *dstPath = calloc(1, len + 1); - if (*dstPath == NULL) return -1; - } - - strncpy(*dstPath, fullPath, (size_t)(delimiterPos - fullPath + 1)); - strncat(*dstPath, fname, (size_t)fileNameLen); - (*dstPath)[len] = 0; - - return rename(fullPath, *dstPath); -} - -void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { - const char* tdengineTmpFileNamePrefix = "tdengine-"; - - char tmpPath[PATH_MAX]; - -#ifdef WINDOWS - char *tmpDir = getenv("tmp"); - if (tmpDir == NULL) { - tmpDir = ""; - } -#else - char *tmpDir = "/tmp/"; -#endif - - strcpy(tmpPath, tmpDir); - strcat(tmpPath, tdengineTmpFileNamePrefix); - if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { - strcat(tmpPath, fileNamePrefix); - strcat(tmpPath, "-%d-%s"); - } - - char rand[8] = {0}; - taosRandStr(rand, tListLen(rand) - 1); - snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); -} - -void taosRandStr(char* str, int32_t size) { - const char* set = "abcdefghijklmnopqrstuvwxyz0123456789-_."; - int32_t len = 39; - - for (int32_t i = 0; i < size; ++i) { - str[i] = set[taosRand() % len]; - } -} - -int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes) { -#if defined WINDOWS - for (int i = 0; i < bytes; ++i) { - int32_t f1 = *(int32_t*)((char*)f1_ucs4 + i * 4); - int32_t f2 = *(int32_t*)((char*)f2_ucs4 + i * 4); - - if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) { - return f1 - f2; - } - else if (f1 == 0 && f2 == 0) { - return 0; - } - - if (f1 != f2) { - return f1 - f2; - } - } - return 0; - -#if 0 - int32_t ucs4_max_len = bytes + 4; - char *f1_mbs = calloc(bytes, 1); - char *f2_mbs = calloc(bytes, 1); - if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) { - return -1; - } - if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) { - return -1; - } - int32_t ret = strcmp(f1_mbs, f2_mbs); - free(f1_mbs); - free(f2_mbs); - return ret; -#endif - -#else - return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / TSDB_NCHAR_SIZE); -#endif -} - -int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { -#ifdef USE_LIBICONV - iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); - size_t ucs4_input_len = ucs4_max_len; - size_t outLen = ucs4_max_len; - if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) { - iconv_close(cd); - return -1; - } - iconv_close(cd); - return (int32_t)(ucs4_max_len - outLen); -#else - mbstate_t state = {0}; - int32_t len = (int32_t) wcsnrtombs(NULL, (const wchar_t **) &ucs4, ucs4_max_len / 4, 0, &state); - if (len < 0) { - return -1; - } - memset(&state, 0, sizeof(state)); - len = wcsnrtombs(mbs, (const wchar_t **) &ucs4, ucs4_max_len / 4, (size_t) len, &state); - if (len < 0) { - return -1; - } - return len; -#endif -} - -bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t* len) { - memset(ucs4, 0, ucs4_max_len); -#ifdef USE_LIBICONV - iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); - size_t ucs4_input_len = mbsLength; - size_t outLeft = ucs4_max_len; - if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) { - iconv_close(cd); - return false; - } - - iconv_close(cd); - if (len != NULL) { - *len = ucs4_max_len - outLeft; - } - - return true; -#else - mbstate_t state = {0}; - int32_t len = mbsnrtowcs((wchar_t *) ucs4, (const char **) &mbs, mbsLength, ucs4_max_len / 4, &state); - return len >= 0; -#endif -} - -bool taosValidateEncodec(const char *encodec) { -#ifdef USE_LIBICONV - iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC); - if (cd == (iconv_t)(-1)) { - return false; - } - iconv_close(cd); -#endif - return true; -} - bool taosGetVersionNumber(char *versionStr, int *versionNubmer) { if (versionStr == NULL || versionNubmer == NULL) { return false; @@ -734,112 +406,3 @@ FORCE_INLINE double taos_align_get_double(const char* pBuf) { *(int64_t*)(&dv) = *(int64_t*)pBuf; return dv; } - -typedef struct CharsetPair { - char *oldCharset; - char *newCharset; -} CharsetPair; - -char *taosCharsetReplace(char *charsetstr) { - CharsetPair charsetRep[] = { - { "utf8", "UTF-8" }, { "936", "CP936" }, - }; - - for (int32_t i = 0; i < tListLen(charsetRep); ++i) { - if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { - return strdup(charsetRep[i].newCharset); - } - } - - return strdup(charsetstr); -} - -void *tmalloc(size_t size) { - if (size <= 0) return NULL; - - void *ret = malloc(size + sizeof(size_t)); - if (ret == NULL) return NULL; - - *(size_t *)ret = size; - - return (void *)((char *)ret + sizeof(size_t)); -} - -void *tcalloc(size_t nmemb, size_t size) { - size_t tsize = nmemb * size; - void * ret = tmalloc(tsize); - if (ret == NULL) return NULL; - - tmemset(ret, 0); - return ret; -} - -size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; } - -void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); } - -void * trealloc(void *ptr, size_t size) { - if (ptr == NULL) return tmalloc(size); - - if (size <= tsizeof(ptr)) return ptr; - - void * tptr = (void *)((char *)ptr - sizeof(size_t)); - size_t tsize = size + sizeof(size_t); - tptr = realloc(tptr, tsize); - if (tptr == NULL) return NULL; - - *(size_t *)tptr = size; - - return (void *)((char *)tptr + sizeof(size_t)); -} - -void tzfree(void *ptr) { - if (ptr) { - free((void *)((char *)ptr - sizeof(size_t))); - } -} - -void taosRemoveDir(char *rootDir) { - DIR *dir = opendir(rootDir); - if (dir == NULL) return; - - struct dirent *de = NULL; - while ((de = readdir(dir)) != NULL) { - if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; - - char filename[1024]; - snprintf(filename, 1023, "%s/%s", rootDir, de->d_name); - if (de->d_type & DT_DIR) { - taosRemoveDir(filename); - } else { - (void)remove(filename); - uInfo("file:%s is removed", filename); - } - } - - closedir(dir); - rmdir(rootDir); - - uInfo("dir:%s is removed", rootDir); -} - -int tmkdir(const char *path, mode_t mode) { - int code = mkdir(path, 0755); - if (code < 0 && errno == EEXIST) code = 0; - return code; -} - -void taosMvDir(char* destDir, char *srcDir) { - if (0 == tsEnableVnodeBak) { - uInfo("vnode backup not enabled"); - return; - } - - char shellCmd[1024+1] = {0}; - - //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir); - (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir); - tSystem(shellCmd); - uInfo("shell cmd:%s is executed", shellCmd); -} - diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 973df7c5a10f2ba84a77718602c8ffddd4b14134..066770e1bb9de5d463a1abd2b4078c3d41f9c574 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -71,11 +71,45 @@ static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void *qhandle) { pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; pRead->pCont = qhandle; pRead->contLen = 0; + pRead->rpcMsg.handle = NULL; atomic_add_fetch_32(&pVnode->refCount, 1); taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead); } +static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void* handle, bool* freeHandle) { + bool continueExec = false; + + int32_t code = TSDB_CODE_SUCCESS; + if ((code = qDumpRetrieveResult(handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) { + if (continueExec) { + vDebug("QInfo:%p add to query task queue for exec", handle); + vnodePutItemIntoReadQueue(pVnode, handle); + pRet->qhandle = handle; + *freeHandle = false; + } else { + vDebug("QInfo:%p exec completed", handle); + *freeHandle = true; + } + } else { + pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); + memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); + *freeHandle = true; + } + + return code; +} + +static void vnodeBuildNoResultQueryRsp(SRspRet* pRet) { + pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); + pRet->len = sizeof(SRetrieveTableRsp); + + memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); + SRetrieveTableRsp* pRsp = pRet->rsp; + + pRsp->completed = true; +} + static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { void *pCont = pReadMsg->pCont; int32_t contLen = pReadMsg->contLen; @@ -98,6 +132,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void*) killQueryMsg->qhandle, pReadMsg->rpcMsg.handle); } else { assert(*qhandle == (void*) killQueryMsg->qhandle); + qKillQuery(*qhandle); qReleaseQInfo(pVnode->qMgmt, (void**) &qhandle, true); } @@ -110,7 +145,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (contLen != 0) { qinfo_t pQInfo = NULL; - code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, pVnode, &pQInfo); + code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo); SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp)); pRsp->code = code; @@ -133,7 +168,6 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pRsp->qhandle = htobe64((uint64_t) pQInfo); } - pQInfo = NULL; if (handle != NULL && vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle); pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; @@ -153,16 +187,34 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } else { assert(pCont != NULL); + handle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) pCont); if (handle == NULL) { vWarn("QInfo:%p invalid qhandle in continuing exec query, conn:%p", (void*) pCont, pReadMsg->rpcMsg.handle); code = TSDB_CODE_QRY_INVALID_QHANDLE; } else { vDebug("vgId:%d, QInfo:%p, dnode continue exec query", pVnode->vgId, (void*) pCont); - code = TSDB_CODE_VND_ACTION_IN_PROGRESS; - qTableQuery(*handle); // do execute query + + bool freehandle = false; + bool buildRes = qTableQuery(*handle); // do execute query + + // build query rsp + if (buildRes) { + // update the connection info according to the retrieve connection + pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*handle); + assert(pReadMsg->rpcMsg.handle != NULL); + + vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle); + code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, *handle, &freehandle); + + // todo test the error code case + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_QRY_HAS_RSP; + } + } + + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freehandle); } - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); } return code; @@ -176,7 +228,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pRetrieve->qhandle = htobe64(pRetrieve->qhandle); pRetrieve->free = htons(pRetrieve->free); - vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, (void*) pRetrieve->qhandle); + vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void*) pRetrieve->qhandle, pRetrieve->free, pReadMsg->rpcMsg.handle); memset(pRet, 0, sizeof(SRspRet)); @@ -185,16 +237,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (handle == NULL || (*handle) != (void*) pRetrieve->qhandle) { code = TSDB_CODE_QRY_INVALID_QHANDLE; vDebug("vgId:%d, invalid qhandle in fetch result, QInfo:%p", pVnode->vgId, (void*) pRetrieve->qhandle); - - pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); - pRet->len = sizeof(SRetrieveTableRsp); - - memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); - SRetrieveTableRsp* pRsp = pRet->rsp; - pRsp->numOfRows = 0; - pRsp->useconds = 0; - pRsp->completed = true; - + + vnodeBuildNoResultQueryRsp(pRet); return code; } @@ -203,35 +247,25 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); - pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); - pRet->len = sizeof(SRetrieveTableRsp); - - memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); - SRetrieveTableRsp* pRsp = pRet->rsp; - pRsp->numOfRows = 0; - pRsp->completed = true; - pRsp->useconds = 0; - + vnodeBuildNoResultQueryRsp(pRet); return code; } bool freeHandle = true; - code = qRetrieveQueryResultInfo(*handle); + bool buildRes = false; + + code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle); if (code != TSDB_CODE_SUCCESS) { //TODO handle malloc failure pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); - } else { // if failed to dump result, free qhandle immediately - if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len)) == TSDB_CODE_SUCCESS) { - if (qHasMoreResultsToRetrieve(*handle)) { - vnodePutItemIntoReadQueue(pVnode, *handle); - pRet->qhandle = *handle; - freeHandle = false; - } else { - qKillQuery(*handle); - freeHandle = true; - } + } else { // result is not ready, return immediately + if (!buildRes) { + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); + return TSDB_CODE_QRY_NOT_READY; } + + code = vnodeDumpQueryResult(pRet, pVnode, *handle, &freeHandle); } qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle); diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c index 43869da477aa4160e457d917b71952b665c34c0b..bf3b0f63ea85be6a491bafdfc6f8d9928f3e6ccd 100644 --- a/src/wal/src/walMain.c +++ b/src/wal/src/walMain.c @@ -101,7 +101,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) { } } - if (tmkdir(path, 0755) != 0) { + if (taosMkDir(path, 0755) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("wal:%s, failed to create directory(%s)", path, strerror(errno)); walRelease(pWal); @@ -420,7 +420,7 @@ int walHandleExistingFiles(const char *path) { if ( strncmp(ent->d_name, walPrefix, plen) == 0) { snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name); snprintf(nname, sizeof(nname), "%s/old/%s", path, ent->d_name); - if (tmkdir(opath, 0755) != 0) { + if (taosMkDir(opath, 0755) != 0) { wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); break; diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index c6981d29022dca56198d6655379472c905984285..1e73893793a9f011f6753e5745226e664ca253f2 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -93,6 +93,8 @@ run general/parser/groupby.sim sleep 2000 run general/parser/tags_filter.sim sleep 2000 +run general/parser/topbot.sim +sleep 2000 run general/parser/union.sim sleep 2000 run general/parser/sliding.sim diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim new file mode 100644 index 0000000000000000000000000000000000000000..a0c46dbc65bac39b33cd54f0c61ac3ba96bf5736 --- /dev/null +++ b/tests/script/general/parser/topbot.sim @@ -0,0 +1,74 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/exec.sh -n dnode1 -s start +sleep 3000 +sql connect + +$dbPrefix = tb_db +$tbPrefix = tb_tb +$stbPrefix = tb_stb +$tbNum = 10 +$rowNum = 1000 +$totalNum = $tbNum * $rowNum +$loops = 200000 +$log = 10000 +$ts0 = 1537146000000 +$delta = 600000 +print ========== topbot.sim +$i = 0 +$db = $dbPrefix . $i +$stb = $stbPrefix . $i + +sql drop database $db -x step1 +step1: +sql create database $db cache 16 maxtables 200 +print ====== create tables +sql use $db +sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int) + +$i = 0 +$ts = $ts0 +$halfNum = $tbNum / 2 +while $i < $halfNum + $tbId = $i + $halfNum + $tb = $tbPrefix . $i + $tb1 = $tbPrefix . $tbId + sql create table $tb using $stb tags( $i ) + sql create table $tb1 using $stb tags( $tbId ) + + $x = 0 + while $x < $rowNum + $xs = $x * $delta + $ts = $ts0 + $xs + $c = $x / 10 + $c = $c * 10 + $c = $x - $c + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) + sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) + $x = $x + 1 + endw + + $i = $i + 1 +endw +print ====== tables created + +sql use $db +##### select from table +print ====== select top/bot from table and check num of rows returned +sql select top(c1, 100) from tb_stb0 +if $row != 100 then + return -1 +endi + +sql select last(c2) from tb_tb9 +if $row != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file