diff --git a/docs/en/14-reference/03-connector/java.mdx b/docs/en/14-reference/03-connector/java.mdx index ff15acf1a9c5dbfd74e6f3101459cfc7bdeda515..310e0a15c61e0d1533332d744782f66085d4e5bb 100644 --- a/docs/en/14-reference/03-connector/java.mdx +++ b/docs/en/14-reference/03-connector/java.mdx @@ -202,6 +202,10 @@ The configuration parameters in the URL are as follows. - batchfetch: true: pull the result set in batch when executing the query; false: pull the result set row by row. The default value is false. batchfetch uses HTTP for data transfer. The JDBC REST connection supports bulk data pulling function in taos-jdbcdriver-2.0.38 and TDengine 2.4.0.12 and later versions. taos-jdbcdriver and TDengine transfer data via WebSocket connection. Compared with HTTP, WebSocket enables JDBC REST connection to support large data volume querying and improve query performance. - charset: specify the charset to parse the string, this parameter is valid only when set batchfetch to true. - batchErrorIgnore: true: when executing executeBatch of Statement, if one SQL execution fails in the middle, continue to execute the following SQL. false: no longer execute any statement after the failed SQL. The default value is: false. +- httpConnectTimeout: REST connection timeout in milliseconds, the default value is 5000 ms. +- httpSocketTimeout: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when batchfetch is false. +- messageWaitTimeout: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when batchfetch is true. +- useSSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection. **Note**: Some configuration items (e.g., locale, timezone) do not work in the REST connection. @@ -257,14 +261,18 @@ In the above example, a connection is established to `taosdemo.com`, port is 603 The configuration parameters in properties are as follows. -- TSDBDriver.PROPERTY_KEY_USER: Login TDengine user name, default value 'root'. +- TSDBDriver.PROPERTY_KEY_USER: login TDengine user name, default value 'root'. - TSDBDriver.PROPERTY_KEY_PASSWORD: user login password, default value 'taosdata'. - TSDBDriver.PROPERTY_KEY_BATCH_LOAD: true: pull the result set in batch when executing query; false: pull the result set row by row. The default value is: false. - TSDBDriver.PROPERTY_KEY_BATCH_ERROR_IGNORE: true: when executing executeBatch of Statement, if there is a SQL execution failure in the middle, continue to execute the following sq. false: no longer execute any statement after the failed SQL. The default value is: false. -- TSDBDriver.PROPERTY_KEY_CONFIG_DIR: Only works when using JDBC native connection. Client configuration file directory path, default value `/etc/taos` on Linux OS, default value `C:/TDengine/cfg` on Windows OS. +- TSDBDriver.PROPERTY_KEY_CONFIG_DIR: only works when using JDBC native connection. Client configuration file directory path, default value `/etc/taos` on Linux OS, default value `C:/TDengine/cfg` on Windows OS. - TSDBDriver.PROPERTY_KEY_CHARSET: In the character set used by the client, the default value is the system character set. - TSDBDriver.PROPERTY_KEY_LOCALE: this only takes effect when using JDBC native connection. Client language environment, the default value is system current locale. - TSDBDriver.PROPERTY_KEY_TIME_ZONE: only takes effect when using JDBC native connection. In the time zone used by the client, the default value is the system's current time zone. +- TSDBDriver.HTTP_CONNECT_TIMEOUT: REST connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection. +- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket timeout in milliseconds, the default value is 5000 ms. It only takes effect when using JDBC REST connection and batchfetch is false. +- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 3000 ms. It only takes effect when using JDBC REST connection and batchfetch is true. +- TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL conneciton, false: not using SSL connection. It only takes effect when using using JDBC REST connection. For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](/reference/config/#Client-Only). ### Priority of configuration parameters @@ -812,11 +820,12 @@ Please refer to: [JDBC example](https://github.com/taosdata/TDengine/tree/develo ## Recent update logs -| taos-jdbcdriver version | major changes | -| :---------------------: | :------------------------------------------: | -| 2.0.38 | JDBC REST connections add bulk pull function | -| 2.0.37 | Added support for json tags | -| 2.0.36 | Add support for schemaless writing | +| taos-jdbcdriver version | major changes | +| :---------------------: | :--------------------------------------------: | +| 2.0.39 - 2.0.40 | Add REST connection/request timeout parameters | +| 2.0.38 | JDBC REST connections add bulk pull function | +| 2.0.37 | Support json tags | +| 2.0.36 | Support schemaless writing | ## Frequently Asked Questions diff --git a/docs/zh/14-reference/03-connector/java.mdx b/docs/zh/14-reference/03-connector/java.mdx index ddab9e5f24c64e51e82cad6e299f3ea0d741b349..838fa2eff8cec8d587d4c387645b22586142303a 100644 --- a/docs/zh/14-reference/03-connector/java.mdx +++ b/docs/zh/14-reference/03-connector/java.mdx @@ -201,6 +201,10 @@ url 中的配置参数如下： - batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。从 taos-jdbcdriver-2.0.38 和 TDengine 2.4.0.12 版本开始，JDBC REST 连接增加批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 - charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 +- httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。 +- httpSocketTimeout: socket 超时时间，单位 ms，默认值为 5000。仅在 batchfetch 设置为 false 时生效。 +- messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。 +- useSSL: 连接中是否使用 SSL。 **注意**：部分配置项（比如：locale、timezone）在 REST 连接中不生效。 @@ -264,7 +268,11 @@ properties 中的配置参数如下： - TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。 - TSDBDriver.PROPERTY_KEY_LOCALE：仅在使用 JDBC 原生连接时生效。 客户端语言环境，默认值系统当前 locale。 - TSDBDriver.PROPERTY_KEY_TIME_ZONE：仅在使用 JDBC 原生连接时生效。 客户端使用的时区，默认值为系统当前时区。 -- 此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](/reference/config/#仅客户端适用)。 +- TSDBDriver.HTTP_CONNECT_TIMEOUT: 连接超时时间，单位 ms， 默认值为 5000。仅在 REST 连接时生效。 +- TSDBDriver.HTTP_SOCKET_TIMEOUT: socket 超时时间，单位 ms，默认值为 5000。仅在 REST 连接且 batchfetch 设置为 false 时生效。 +- TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 REST 连接且 batchfetch 设置为 true 时生效。 +- TSDBDriver.PROPERTY_KEY_USE_SSL: 连接中是否使用 SSL。仅在 REST 连接时生效。 + 此外对 JDBC 原生连接，通过指定 URL 和 Properties 还可以指定其他参数，比如日志级别、SQL 长度等。更多详细配置请参考[客户端配置](/reference/config/#仅客户端适用)。 ### 配置参数的优先级 @@ -809,6 +817,7 @@ Query OK, 1 row(s) in set (0.000141s) | taos-jdbcdriver 版本 | 主要变化 | | :------------------: | :----------------------------: | +| 2.0.39 - 2.0.40 | 增加 REST 连接/请求 超时设置 | | 2.0.38 | JDBC REST 连接增加批量拉取功能 | | 2.0.37 | 增加对 json tag 支持 | | 2.0.36 | 增加对 schemaless 写入支持 | diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 8b0a836ad2fd26aa1b11347714dd583552d21c84..9e289117845cc49b07df69135b17ba80596efa0a 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -40,6 +40,7 @@ typedef struct SReadHandle { bool initMetaReader; bool initTableReader; bool initTqReader; + int32_t numOfVgroups; } SReadHandle; // in queue mode, data streams are seperated by msg diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index a93cf1f9b81faec1d0bdb60361a4de3b44749371..58739b4af7b46f4c8c800c90851988610282899e 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -162,9 +162,12 @@ typedef struct SRequestConnInfo { SEpSet mgmtEps; } SRequestConnInfo; +typedef void (*__freeFunc)(void *param); + typedef struct SMsgSendInfo { __async_send_cb_fn_t fp; // async callback function STargetInfo target; // for update epset + __freeFunc paramFreeFp; void* param; uint64_t requestId; uint64_t requestObjRefId; @@ -188,6 +191,8 @@ int32_t cleanupTaskQueue(); */ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); +void destroySendMsgInfo(SMsgSendInfo* pMsgBody); + int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void* ctx); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 8c69c0f2de498fcf463e0e2d8624db29cc610772..a484492d08acc0f9a4e94cbe74f28057494c90ad 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -262,6 +262,7 @@ typedef struct SStreamTask { int64_t startVer; int64_t checkpointVer; int64_t processedVer; + int32_t numOfVgroups; // children info SArray* childEpInfo; // SArray diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 06a58af0e3895b46ef680d85f556a172b68bdbfa..510f816959fab340d56399ef3c35c696ac83d238 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -26,7 +26,7 @@ extern "C" { extern bool gRaftDetailLog; -#define SYNC_RESP_TTL_MS 5000 +#define SYNC_RESP_TTL_MS 10000 #define SYNC_MAX_BATCH_SIZE 500 #define SYNC_INDEX_BEGIN 0 diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index b52df105b1812979b89e7900525da4ec0c7d70e7..6a583842d121f836d1fcaaf5aa3dab2d25b072df 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -286,13 +286,10 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { if (pInst == NULL || NULL == *pInst) { taosThreadMutexUnlock(&appInfo.mutex); tscError("cluster not exist, key:%s", key); - taosMemoryFreeClear(param); tFreeClientHbBatchRsp(&pRsp); return -1; } - taosMemoryFreeClear(param); - if (code != 0) { (*pInst)->onlineDnodes = ((*pInst)->totalDnodes ? 0 : -1); tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), (*pInst)->onlineDnodes, (*pInst)->totalDnodes); @@ -716,6 +713,7 @@ static void *hbThreadFunc(void *param) { pInfo->msgInfo.len = tlen; pInfo->msgType = TDMT_MND_HEARTBEAT; pInfo->param = strdup(pAppHbMgr->key); + pInfo->paramFreeFp = taosMemoryFree; pInfo->requestId = generateRequestId(); pInfo->requestObjRefId = 0; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 73374c7f774beecca4c9ff78ed4e3ff87c48c4e7..2ddf843a08d6932c37e8650789f773979f81dba1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -29,7 +29,6 @@ static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); -static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); static bool stringLengthCheck(const char* str, size_t maxsize) { if (str == NULL) { @@ -1215,13 +1214,6 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { return pMsgSendInfo; } -static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { - assert(pMsgBody != NULL); - taosMemoryFreeClear(pMsgBody->target.dbFName); - taosMemoryFreeClear(pMsgBody->msgInfo.pData); - taosMemoryFreeClear(pMsgBody); -} - void updateTargetEpSet(SMsgSendInfo* pSendInfo, STscObj* pTscObj, SRpcMsg* pMsg, SEpSet* pEpSet) { if (NULL == pEpSet) { return; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 520a566e2b4d077a666aa4be5ab94a20ff00f0b6..db33532aef36d7b4f3fc7f67502a92728b44ea57 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -255,6 +255,8 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid); } + taosMemoryFree(pMsg->pData); + if (pRequest->body.queryFp != NULL) { pRequest->body.queryFp(pRequest->body.param, pRequest, code); } else { @@ -278,6 +280,8 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) { pRequest->body.resInfo.execRes.res = alterRsp.pMeta; } + taosMemoryFree(pMsg->pData); + if (pRequest->body.queryFp != NULL) { SExecResult* pRes = &pRequest->body.resInfo.execRes; @@ -387,6 +391,8 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) { tFreeSShowVariablesRsp(&rsp); } + taosMemoryFree(pMsg->pData); + if (pRequest->body.queryFp != NULL) { pRequest->body.queryFp(pRequest->body.param, pRequest, code); } else { diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 18dc4d16594fdf39388d51fe223fe4095766465f..4ba3750b4cfef2b28a37f3def4e7d364126796f6 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -504,6 +504,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT pMsgSendInfo->requestId = generateRequestId(); pMsgSendInfo->requestObjRefId = 0; pMsgSendInfo->param = pParam; + pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->fp = tmqCommitCb2; pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; // send msg diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index ec0ea90f46069d048009dbb887305512d11afed3..a46938590e19d2465d66eec9d012caa9172c6ca3 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -383,6 +383,11 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { // exec pInnerTask->execType = TASK_EXEC__PIPE; + SDbObj* pSourceDb = mndAcquireDb(pMnode, pStream->sourceDb); + ASSERT(pDbObj != NULL); + sdbRelease(pSdb, pSourceDb); + pInnerTask->numOfVgroups = pSourceDb->cfg.numOfVgroups; + if (tsSchedStreamToSnode) { SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode); if (pSnode == NULL) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index fb05aeecd935c0ae1c4bcb2f6247fe23cfa6bcc4..1802f9be1a58425feaef1d44ea5faab987b2eff1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -591,7 +591,11 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { }; pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); } else { - pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL); + SReadHandle mgHandle = { + .vnode = NULL, + .numOfVgroups = pTask->numOfVgroups, + }; + pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle); } ASSERT(pTask->exec.executor); } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index fb9f588bae5c1a5ab14a25a6dbb0e47e831c1e4c..9003de97d7bd63c0b9cff5f868824179342ec417 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -532,6 +532,14 @@ typedef struct SCtgOperation { } \ } while (0) +#define CTG_API_JENTER() do { \ + CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); \ + CTG_LOCK(CTG_READ, &gCtgMgmt.lock); \ + if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { \ + CTG_ERR_JRET(TSDB_CODE_CTG_OUT_OF_SERVICE); \ + } \ +} while (0) + #define CTG_API_LEAVE_NOLOCK(c) do { \ int32_t __code = c; \ diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 8e0a5b7de3753baaf08d4628032f40dcf51c711e..1e375471f9861eb4c6307463b7e4fe113753f289 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -244,10 +244,11 @@ int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t ctgHandleMsgCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { SCtgTaskCallbackParam* cbParam = (SCtgTaskCallbackParam*)param; int32_t code = 0; + SCtgJob* pJob = NULL; - CTG_API_ENTER(); + CTG_API_JENTER(); - SCtgJob* pJob = taosAcquireRef(gCtgMgmt.jobPool, cbParam->refId); + pJob = taosAcquireRef(gCtgMgmt.jobPool, cbParam->refId); if (NULL == pJob) { qDebug("ctg job refId 0x%" PRIx64 " already dropped", cbParam->refId); goto _return; @@ -266,8 +267,6 @@ _return: if (pJob) { taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId); } - - taosMemoryFree(param); CTG_API_LEAVE(code); } @@ -293,6 +292,7 @@ int32_t ctgMakeMsgSendInfo(SCtgTask* pTask, int32_t msgType, SMsgSendInfo **pMsg param->taskId = pTask->taskId; msgSendInfo->param = param; + msgSendInfo->paramFreeFp = taosMemoryFree; msgSendInfo->fp = ctgHandleMsgCallback; *pMsgSendInfo = msgSendInfo; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index a575e355f1dc7a7aa866bdeb7f548005af10d7c7..1c08fafaa3c5f2b9e4fd12fb68c0d5a5f343a4c7 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -91,7 +91,6 @@ _return: tsem_post(&pInserter->ready); taosMemoryFree(pMsg->pData); - taosMemoryFree(param); return TSDB_CODE_SUCCESS; } @@ -110,6 +109,7 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs pParam->pInserter = pInserter; pMsgSendInfo->param = pParam; + pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->msgInfo.pData = pMsg; pMsgSendInfo->msgInfo.len = ntohl(pMsg->length); pMsgSendInfo->msgType = TDMT_VND_SUBMIT; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 06bb096e59999c3ab4125fa1692239d9bf2c2e6d..d3a7fc51eb18fffb9360b465befcc58d79901d82 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1994,16 +1994,9 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { tsem_post(&pExchangeInfo->ready); taosReleaseRef(exchangeObjRefPool, pWrapper->exchangeId); - taosMemoryFree(pWrapper); return TSDB_CODE_SUCCESS; } -static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { - assert(pMsgBody != NULL); - taosMemoryFreeClear(pMsgBody->msgInfo.pData); - taosMemoryFreeClear(pMsgBody); -} - void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; assert(pMsg->info.ahandle != NULL); @@ -2063,6 +2056,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf pWrapper->sourceIndex = sourceIndex; pMsgSendInfo->param = pWrapper; + pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->msgInfo.pData = pMsg; pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); pMsgSendInfo->msgType = pSource->fetchMsgType; @@ -4424,7 +4418,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN, }; - if (pHandle) { + if (pHandle->vnode) { int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, pTableListInfo, pTagCond, pTagIndexCond, GET_TASKID(pTaskInfo)); if (code) { @@ -4590,7 +4584,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t children = 0; pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL == type) { - int32_t children = 1; + int32_t children = pHandle->numOfVgroups; pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { pOptr = createSortOperatorInfo(ops[0], (SSortPhysiNode*)pPhyNode, pTaskInfo); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e60f6f8a5b662abb858c882d182ef886adc1138b..57ba08e8fa6cd6c0c73f2abb2f8e66d708b51189 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1539,7 +1539,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys goto _error; } - if (pHandle) { + if (pHandle->vnode) { SOperatorInfo* pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo); STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info; if (pHandle->version > 0) { diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 1dc3db033bcfb0905b1552bfc35a7310f976d494..82b6e7b236c29b7bede8b39320c23c1b1fb2dc68 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1923,15 +1923,18 @@ int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** return partitionLogicCond(pCondition, pPrimaryKeyCond, pTagIndexCond, pTagCond, pOtherCond); } + bool needOutput = false; switch (classifyCondition(*pCondition)) { case COND_TYPE_PRIMARY_KEY: if (NULL != pPrimaryKeyCond) { *pPrimaryKeyCond = *pCondition; + needOutput = true; } break; case COND_TYPE_TAG_INDEX: if (NULL != pTagIndexCond) { *pTagIndexCond = *pCondition; + needOutput = true; } if (NULL != pTagCond) { SNode* pTempCond = *pCondition; @@ -1942,21 +1945,26 @@ int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** } } *pTagCond = pTempCond; + needOutput = true; } break; case COND_TYPE_TAG: if (NULL != pTagCond) { *pTagCond = *pCondition; + needOutput = true; } break; case COND_TYPE_NORMAL: default: if (NULL != pOtherCond) { *pOtherCond = *pCondition; + needOutput = true; } break; } - *pCondition = NULL; + if (needOutput) { + *pCondition = NULL; + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index eeb44c4f82db68b445e944ed1b4870686187f740..6b1476fe46ee10737e0d944259b3425f3aba1d40 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -138,6 +138,16 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) return 0; } +void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { + assert(pMsgBody != NULL); + taosMemoryFreeClear(pMsgBody->target.dbFName); + taosMemoryFreeClear(pMsgBody->msgInfo.pData); + if (pMsgBody->paramFreeFp) { + (*pMsgBody->paramFreeFp)(pMsgBody->param); + } + taosMemoryFreeClear(pMsgBody); +} + int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void* rpcCtx) { char* pMsg = rpcMallocCont(pInfo->msgInfo.len); diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 65b45cc612208848380075e1490f908110d4f0d5..02e878f4f80731a21c30d17b600da85810836b38 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -277,7 +277,7 @@ typedef struct SSchJob { bool fetched; int32_t resNumOfRows; SSchResInfo userRes; - const char *sql; + char *sql; SQueryProfileSummary summary; } SSchJob; @@ -461,7 +461,6 @@ int32_t schJobFetchRows(SSchJob *pJob); int32_t schJobFetchRowsA(SSchJob *pJob); int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execId); int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId* pEpId, SArray* pStatusList); -void schFreeSMsgSendInfo(SMsgSendInfo *msgSendInfo); char* schGetOpStr(SCH_OP_TYPE type); int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 13a369fac95ea28623fb83d31ad2afc9889e013b..50f8b9102379d7a5280c1f10654263719e43fb90 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -675,6 +675,7 @@ void schFreeJobImpl(void *job) { taosMemoryFreeClear(pJob->userRes.execRes); taosMemoryFreeClear(pJob->fetchRes); + taosMemoryFreeClear(pJob->sql); taosMemoryFree(pJob); int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); @@ -718,7 +719,9 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->attr.explainMode = pReq->pDag->explainInfo.mode; pJob->conn = *pReq->pConn; - pJob->sql = pReq->sql; + if (pReq->sql) { + pJob->sql = strdup(pReq->sql); + } pJob->pDag = pReq->pDag; pJob->chkKillFp = pReq->chkKillFp; pJob->chkKillParam = pReq->chkKillParam; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 6983bbf0130cf382eb20772c12a5f0cd3503f9ad..5452ca31a516fef97f81e4ad2eefd4f5ce1e4626 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -386,7 +386,6 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { schProcessOnCbEnd(pJob, pTask, code); taosMemoryFreeClear(pMsg->pData); - taosMemoryFreeClear(param); qDebug("end to handle rsp msg, type:%s, handle:%p, code:%s", TMSG_INFO(pMsg->msgType), pMsg->handle, tstrerror(rspCode)); @@ -398,7 +397,6 @@ int32_t schHandleDropCallback(void *param, SDataBuf *pMsg, int32_t code) { SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " drop task rsp received, code:0x%x", pParam->queryId, pParam->taskId, code); - taosMemoryFreeClear(param); return TSDB_CODE_SUCCESS; } @@ -447,8 +445,8 @@ int32_t schHandleHbCallback(void *param, SDataBuf *pMsg, int32_t code) { SCH_ERR_JRET(schProcessOnTaskStatusRsp(&rsp.epId, rsp.taskStatus)); _return: + tFreeSSchedulerHbRsp(&rsp); - taosMemoryFree(param); taosMemoryFree(pMsg->pData); SCH_RET(code); } @@ -514,7 +512,9 @@ int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void *msg, uint3 SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + msgSendInfo->paramFreeFp = taosMemoryFree; SCH_ERR_JRET(schMakeCallbackParam(pJob, pTask, msgType, isHb, trans, &msgSendInfo->param)); + SCH_ERR_JRET(schGetCallbackFp(msgType, &msgSendInfo->fp)); if (pJob) { @@ -535,7 +535,7 @@ int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void *msg, uint3 _return: - schFreeSMsgSendInfo(msgSendInfo); + destroySendMsgInfo(msgSendInfo); SCH_RET(code); } @@ -676,6 +676,7 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { param->pTrans = pJob->conn.pTrans; pMsgSendInfo->param = param; + pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->fp = fp; SRpcCtxVal ctxVal = {.val = pMsgSendInfo, .clone = schCloneSMsgSendInfo}; @@ -795,6 +796,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) { pDst->param = NULL; SCH_ERR_JRET(schCloneCallbackParam(pSrc->param, (SSchCallbackParamHeader **)&pDst->param)); + pDst->paramFreeFp = taosMemoryFree; *dst = pDst; @@ -861,8 +863,7 @@ _return: } if (pMsgSendInfo) { - taosMemoryFreeClear(pMsgSendInfo->param); - taosMemoryFreeClear(pMsgSendInfo); + destroySendMsgInfo(pMsgSendInfo); } SCH_RET(code); diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index f848dfa2101f7f5ab23c860f8e34f788fe8054cb..6f12780ff932a01a3303a175ec0d2be45b0875a2 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -50,6 +50,12 @@ char* schGetOpStr(SCH_OP_TYPE type) { } } +void schFreeHbTrans(SSchHbTrans *pTrans) { + rpcReleaseHandle(pTrans->trans.pHandle, TAOS_CONN_CLIENT); + + schFreeRpcCtx(&pTrans->rpcCtx); +} + void schCleanClusterHb(void* pTrans) { SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); @@ -57,7 +63,7 @@ void schCleanClusterHb(void* pTrans) { while (hb) { if (hb->trans.pTrans == pTrans) { SQueryNodeEpId* pEpId = taosHashGetKey(hb, NULL); - rpcReleaseHandle(hb->trans.pHandle, TAOS_CONN_CLIENT); + schFreeHbTrans(hb); taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); } @@ -68,8 +74,6 @@ void schCleanClusterHb(void* pTrans) { } int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId) { - return TSDB_CODE_SUCCESS; // TODO ENABLE IT WHEN RPC IS READY - int32_t code = 0; SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); @@ -82,7 +86,7 @@ int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *ep int64_t taskNum = atomic_load_64(&hb->taskNum); if (taskNum <= 0) { - rpcReleaseHandle(hb->trans.pHandle, TAOS_CONN_CLIENT); + schFreeHbTrans(hb); taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); } SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); @@ -265,9 +269,7 @@ void schFreeRpcCtxVal(const void *arg) { } SMsgSendInfo *pMsgSendInfo = (SMsgSendInfo *)arg; - taosMemoryFreeClear(pMsgSendInfo->param); - taosMemoryFreeClear(pMsgSendInfo->msgInfo.pData); - taosMemoryFreeClear(pMsgSendInfo); + destroySendMsgInfo(pMsgSendInfo); } void schFreeRpcCtx(SRpcCtx *pCtx) { @@ -290,15 +292,6 @@ void schFreeRpcCtx(SRpcCtx *pCtx) { } } -void schFreeSMsgSendInfo(SMsgSendInfo *msgSendInfo) { - if (NULL == msgSendInfo) { - return; - } - - taosMemoryFree(msgSendInfo->param); - taosMemoryFree(msgSendInfo); -} - int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) { int32_t s = taosHashGetSize(pTaskList); if (s <= 0) { diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 2d15c31bf1eb82e11c8687ba065765b0986f4036..5921e44a9c51cc650d122e85e75b98ec0531acac 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -64,6 +64,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->numOfVgroups) < 0) return -1; int32_t epSz = taosArrayGetSize(pTask->childEpInfo); if (tEncodeI32(pEncoder, epSz) < 0) return -1; @@ -118,6 +119,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->numOfVgroups) < 0) return -1; int32_t epSz; if (tDecodeI32(pDecoder, &epSz) < 0) return -1; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index ddb2b9355e1540bc1b52d3fe9ddb9d3546e57bf8..e11f610dd4aab0ef39dd357e791e3104905458f6 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1552,7 +1552,8 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { } else { snprintf(logBuf, sizeof(logBuf), "%s", str); } - sDebug("%s", logBuf); + // sDebug("%s", logBuf); + sInfo("%s", logBuf); } else { int len = 256 + userStrLen; @@ -1573,7 +1574,8 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { } else { snprintf(s, len, "%s", str); } - sDebug("%s", s); + // sDebug("%s", s); + sInfo("%s", s); taosMemoryFree(s); }