diff --git a/Jenkinsfile2 b/Jenkinsfile2 index a2b55e3acca0c141a2d550ccabb5bb129adb3d7e..950b316bab8110ef5da8522023b3971e2f35d4da 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -4,8 +4,10 @@ import jenkins.model.CauseOfInterruption node { } -def skipbuild=0 -def win_stop=0 +win_test_stage = 0 +linux_ready = 0 +linux_node_ip = "" +linux_node_pass = "" def abortPreviousBuilds() { def currentJobName = env.JOB_NAME @@ -227,6 +229,16 @@ def pre_test_win(){ cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community git submodule update --init --recursive ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python + git branch + git reset --hard + git pull + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python + git log -5 + ''' } def pre_test_build_win() { bat ''' @@ -246,8 +258,38 @@ def pre_test_build_win() { jom -j 6 || exit 8 time /t ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python + python -m pip install . + xcopy /e/y/i/f C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 + ''' return 1 } +def run_win_ctest() { + bat ''' + echo "windows ctest ..." + time /t + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug + ctest -j 1 || exit 7 + time /t + ''' +} +def run_win_test() { + echo "LINUX NODE: ${linux_node_ip} - ${linux_node_pass}" + bat ''' + echo "windows test ..." + cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python + python -m pip install . + xcopy /e/y/i/f C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 + ls -l C:\\Windows\\System32\\taos.dll + time /t + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community\\tests\\system-test + echo "node: ''' + linux_node_ip + ''':''' + linux_node_pass + '''" + echo "testing ..." + test-all.bat "{\\\"host\\\":\\\"''' + linux_node_ip + '''\\\",\\\"port\\\":22,\\\"user\\\":\\\"root\\\",\\\"password\\\":\\\"''' + linux_node_pass + '''\\\",\\\"path\\\":\\\"/var/lib/jenkins/workspace/TDinternal\\\"}" + time /t + ''' +} pipeline { agent none @@ -264,8 +306,22 @@ pipeline { stage('windows test') { agent{label " windows10_01 || windows10_02 || windows10_03 || windows10_04 "} steps { - pre_test_win() - pre_test_build_win() + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + timeout(time: 55, unit: 'MINUTES'){ + pre_test_win() + pre_test_build_win() + run_win_ctest() + script { + while(linux_ready == 0) { + sleep(8) + } + } + run_win_test() + } + } + script { + win_test_stage = 1 + } } } stage('linux test') { @@ -275,22 +331,78 @@ pipeline { changeRequest() } steps { - timeout(time: 40, unit: 'MINUTES'){ - pre_test() - script { - sh ''' - cd ${WKC}/tests/parallel_test - date - time ./container_build.sh -w ${WKDIR} -t 8 -e - rm -f /tmp/cases.task - ./collect_cases.sh -e - ''' - sh ''' - cd ${WKC}/tests/parallel_test - export DEFAULT_RETRY_TIME=2 - date - timeout 2100 time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480 - ''' + script { + linux_node_ip = sh ( + script: 'jq .ip /home/node_info.json | sed "s/\\\"//g"', + returnStdout: true + ).trim() + linux_node_pass = sh ( + script: 'jq .password /home/node_info.json | sed "s/\\\"//g" |sed "s/\\!/^^^^^^^^\\!/g"', + returnStdout: true + ).trim() + echo "${linux_node_ip}:${linux_node_pass}" + } + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + timeout(time: 40, unit: 'MINUTES'){ + pre_test() + script { + sh ''' + date + rm -rf ${WKC}/debug + cd ${WKC}/tests/parallel_test + time ./container_build.sh -w ${WKDIR} -t 8 -e + rm -f /tmp/cases.task + ./collect_cases.sh -e + ''' + sh ''' + cd ${WKC}/tests/parallel_test + export DEFAULT_RETRY_TIME=2 + date + timeout 2100 time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480 + ''' + } + } + } + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + timeout(time: 15, unit: 'MINUTES'){ + script { + sh ''' + echo "packaging ..." + date + rm -rf ${WKC}/release/* + cd ${WKC}/packaging + ./release.sh -v cluster -n 3.0.0.100 -s static + ''' + sh ''' + echo "install ..." + cd ${WKC}/release + tar xzf TDengine-enterprise-server-3.0.0.100-Linux-x64.tar.gz + cd TDengine-enterprise-server-3.0.0.100 + service taosd stop || : + rm -rf /var/lib/taos + ./install.sh -e no + ''' + sh ''' + echo "checking ..." + which taos + which taosd + rm -rf ${WK}/debug + mv ${WKC}/debug ${WK}/ + ''' + sh ''' + echo "install taospy ..." + cd ${WKPY} + pip3 install . + ''' + } + } + } + script { + linux_ready = 1 + } + script { + while(win_test_stage == 0){ + sleep(12) } } } diff --git a/cmake/cmake.options b/cmake/cmake.options index ab3c5ac1ad08b98ee2dbe09692584be63e477d71..62816a80d52f0460b9b14035c0032fef8dd7eafa 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -146,6 +146,6 @@ option( option( BUILD_WITH_INVERTEDINDEX "If use invertedIndex" - OFF + ON ) diff --git a/docs-cn/14-reference/02-rest-api/02-rest-api.mdx b/docs-cn/14-reference/02-rest-api/02-rest-api.mdx index c7680ab3e9e109dbb328711f62881283241444fb..43099319b9c5bb1420c199cfa9f7def0b2c44d3d 100644 --- a/docs-cn/14-reference/02-rest-api/02-rest-api.mdx +++ b/docs-cn/14-reference/02-rest-api/02-rest-api.mdx @@ -16,7 +16,7 @@ RESTful 接口不依赖于任何 TDengine 的库，因此客户端不需要安 在已经安装 TDengine 服务器端的情况下，可以按照如下方式进行验证。 -下面以 Ubuntu 环境中使用 curl 工具（确认已经安装）来验证 RESTful 接口的正常。 +下面以 Ubuntu 环境中使用 curl 工具（确认已经安装）来验证 RESTful 接口的正常，验证前请确认 taosAdapter 服务已开启，在 Linux 系统上此服务默认由 systemd 管理，使用命令 `systemctl start taosadapter` 启动。 下面示例是列出所有的数据库，请把 h1.taosdata.com 和 6041（缺省值）替换为实际运行的 TDengine 服务 FQDN 和端口号： diff --git a/include/client/taos.h b/include/client/taos.h index b65091f52bdd218138891970f079158033cb2d69..7f317b3a9d0f47768e106b1745fa1ec7325b449a 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -103,10 +103,10 @@ typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); typedef struct TAOS_MULTI_BIND { int buffer_type; - void *buffer; + void * buffer; uintptr_t buffer_length; - int32_t *length; - char *is_null; + int32_t * length; + char * is_null; int num; } TAOS_MULTI_BIND; @@ -130,7 +130,7 @@ DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT setConfRet taos_set_config(const char *config); DLL_EXPORT int taos_init(void); -DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); +DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); @@ -147,22 +147,21 @@ DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name DLL_EXPORT int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields); DLL_EXPORT int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields); -DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); -DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); -DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); -DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); -DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); -DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); -DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); +DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); +DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); +DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); +DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); +DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); +DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); -DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); +DLL_EXPORT char * taos_stmt_errstr(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); -DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result @@ -180,11 +179,11 @@ DLL_EXPORT bool taos_is_update_query(TAOS_RES *res); DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); DLL_EXPORT int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows); DLL_EXPORT int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData); -DLL_EXPORT int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex); +DLL_EXPORT int * taos_get_column_data_offset(TAOS_RES *res, int columnIndex); DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); DLL_EXPORT void taos_reset_current_db(TAOS *taos); -DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); +DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); DLL_EXPORT const char *taos_get_server_info(TAOS *taos); @@ -205,7 +204,7 @@ DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub); DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress); #endif -DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); +DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); /* --------------------------TMQ INTERFACE------------------------------- */ @@ -230,7 +229,7 @@ DLL_EXPORT tmq_list_t *tmq_list_new(); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); DLL_EXPORT void tmq_list_destroy(tmq_list_t *); DLL_EXPORT int32_t tmq_list_get_size(const tmq_list_t *); -DLL_EXPORT char **tmq_list_to_c_array(const tmq_list_t *); +DLL_EXPORT char ** tmq_list_to_c_array(const tmq_list_t *); DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen); @@ -241,7 +240,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list); DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); -DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout); +DLL_EXPORT TAOS_RES * tmq_consumer_poll(tmq_t *tmq, int64_t timeout); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets); DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param); @@ -261,7 +260,7 @@ enum tmq_conf_res_t { typedef enum tmq_conf_res_t tmq_conf_res_t; -DLL_EXPORT tmq_conf_t *tmq_conf_new(); +DLL_EXPORT tmq_conf_t * tmq_conf_new(); DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param); diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 9b71e8c454948b9be257986e01ee146a09a10e55..8d69bb6c701f24e71d645f358e75168d3f89b356 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -71,20 +71,14 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet); #define colDataGetData(p1_, r_) \ ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_)) -static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { - if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) { - if (colDataIsNull_var(pColumnInfoData, row)) { - return true; - } - char* data = colDataGetVarData(pColumnInfoData, row); - return (*data == TSDB_DATA_TYPE_NULL); - } +#define IS_JSON_NULL(type,data) ((type) == TSDB_DATA_TYPE_JSON && *(data) == TSDB_DATA_TYPE_NULL) +static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { if (!pColumnInfoData->hasNull) { return false; } - if (pColumnInfoData->info.type == TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) { + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return colDataIsNull_var(pColumnInfoData, row); } else { if (pColumnInfoData->nullbitmap == NULL) { @@ -186,6 +180,8 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u *(double*)p = *(double*)v; } +int32_t getJsonValueLen(const char *data); + int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2); @@ -227,7 +223,10 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); -void blockDebugShowData(const SArray* dataBlocks); +void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); +const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); + +void blockDebugShowData(const SArray* dataBlocks, const char* flag); int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, tb_uid_t suid); @@ -246,54 +245,6 @@ static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32 colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); } -static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, - int8_t needCompress) { - int32_t* actualLen = (int32_t*)data; - data += sizeof(int32_t); - - uint64_t* groupId = (uint64_t*)data; - data += sizeof(uint64_t); - - int32_t* colSizes = (int32_t*)data; - data += numOfCols * sizeof(int32_t); - - *dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t)); - - int32_t numOfRows = pBlock->info.rows; - for (int32_t col = 0; col < numOfCols; ++col) { - SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, col); - - // copy the null bitmap - if (IS_VAR_DATA_TYPE(pColRes->info.type)) { - size_t metaSize = numOfRows * sizeof(int32_t); - memcpy(data, pColRes->varmeta.offset, metaSize); - data += metaSize; - (*dataLen) += metaSize; - } else { - int32_t len = BitmapLen(numOfRows); - memcpy(data, pColRes->nullbitmap, len); - data += len; - (*dataLen) += len; - } - - if (needCompress) { - colSizes[col] = blockCompressColData(pColRes, numOfRows, data, needCompress); - data += colSizes[col]; - (*dataLen) += colSizes[col]; - } else { - colSizes[col] = colDataGetLength(pColRes, numOfRows); - (*dataLen) += colSizes[col]; - memmove(data, pColRes->pData, colSizes[col]); - data += colSizes[col]; - } - - colSizes[col] = htonl(colSizes[col]); - } - - *actualLen = *dataLen; - *groupId = pBlock->info.groupId; -} - #ifdef __cplusplus } #endif diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 10bc6a61764f6500305e0712650e93d55255c58a..2e69640a065f9049f2bcb2a2c3f190f0178fca3f 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -34,6 +34,7 @@ typedef struct SValue SValue; typedef struct SColVal SColVal; typedef struct STSRow2 STSRow2; typedef struct STSRowBuilder STSRowBuilder; +typedef struct SColData SColData; typedef struct STagVal STagVal; typedef struct STag STag; @@ -41,11 +42,15 @@ typedef struct STag STag; int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); void tTSchemaDestroy(STSchema *pTSchema); +// SValue +int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type); + // STSRow2 #define COL_VAL_NONE(CID) ((SColVal){.cid = (CID), .isNone = 1}) #define COL_VAL_NULL(CID) ((SColVal){.cid = (CID), .isNull = 1}) #define COL_VAL_VALUE(CID, V) ((SColVal){.cid = (CID), .value = (V)}) +int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow); int32_t tTSRowClone(const STSRow2 *pRow, STSRow2 **ppRow); void tTSRowFree(STSRow2 *pRow); void tTSRowGet(STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); @@ -54,24 +59,24 @@ int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow); // STSRowBuilder -#if 0 -int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema); -void tTSRowBuilderClear(STSRowBuilder *pBuilder); -void tTSRowBuilderReset(STSRowBuilder *pBuilder); -int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData); -int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); -#endif +#define tsRowBuilderInit() ((STSRowBuilder){0}) +#define tsRowBuilderClear(B) \ + do { \ + if ((B)->pBuf) { \ + taosMemoryFree((B)->pBuf); \ + } \ + } while (0) // STag int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); void tTagFree(STag *pTag); bool tTagGet(const STag *pTag, STagVal *pTagVal); -char* tTagValToData(const STagVal *pTagVal, bool isJson); +char *tTagValToData(const STagVal *pTagVal, bool isJson); int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); int32_t tTagToValArray(const STag *pTag, SArray **ppArray); -void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove -void debugCheckTags(STag *pTag); // TODO: remove +void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove +void debugCheckTags(STag *pTag); // TODO: remove // STRUCT ================= struct STColumn { @@ -106,17 +111,9 @@ struct STSRow2 { }; struct STSRowBuilder { - STSchema *pTSchema; - int32_t szBitMap1; - int32_t szBitMap2; - int32_t szKVBuf; - uint8_t *pKVBuf; - int32_t szTPBuf; - uint8_t *pTPBuf; - int32_t iCol; - int32_t vlenKV; - int32_t vlenTP; - STSRow2 row; + STSRow2 tsRow; + int32_t szBuf; + uint8_t *pBuf; }; struct SValue { @@ -154,7 +151,7 @@ struct STagVal { }; int8_t type; union { - int64_t i64; + int64_t i64; struct { uint32_t nData; uint8_t *pData; @@ -162,7 +159,7 @@ struct STagVal { }; }; -#define TD_TAG_JSON ((int8_t)0x40) // distinguish JSON string and JSON value with the highest bit +#define TD_TAG_JSON ((int8_t)0x40) // distinguish JSON string and JSON value with the highest bit #define TD_TAG_LARGE ((int8_t)0x20) struct STag { int8_t flags; @@ -173,6 +170,12 @@ struct STag { }; #pragma pack(pop) +struct SColData { + int16_t cid; + uint32_t nData; + uint8_t *pData; +}; + #if 1 //================================================================================================================================================ // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP @@ -422,4 +425,3 @@ int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToM #endif #endif /*_TD_COMMON_DATA_FORMAT_H_*/ - diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 2a4ef565dd1c6b6742446adee2daf953665b99e5..30ae6c2adb49a811803d04309f43f3068065269c 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -45,6 +45,8 @@ extern bool tsEnableSlaveQuery; extern bool tsPrintAuth; extern int64_t tsTickPerMin[3]; +extern int32_t tsCountAlwaysReturnValue; + // multi-process extern int32_t tsMultiProcess; extern int32_t tsMnodeShmSize; @@ -102,7 +104,6 @@ extern int32_t tsMaxStreamComputDelay; extern int32_t tsStreamCompStartDelay; extern int32_t tsRetryStreamCompDelay; extern float tsStreamComputDelayRatio; // the delayed computing ration of the whole time window -extern int32_t tsProjectExecInterval; extern int64_t tsMaxRetentWindow; // build info diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 424e99760c94ba77ec4eccd2d10a7dbdc9a1dc24..47c26ed4317c49a3436ef18d8750b1c87b139504 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -177,8 +177,8 @@ typedef struct { typedef struct SField { char name[TSDB_COL_NAME_LEN]; uint8_t type; - int32_t bytes; int8_t flags; + int32_t bytes; } SField; typedef struct SRetention { @@ -241,6 +241,7 @@ typedef struct { int32_t schemaLen; // schema length, if length is 0, no schema exists int16_t numOfRows; // total number of rows in current submit block // head of SSubmitBlk + int32_t numOfBlocks; const void* pMsg; } SSubmitMsgIter; @@ -571,13 +572,6 @@ int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pR int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); void tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp); -typedef struct { - int16_t colId; // column id - int16_t colIndex; // column index in colList if it is a normal column or index in tagColList if a tag - int16_t flag; // denote if it is a tag or a normal column - char name[TSDB_DB_FNAME_LEN]; -} SColIndex; - typedef struct { int16_t lowerRelOptr; int16_t upperRelOptr; @@ -1128,13 +1122,13 @@ typedef struct { SSchema* pSchemas; } STableMetaRsp; -typedef struct { +typedef struct { STableMetaRsp* pMeta; } SMAlterStbRsp; -int32_t tEncodeSMAlterStbRsp(SEncoder *pEncoder, const SMAlterStbRsp *pRsp); -int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp); -void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp); +int32_t tEncodeSMAlterStbRsp(SEncoder* pEncoder, const SMAlterStbRsp* pRsp); +int32_t tDecodeSMAlterStbRsp(SDecoder* pDecoder, SMAlterStbRsp* pRsp); +void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp); int32_t tSerializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); @@ -1988,6 +1982,7 @@ typedef struct { typedef struct { SClientHbKey connKey; + int64_t clusterId; SQueryHbReqBasic* query; SHashObj* info; // hash } SClientHbReq; @@ -2303,21 +2298,28 @@ int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); typedef struct { - int8_t version; // for compatibility(default 0) - int8_t intervalUnit; // MACRO: TIME_UNIT_XXX - int8_t slidingUnit; // MACRO: TIME_UNIT_XXX - int8_t timezoneInt; // sma data expired if timezone changes. - int32_t dstVgId; - char indexName[TSDB_INDEX_NAME_LEN]; - int32_t exprLen; - int32_t tagsFilterLen; - int64_t indexUid; - tb_uid_t tableUid; // super/child/common table uid - int64_t interval; - int64_t offset; // use unit by precision of DB - int64_t sliding; - char* expr; // sma expression - char* tagsFilter; + int32_t vgId; + SEpSet epSet; +} SVgEpSet; + +typedef struct { + int8_t version; // for compatibility(default 0) + int8_t intervalUnit; // MACRO: TIME_UNIT_XXX + int8_t slidingUnit; // MACRO: TIME_UNIT_XXX + int8_t timezoneInt; // sma data expired if timezone changes. + int32_t dstVgId; + char indexName[TSDB_INDEX_NAME_LEN]; + int32_t exprLen; + int32_t tagsFilterLen; + int32_t numOfVgroups; + int64_t indexUid; + tb_uid_t tableUid; // super/child/common table uid + int64_t interval; + int64_t offset; // use unit by precision of DB + int64_t sliding; + char* expr; // sma expression + char* tagsFilter; + SVgEpSet* pVgEpSet; } STSma; // Time-range-wise SMA typedef STSma SVCreateTSmaReq; @@ -2354,19 +2356,19 @@ typedef struct { STSma* tSma; } STSmaWrapper; -static FORCE_INLINE void tdDestroyTSma(STSma* pSma) { +static FORCE_INLINE void tDestroyTSma(STSma* pSma) { if (pSma) { taosMemoryFreeClear(pSma->expr); taosMemoryFreeClear(pSma->tagsFilter); } } -static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) { +static FORCE_INLINE void tDestroyTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) { if (pSW) { if (pSW->tSma) { if (deepCopy) { for (uint32_t i = 0; i < pSW->number; ++i) { - tdDestroyTSma(pSW->tSma + i); + tDestroyTSma(pSW->tSma + i); } } taosMemoryFreeClear(pSW->tSma); @@ -2374,8 +2376,8 @@ static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) } } -static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) { - tdDestroyTSmaWrapper(pSW, deepCopy); +static FORCE_INLINE void* tFreeTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) { + tDestroyTSmaWrapper(pSW, deepCopy); taosMemoryFreeClear(pSW); return NULL; } @@ -2403,16 +2405,26 @@ static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) { } typedef struct { - int64_t tsmaIndexUid; + int64_t indexUid; STimeWindow queryWindow; } SVGetTsmaExpWndsReq; +#define SMA_WNDS_EXPIRE_FLAG (0x1) +#define SMA_WNDS_IS_EXPIRE(flag) (((flag)&SMA_WNDS_EXPIRE_FLAG) != 0) +#define SMA_WNDS_SET_EXPIRE(flag) ((flag) |= SMA_WNDS_EXPIRE_FLAG) + typedef struct { - int64_t tsmaIndexUid; + int64_t indexUid; + int8_t flags; // 0x1 all window expired int32_t numExpWnds; - TSKEY* expWndsStartTs; + TSKEY wndSKeys[]; } SVGetTsmaExpWndsRsp; +int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder* pCoder, const SVGetTsmaExpWndsReq* pReq); +int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder* pCoder, SVGetTsmaExpWndsReq* pReq); +int32_t tEncodeSVGetTSmaExpWndsRsp(SEncoder* pCoder, const SVGetTsmaExpWndsRsp* pReq); +int32_t tDecodeSVGetTsmaExpWndsRsp(SDecoder* pCoder, SVGetTsmaExpWndsRsp* pReq); + typedef struct { int idx; } SMCreateFullTextReq; @@ -2676,23 +2688,27 @@ typedef struct { int32_t tEncodeSVSubmitReq(SEncoder* pCoder, const SVSubmitReq* pReq); int32_t tDecodeSVSubmitReq(SDecoder* pCoder, SVSubmitReq* pReq); -// TDMT_VND_DELETE typedef struct { - TSKEY sKey; - TSKEY eKey; - - // super table - char* stbName; - - // child/normal - char* tbName; + int64_t delUid; + int64_t tbUid; // super/child/normal table + int8_t type; // table type + int16_t nWnds; + char* tbFullName; + char* subPlan; + STimeWindow wnds[]; } SVDeleteReq; +int32_t tEncodeSVDeleteReq(SEncoder* pCoder, const SVDeleteReq* pReq); +int32_t tDecodeSVDeleteReq(SDecoder* pCoder, SVDeleteReq* pReq); + typedef struct { int32_t code; - // TODO + int64_t affectedRows; } SVDeleteRsp; +int32_t tEncodeSVDeleteRsp(SEncoder* pCoder, const SVDeleteRsp* pReq); +int32_t tDecodeSVDeleteRsp(SDecoder* pCoder, SVDeleteRsp* pReq); + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 9fa657a2a6ad78fdd70ed1b4e2ed816b06780351..e99377f9b4b27871506d1739520060b8caa51417 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -38,7 +38,7 @@ typedef enum { QUEUE_MAX, } EQueueType; -typedef int32_t (*PutToQueueFp)(void* pMgmt, SRpcMsg* pMsg); +typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg); typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg); typedef void (*SendRspFp)(SRpcMsg* pMsg); @@ -50,7 +50,7 @@ typedef void (*ReportStartup)(const char* name, const char* desc); typedef struct { void* mgmt; void* clientRpc; - PutToQueueFp queueFps[QUEUE_MAX]; + PutToQueueFp putToQueueFp; GetQueueSizeFp qsizeFp; SendReqFp sendReqFp; SendRspFp sendRspFp; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 860674b34361a42bb8b6ef2267d32b191b1c78d0..73f8515f22df69b8a2a4bf2f2cfff2170331a903 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -67,109 +67,103 @@ enum { enum { #endif - // Requests handled by DNODE TD_NEW_MSG_SEG(TDMT_DND_MSG) TD_DEF_MSG_TYPE(TDMT_DND_CREATE_MNODE, "dnode-create-mnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_ALTER_MNODE, "dnode-alter-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_MNODE, "dnode-drop-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CREATE_QNODE, "dnode-create-qnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_ALTER_QNODE, "dnode-alter-qnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_QNODE, "dnode-drop-qnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CREATE_SNODE, "dnode-create-snode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_ALTER_SNODE, "dnode-alter-snode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_SNODE, "dnode-drop-snode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CREATE_BNODE, "dnode-create-bnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_ALTER_BNODE, "dnode-alter-bnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_BNODE, "dnode-drop-bnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_CREATE_VNODE, "dnode-create-vnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "dnode-server-status", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "dnode-net-test", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CREATE_VNODE, "create-vnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "drop-vnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "server-status", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "net-test", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "config-dnode", NULL, NULL) - // Requests handled by MNODE TD_NEW_MSG_SEG(TDMT_MND_MSG) - TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "mnode-connect", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_ACCT, "mnode-create-acct", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_ACCT, "mnode-alter-acct", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_ACCT, "mnode-drop-acct", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_USER, "mnode-create-user", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_USER, "mnode-alter-user", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_USER, "mnode-drop-user", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_GET_USER_AUTH, "mnode-get-user-auth", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_DNODE, "mnode-create-dnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_DNODE, "mnode-config-dnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_DNODE, "mnode-alter-dnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_DNODE, "mnode-drop-dnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_MNODE, "mnode-create-mnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_MNODE, "mnode-alter-mnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_MNODE, "mnode-drop-mnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_QNODE, "mnode-create-qnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_QNODE, "mnode-alter-qnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_QNODE, "mnode-drop-qnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SNODE, "mnode-create-snode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_SNODE, "mnode-alter-snode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_SNODE, "mnode-drop-snode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_BNODE, "mnode-create-bnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_BNODE, "mnode-alter-bnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_BNODE, "mnode-drop-bnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_DB, "mnode-create-db", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_DB, "mnode-drop-db", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_USE_DB, "mnode-use-db", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_DB, "mnode-alter-db", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_SYNC_DB, "mnode-sync-db", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_DB, "mnode-compact-db", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_FUNC, "mnode-create-func", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_FUNC, "mnode-retrieve-func", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_FUNC, "mnode-drop-func", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "mnode-create-stb", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "mnode-alter-stb", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SMA, "mnode-create-sma", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_SMA, "mnode-drop-sma", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_TABLE_META, "mnode-table-meta", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_QNODE_LIST, "mnode-qnode-list", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "mnode-kill-query", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "mnode-kill-conn", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "mnode-heartbeat", SClientHbBatchReq, SClientHbBatchRsp) - TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_SYSTABLE_RETRIEVE, "mnode-systable-retrieve", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "mnode-trans-tmr", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "mnode-kill-trans", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "mnode-telem-tmr", SMTimerReq, SMTimerReq) - TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "mnode-grant", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "mnode-auth", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SMCreateTopicReq, SMCreateTopicRsp) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "mnode-mq-ask-ep", SMqAskEpReq, SMqAskEpRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMqConsumerLostMsg, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_RECOVER, "mnode-mq-consumer-recover", SMqConsumerRecoverMsg, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_DROP_CGROUP, "mnode-mq-drop-cgroup", SMqDropCGroupReq, SMqDropCGroupRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_STREAM, "mnode-drop-stream", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_INDEX, "mnode-create-index", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_INDEX, "mnode-drop-index", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_GET_DB_CFG, "mnode-get-db-cfg", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_GET_INDEX, "mnode-get-index", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_APPLY_MSG, "mnode-apply-msg", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_ACCT, "create-acct", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_ACCT, "alter-acct", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_ACCT, "drop-acct", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_USER, "create-user", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_USER, "alter-user", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_USER, "drop-user", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_GET_USER_AUTH, "get-user-auth", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_DNODE, "create-dnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_DNODE, "config-dnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_DNODE, "drop-dnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_MNODE, "create-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_MNODE, "alter-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_MNODE, "drop-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_QNODE, "create-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_QNODE, "alter-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_QNODE, "drop-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_QNODE_LIST, "qnode-list", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SNODE, "create-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_SNODE, "alter-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_SNODE, "drop-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_BNODE, "create-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_BNODE, "alter-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_BNODE, "drop-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_DB, "create-db", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_DB, "drop-db", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_USE_DB, "use-db", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_DB, "alter-db", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_SYNC_DB, "sync-db", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_DB, "compact-db", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_GET_DB_CFG, "get-db-cfg", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "vgroup-list", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_FUNC, "create-func", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_FUNC, "retrieve-func", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_FUNC, "drop-func", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "create-stb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "alter-stb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "drop-stb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TABLE_META, "table-meta", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SMA, "create-sma", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_SMA, "drop-sma", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "create-stream", SCMCreateStreamReq, SCMCreateStreamRsp) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "alter-stream", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_STREAM, "drop-stream", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_INDEX, "create-index", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_INDEX, "drop-index", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_GET_INDEX, "get-index", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "create-topic", SMCreateTopicReq, SMCreateTopicRsp) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "alter-topic", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "drop-topic", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "subscribe", SCMSubscribeReq, SCMSubscribeRsp) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "ask-ep", SMqAskEpReq, SMqAskEpRsp) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "consumer-lost", SMqConsumerLostMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_RECOVER, "consumer-recover", SMqConsumerRecoverMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "do-rebalance", SMqDoRebalanceMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_DROP_CGROUP, "drop-cgroup", SMqDropCGroupReq, SMqDropCGroupRsp) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mq-tmr", SMTimerReq, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "telem-tmr", SMTimerReq, SMTimerReq) + TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "trans-tmr", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "kill-trans", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "kill-query", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "kill-conn", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "heartbeat", SClientHbBatchReq, SClientHbBatchRsp) + TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "status", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "show", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_SYSTABLE_RETRIEVE, "retrieve", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "grant", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "auth", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_APPLY_MSG, "mnode-apply", NULL, NULL) - // Requests handled by VNODE TD_NEW_MSG_SEG(TDMT_VND_MSG) - TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "vnode-submit", SSubmitReq, SSubmitRsp) - TD_DEF_MSG_TYPE(TDMT_VND_QUERY, "vnode-query", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_FETCH, "vnode-fetch", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TABLE, "vnode-create-table", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TABLE, "vnode-alter-table", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_DROP_TABLE, "vnode-drop-table", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_UPDATE_TAG_VAL, "vnode-update-tag-val", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp) + TD_DEF_MSG_TYPE(TDMT_VND_QUERY, "query", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_QUERY_CONTINUE, "query-continue", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_QUERY_HEARTBEAT, "query-heartbeat", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_FETCH, "fetch", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TABLE, "create-table", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TABLE, "alter-table", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_DROP_TABLE, "drop-table", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_UPDATE_TAG_VAL, "update-tag-val", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TABLE_META, "vnode-table-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TABLES_META, "vnode-tables-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_STB, "vnode-create-stb", SVCreateStbReq, NULL) @@ -186,67 +180,36 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TOPIC, "vnode-create-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TOPIC, "vnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_TOPIC, "vnode-drop-topic", NULL, NULL) -// TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES, "vnode-show-tables", SVShowTablesReq, SVShowTablesRsp) -// TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES_FETCH, "vnode-show-tables-fetch", SVShowTablesFetchReq, SVShowTablesFetchRsp) - TD_DEF_MSG_TYPE(TDMT_VND_QUERY_CONTINUE, "vnode-query-continue", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_QUERY_HEARTBEAT, "vnode-query-heartbeat", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_EXPLAIN, "vnode-explain", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqPollReq, SMqDataBlkRsp) TD_DEF_MSG_TYPE(TDMT_VND_TASK_DEPLOY, "vnode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_TASK_RUN, "vnode-stream-task-run", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TASK_DISPATCH, "vnode-stream-task-dispatch", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TASK_RECOVER, "vnode-stream-task-recover", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp) TD_DEF_MSG_TYPE(TDMT_VND_GET_TSMA_EXP_WNDS, "vnode-get-tsma-expired-windows", SVGetTsmaExpWndsReq, SVGetTsmaExpWndsRsp) + TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp) + TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "compact", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING, "vnode-sync-ping", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING_REPLY, "vnode-sync-ping-reply", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST, "vnode-sync-client-request", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, "vnode-sync-client-request-reply", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE, "vnode-sync-request-vote", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE_REPLY, "vnode-sync-request-vote-reply", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES, "vnode-sync-append-entries", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, "vnode-sync-append-entries-reply", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_NOOP, "vnode-sync-noop", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_UNKNOWN, "vnode-sync-unknown", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CONFIG_CHANGE, "vnode-sync-config-change", NULL, NULL) - - TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "vnode-alter-config", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "vnode-alter-replica", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "vnode-compact", NULL, NULL) - - TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "vnode-delete-data", SVDeleteReq, SVDeleteRsp) - - // Requests handled by QNODE TD_NEW_MSG_SEG(TDMT_QND_MSG) - // Requests handled by SNODE TD_NEW_MSG_SEG(TDMT_SND_MSG) TD_DEF_MSG_TYPE(TDMT_SND_TASK_DEPLOY, "snode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) - //TD_DEF_MSG_TYPE(TDMT_SND_TASK_EXEC, "snode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp) - //TD_DEF_MSG_TYPE(TDMT_SND_TASK_PIPE_EXEC, "snode-task-pipe-exec", SStreamTaskExecReq, SStreamTaskExecRsp) - //TD_DEF_MSG_TYPE(TDMT_SND_TASK_MERGE_EXEC, "snode-task-merge-exec", SStreamTaskExecReq, SStreamTaskExecRsp) - TD_DEF_MSG_TYPE(TDMT_SND_TASK_RUN, "snode-stream-task-run", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SND_TASK_DISPATCH, "snode-stream-task-dispatch", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SND_TASK_RECOVER, "snode-stream-task-recover", NULL, NULL) - // Requests handled by SCHEDULER TD_NEW_MSG_SEG(TDMT_SCH_MSG) - TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "scheduler-link-broken", NULL, NULL) - - // Monitor info exchange between processes + TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL) + TD_NEW_MSG_SEG(TDMT_MON_MSG) TD_DEF_MSG_TYPE(TDMT_MON_MM_INFO, "monitor-minfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_VM_INFO, "monitor-vinfo", NULL, NULL) @@ -256,6 +219,22 @@ enum { TD_DEF_MSG_TYPE(TDMT_MON_VM_LOAD, "monitor-vload", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_MM_LOAD, "monitor-mload", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_QM_LOAD, "monitor-qload", NULL, NULL) + + TD_NEW_MSG_SEG(TDMT_SYNC_MSG) + TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timeout", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_PING, "sync-ping", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_PING_REPLY, "sync-ping-reply", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_CLIENT_REQUEST, "sync-client-request", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_CLIENT_REQUEST_REPLY, "sync-client-request-reply", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_REQUEST_VOTE, "sync-request-vote", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_REQUEST_VOTE_REPLY, "sync-request-vote-reply", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_APPEND_ENTRIES, "sync-append-entries", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_APPEND_ENTRIES_REPLY, "sync-append-entries-reply", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_NOOP, "sync-noop", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_UNKNOWN, "sync-unknown", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_COMMON_RESPONSE, "sync-common-response", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_APPLY_MSG, "sync-apply-msg", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_CONFIG_CHANGE, "sync-config-change", NULL, NULL) #if defined(TD_MSG_NUMBER_) TDMT_MAX diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index c3b0e54f3da416ccc2e2b2dbd6c05ec356a50a30..d28f87cb6ad8925a26753bfc053e98b0ebe331ae 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -193,65 +193,66 @@ #define TK_REDISTRIBUTE 175 #define TK_SPLIT 176 #define TK_SYNCDB 177 -#define TK_NULL 178 -#define TK_NK_QUESTION 179 -#define TK_NK_ARROW 180 -#define TK_ROWTS 181 -#define TK_TBNAME 182 -#define TK_QSTARTTS 183 -#define TK_QENDTS 184 -#define TK_WSTARTTS 185 -#define TK_WENDTS 186 -#define TK_WDURATION 187 -#define TK_CAST 188 -#define TK_NOW 189 -#define TK_TODAY 190 -#define TK_TIMEZONE 191 -#define TK_COUNT 192 -#define TK_FIRST 193 -#define TK_LAST 194 -#define TK_LAST_ROW 195 -#define TK_BETWEEN 196 -#define TK_IS 197 -#define TK_NK_LT 198 -#define TK_NK_GT 199 -#define TK_NK_LE 200 -#define TK_NK_GE 201 -#define TK_NK_NE 202 -#define TK_MATCH 203 -#define TK_NMATCH 204 -#define TK_CONTAINS 205 -#define TK_JOIN 206 -#define TK_INNER 207 -#define TK_SELECT 208 -#define TK_DISTINCT 209 -#define TK_WHERE 210 -#define TK_PARTITION 211 -#define TK_BY 212 -#define TK_SESSION 213 -#define TK_STATE_WINDOW 214 -#define TK_SLIDING 215 -#define TK_FILL 216 -#define TK_VALUE 217 -#define TK_NONE 218 -#define TK_PREV 219 -#define TK_LINEAR 220 -#define TK_NEXT 221 -#define TK_HAVING 222 -#define TK_ORDER 223 -#define TK_SLIMIT 224 -#define TK_SOFFSET 225 -#define TK_LIMIT 226 -#define TK_OFFSET 227 -#define TK_ASC 228 -#define TK_NULLS 229 -#define TK_ID 230 -#define TK_NK_BITNOT 231 -#define TK_INSERT 232 -#define TK_VALUES 233 -#define TK_IMPORT 234 -#define TK_NK_SEMI 235 -#define TK_FILE 236 +#define TK_DELETE 178 +#define TK_NULL 179 +#define TK_NK_QUESTION 180 +#define TK_NK_ARROW 181 +#define TK_ROWTS 182 +#define TK_TBNAME 183 +#define TK_QSTARTTS 184 +#define TK_QENDTS 185 +#define TK_WSTARTTS 186 +#define TK_WENDTS 187 +#define TK_WDURATION 188 +#define TK_CAST 189 +#define TK_NOW 190 +#define TK_TODAY 191 +#define TK_TIMEZONE 192 +#define TK_COUNT 193 +#define TK_FIRST 194 +#define TK_LAST 195 +#define TK_LAST_ROW 196 +#define TK_BETWEEN 197 +#define TK_IS 198 +#define TK_NK_LT 199 +#define TK_NK_GT 200 +#define TK_NK_LE 201 +#define TK_NK_GE 202 +#define TK_NK_NE 203 +#define TK_MATCH 204 +#define TK_NMATCH 205 +#define TK_CONTAINS 206 +#define TK_JOIN 207 +#define TK_INNER 208 +#define TK_SELECT 209 +#define TK_DISTINCT 210 +#define TK_WHERE 211 +#define TK_PARTITION 212 +#define TK_BY 213 +#define TK_SESSION 214 +#define TK_STATE_WINDOW 215 +#define TK_SLIDING 216 +#define TK_FILL 217 +#define TK_VALUE 218 +#define TK_NONE 219 +#define TK_PREV 220 +#define TK_LINEAR 221 +#define TK_NEXT 222 +#define TK_HAVING 223 +#define TK_ORDER 224 +#define TK_SLIMIT 225 +#define TK_SOFFSET 226 +#define TK_LIMIT 227 +#define TK_OFFSET 228 +#define TK_ASC 229 +#define TK_NULLS 230 +#define TK_ID 231 +#define TK_NK_BITNOT 232 +#define TK_INSERT 233 +#define TK_VALUES 234 +#define TK_IMPORT 235 +#define TK_NK_SEMI 236 +#define TK_FILE 237 #define TK_NK_SPACE 300 #define TK_NK_COMMENT 301 diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 31cdb28690caeb6610d4b5e4ec6307952a0760aa..16c59465cc1f71d1f9e3cc6a2e65d83939247322 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -30,7 +30,7 @@ typedef uint64_t TDRowVerT; typedef int16_t col_id_t; typedef int8_t col_type_t; typedef int32_t col_bytes_t; -typedef int32_t schema_ver_t; +typedef int32_t schema_ver_t; typedef int32_t func_id_t; #pragma pack(push, 1) @@ -49,8 +49,9 @@ typedef struct { #define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v)) #define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) -#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON)) -#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) +#define IS_VAR_DATA_TYPE(t) \ + (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON)) +#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR)) #define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) @@ -142,52 +143,56 @@ typedef struct { } \ } while (0) -#define NUM_TO_STRING(_inputType, _input, _outputBytes, _output) \ - do { \ - switch (_inputType) { \ - case TSDB_DATA_TYPE_TINYINT: \ - snprintf(_output, (int32_t)(_outputBytes), "%d", *(int8_t *)(_input)); \ - break; \ - case TSDB_DATA_TYPE_UTINYINT: \ - snprintf(_output, (int32_t)(_outputBytes), "%d", *(uint8_t *)(_input)); \ - break; \ - case TSDB_DATA_TYPE_SMALLINT: \ - snprintf(_output, (int32_t)(_outputBytes), "%d", *(int16_t *)(_input)); \ - break; \ - case TSDB_DATA_TYPE_USMALLINT: \ - snprintf(_output, (int32_t)(_outputBytes), "%d", *(uint16_t *)(_input)); \ - break; \ - case TSDB_DATA_TYPE_TIMESTAMP: \ - case TSDB_DATA_TYPE_BIGINT: \ - snprintf(_output, (int32_t)(_outputBytes), "%" PRId64, *(int64_t *)(_input)); \ - break; \ - case TSDB_DATA_TYPE_UBIGINT: \ - snprintf(_output, (int32_t)(_outputBytes), "%" PRIu64, *(uint64_t *)(_input)); \ - break; \ - case TSDB_DATA_TYPE_FLOAT: \ - snprintf(_output, (int32_t)(_outputBytes), "%f", *(float *)(_input)); \ - break; \ - case TSDB_DATA_TYPE_DOUBLE: \ - snprintf(_output, (int32_t)(_outputBytes), "%f", *(double *)(_input)); \ - break; \ - case TSDB_DATA_TYPE_UINT: \ - snprintf(_output, (int32_t)(_outputBytes), "%u", *(uint32_t *)(_input)); \ - break; \ - default: \ - snprintf(_output, (int32_t)(_outputBytes), "%d", *(int32_t *)(_input)); \ - break; \ - } \ +#define NUM_TO_STRING(_inputType, _input, _outputBytes, _output) \ + do { \ + switch (_inputType) { \ + case TSDB_DATA_TYPE_TINYINT: \ + snprintf(_output, (int32_t)(_outputBytes), "%d", *(int8_t *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_UTINYINT: \ + snprintf(_output, (int32_t)(_outputBytes), "%d", *(uint8_t *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_SMALLINT: \ + snprintf(_output, (int32_t)(_outputBytes), "%d", *(int16_t *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_USMALLINT: \ + snprintf(_output, (int32_t)(_outputBytes), "%d", *(uint16_t *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_TIMESTAMP: \ + case TSDB_DATA_TYPE_BIGINT: \ + snprintf(_output, (int32_t)(_outputBytes), "%" PRId64, *(int64_t *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_UBIGINT: \ + snprintf(_output, (int32_t)(_outputBytes), "%" PRIu64, *(uint64_t *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_FLOAT: \ + snprintf(_output, (int32_t)(_outputBytes), "%f", *(float *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_DOUBLE: \ + snprintf(_output, (int32_t)(_outputBytes), "%f", *(double *)(_input)); \ + break; \ + case TSDB_DATA_TYPE_UINT: \ + snprintf(_output, (int32_t)(_outputBytes), "%u", *(uint32_t *)(_input)); \ + break; \ + default: \ + snprintf(_output, (int32_t)(_outputBytes), "%d", *(int32_t *)(_input)); \ + break; \ + } \ } while (0) + //TODO: use varchar(0) to represent NULL type +#define IS_VAR_NULL_TYPE(_t, _b) ((_t) == TSDB_DATA_TYPE_VARCHAR && (_b) == 0) #define IS_NULL_TYPE(_t) ((_t) == TSDB_DATA_TYPE_NULL) + #define IS_SIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT) #define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT) #define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) #define IS_INTEGER_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t))) #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) -#define IS_MATHABLE_TYPE(_t) (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) +#define IS_MATHABLE_TYPE(_t) \ + (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) #define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) #define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX) @@ -244,7 +249,7 @@ typedef struct tDataTypeDescriptor { int16_t type; int16_t nameLen; int32_t bytes; - char *name; + char * name; int64_t minValue; int64_t maxValue; int32_t (*compFunc)(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, @@ -277,4 +282,4 @@ void *getDataMax(int32_t type); } #endif -#endif /*_TD_COMMON_TTYPE_H_*/ +#endif /*_TD_COMMON_TTYPE_H_*/ diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index f0e642bc9af8060d0b6bc0380f2c85284f307642..ee237741c37134fb80dbae3623168804e9f4d18e 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -68,6 +68,7 @@ typedef struct SCatalogReq { SArray *pIndex; // element is index name SArray *pUser; // element is SUserAuthInfo bool qNodeRequired; // valid qnode + bool forceUpdate; } SCatalogReq; typedef struct SMetaData { @@ -280,7 +281,7 @@ int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet *epSet); -int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId); +int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId, bool forceUpdate); /** diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 5379a8f712cde79c29ca23e2baac2ac4985450e7..288248422b8288b98d8f0fccaef040186294cb76 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -156,18 +156,6 @@ int64_t qGetQueriedTableUid(qTaskInfo_t tinfo); */ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t tagCondLen, SArray* pTableIdList); -/** - * Create the table group according to the group by tags info - * @param pTableIdList - * @param skey - * @param groupInfo - * @param groupByIndex - * @param numOfIndex - * @return - */ -// int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* -// groupByIndex, int32_t numOfIndex); - /** * Update the table id list of a given query. * @param uid child table uid diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 21b73090554cc951aac82b4d9adb1cb7d847bff2..e8cb363e08fa65385d36762face331f5de5cf1eb 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -61,56 +61,9 @@ typedef struct SFileBlockInfo { #define TSDB_BLOCK_DIST_STEP_ROWS 8 #define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results -#define FUNCTION_TYPE_SCALAR 1 -#define FUNCTION_TYPE_AGG 2 - #define TOP_BOTTOM_QUERY_LIMIT 100 #define FUNCTIONS_NAME_MAX_LENGTH 16 -#define FUNCTION_INVALID_ID -1 -#define FUNCTION_COUNT 0 -#define FUNCTION_SUM 1 -#define FUNCTION_AVG 2 -#define FUNCTION_MIN 3 -#define FUNCTION_MAX 4 -#define FUNCTION_STDDEV 5 -#define FUNCTION_PERCT 6 -#define FUNCTION_APERCT 7 -#define FUNCTION_FIRST 8 -#define FUNCTION_LAST 9 -#define FUNCTION_LAST_ROW 10 -#define FUNCTION_TOP 11 -#define FUNCTION_BOTTOM 12 -#define FUNCTION_SPREAD 13 -#define FUNCTION_TWA 14 -#define FUNCTION_LEASTSQR 15 - -#define FUNCTION_TS 16 -#define FUNCTION_TS_DUMMY 17 -#define FUNCTION_TAG_DUMMY 18 -#define FUNCTION_TS_COMP 19 - -#define FUNCTION_TAG 20 -#define FUNCTION_PRJ 21 - -#define FUNCTION_TAGPRJ 22 -#define FUNCTION_ARITHM 23 -#define FUNCTION_DIFF 24 - -#define FUNCTION_FIRST_DST 25 -#define FUNCTION_LAST_DST 26 -#define FUNCTION_STDDEV_DST 27 -#define FUNCTION_INTERP 28 - -#define FUNCTION_RATE 29 -#define FUNCTION_IRATE 30 -#define FUNCTION_TID_TAG 31 -#define FUNCTION_DERIVATIVE 32 -#define FUNCTION_BLKINFO 33 - - -#define FUNCTION_COV 38 - typedef struct SResultRowEntryInfo { bool initialized:1; // output buffer has been initialized bool complete:1; // query has completed @@ -180,10 +133,9 @@ typedef struct SqlFunctionCtx { char *pOutput; // final result output buffer, point to sdata->data int32_t numOfParams; SFunctParam *param; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param - int64_t *ptsList; // corresponding timestamp array list + int64_t *ptsList; // corresponding timestamp array list, todo remove it SColumnInfoData *pTsOutput; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ int32_t offset; - SVariant tag; struct SResultRowEntryInfo *resultInfo; SSubsidiaryResInfo subsidiaries; SPoint1 start; @@ -210,9 +162,6 @@ enum { typedef struct tExprNode { int32_t nodeType; union { - SSchema *pSchema;// column node - struct SVariant *pVal; // value node - struct {// function node char functionName[FUNCTIONS_NAME_MAX_LENGTH]; // todo refactor int32_t functionId; @@ -255,47 +204,23 @@ struct SScalarParam { int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength, bool isSuperTable); -bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId); - void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num); void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock); bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// fill api -struct SFillInfo; -struct SFillColInfo; - typedef struct SPoint { int64_t key; void * val; } SPoint; -//void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); -//void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); -//void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); -//struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const SValueNode* val); -//bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); -// -//struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, -// SInterval* pInterval, int32_t fillType, -// struct SFillColInfo* pCol, const char* id); -// -//void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); -//int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity); -//int64_t getFillInfoStart(struct SFillInfo *pFillInfo); - int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType); /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // udf api struct SUdfInfo; -void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); -void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); - /** * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf * @return error code diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 180c7e7216153f0cdfd5b4240de89bc586fd9b88..bd601f1d9fa5b9cb1dc287b14970d1d143dd99d3 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -193,8 +193,9 @@ void indexInit(); /* index filter */ typedef struct SIndexMetaArg { - void* metaHandle; void* metaEx; + void* idx; + void* ivtIdx; uint64_t suid; } SIndexMetaArg; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 41c196c916b3cdef97bc68205b8b2ee0f0e472aa..29fabb541833e96124f3ec2a5033df1b2910e9f4 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -180,6 +180,7 @@ typedef enum ENodeType { QUERY_NODE_KILL_CONNECTION_STMT, QUERY_NODE_KILL_QUERY_STMT, QUERY_NODE_KILL_TRANSACTION_STMT, + QUERY_NODE_DELETE_STMT, QUERY_NODE_QUERY, // logic plan node @@ -187,7 +188,7 @@ typedef enum ENodeType { QUERY_NODE_LOGIC_PLAN_JOIN, QUERY_NODE_LOGIC_PLAN_AGG, QUERY_NODE_LOGIC_PLAN_PROJECT, - QUERY_NODE_LOGIC_PLAN_VNODE_MODIF, + QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY, QUERY_NODE_LOGIC_PLAN_EXCHANGE, QUERY_NODE_LOGIC_PLAN_MERGE, QUERY_NODE_LOGIC_PLAN_WINDOW, @@ -209,17 +210,21 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, QUERY_NODE_PHYSICAL_PLAN_MERGE, QUERY_NODE_PHYSICAL_PLAN_SORT, - QUERY_NODE_PHYSICAL_PLAN_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_FILL, QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW, QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION_WINDOW, QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION_WINDOW, QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW, + QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW, QUERY_NODE_PHYSICAL_PLAN_PARTITION, QUERY_NODE_PHYSICAL_PLAN_DISPATCH, QUERY_NODE_PHYSICAL_PLAN_INSERT, + QUERY_NODE_PHYSICAL_PLAN_DELETE, QUERY_NODE_PHYSICAL_SUBPLAN, QUERY_NODE_PHYSICAL_PLAN } ENodeType; diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 5892bffac2f20c0cc177cb9937256f82e45a6956..38c3f150efae86f2ff560e8bbb15008e88107852 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -37,29 +37,30 @@ typedef struct SLogicNode { typedef enum EScanType { SCAN_TYPE_TAG = 1, SCAN_TYPE_TABLE, SCAN_TYPE_SYSTEM_TABLE, SCAN_TYPE_STREAM } EScanType; typedef struct SScanLogicNode { - SLogicNode node; - SNodeList* pScanCols; - SNodeList* pScanPseudoCols; - struct STableMeta* pMeta; - SVgroupsInfo* pVgroupList; - EScanType scanType; - uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count - STimeWindow scanRange; - SName tableName; - bool showRewrite; - double ratio; - SNodeList* pDynamicScanFuncs; - int32_t dataRequired; - int64_t interval; - int64_t offset; - int64_t sliding; - int8_t intervalUnit; - int8_t slidingUnit; - SNode* pTagCond; - int8_t triggerType; - int64_t watermark; - int16_t tsColId; - double filesFactor; + SLogicNode node; + SNodeList* pScanCols; + SNodeList* pScanPseudoCols; + int8_t tableType; + uint64_t tableId; + SVgroupsInfo* pVgroupList; + EScanType scanType; + uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count + STimeWindow scanRange; + SName tableName; + bool showRewrite; + double ratio; + SNodeList* pDynamicScanFuncs; + int32_t dataRequired; + int64_t interval; + int64_t offset; + int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; + SNode* pTagCond; + int8_t triggerType; + int64_t watermark; + int16_t tsColId; + double filesFactor; } SScanLogicNode; typedef struct SJoinLogicNode { @@ -85,12 +86,20 @@ typedef struct SProjectLogicNode { int64_t soffset; } SProjectLogicNode; -typedef struct SVnodeModifLogicNode { - SLogicNode node; - int32_t msgType; - SArray* pDataBlocks; - SVgDataBlocks* pVgDataBlocks; -} SVnodeModifLogicNode; +typedef enum EModifyTableType { MODIFY_TABLE_TYPE_INSERT = 1, MODIFY_TABLE_TYPE_DELETE } EModifyTableType; + +typedef struct SVnodeModifyLogicNode { + SLogicNode node; + EModifyTableType modifyType; + int32_t msgType; + SArray* pDataBlocks; + SVgDataBlocks* pVgDataBlocks; + SNode* pModifyRows; // SColumnNode + uint64_t tableId; + int8_t tableType; // table type + char tableFName[TSDB_TABLE_FNAME_LEN]; + STimeWindow deleteTimeRange; +} SVnodeModifyLogicNode; typedef struct SExchangeLogicNode { SLogicNode node; @@ -106,21 +115,30 @@ typedef struct SMergeLogicNode { typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType; +typedef enum EIntervalAlgorithm { + INTERVAL_ALGO_HASH = 1, + INTERVAL_ALGO_SORT_MERGE, + INTERVAL_ALGO_STREAM_FINAL, + INTERVAL_ALGO_STREAM_SEMI, + INTERVAL_ALGO_STREAM_SINGLE, +} EIntervalAlgorithm; + typedef struct SWindowLogicNode { - SLogicNode node; - EWindowType winType; - SNodeList* pFuncs; - int64_t interval; - int64_t offset; - int64_t sliding; - int8_t intervalUnit; - int8_t slidingUnit; - int64_t sessionGap; - SNode* pTspk; - SNode* pStateExpr; - int8_t triggerType; - int64_t watermark; - double filesFactor; + SLogicNode node; + EWindowType winType; + SNodeList* pFuncs; + int64_t interval; + int64_t offset; + int64_t sliding; + int8_t intervalUnit; + int8_t slidingUnit; + int64_t sessionGap; + SNode* pTspk; + SNode* pStateExpr; + int8_t triggerType; + int64_t watermark; + double filesFactor; + EIntervalAlgorithm intervalAlgo; } SWindowLogicNode; typedef struct SFillLogicNode { @@ -222,6 +240,7 @@ typedef struct STableScanPhysiNode { double ratio; int32_t dataRequired; SNodeList* pDynamicScanFuncs; + SNodeList* pPartitionKeys; int64_t interval; int64_t offset; int64_t sliding; @@ -300,7 +319,10 @@ typedef struct SIntervalPhysiNode { int8_t slidingUnit; } SIntervalPhysiNode; +typedef SIntervalPhysiNode SSortMergeIntervalPhysiNode; typedef SIntervalPhysiNode SStreamIntervalPhysiNode; +typedef SIntervalPhysiNode SStreamFinalIntervalPhysiNode; +typedef SIntervalPhysiNode SStreamSemiIntervalPhysiNode; typedef struct SFillPhysiNode { SPhysiNode node; @@ -328,6 +350,8 @@ typedef struct SStateWinodwPhysiNode { SNode* pStateKey; } SStateWinodwPhysiNode; +typedef SStateWinodwPhysiNode SStreamStateWinodwPhysiNode; + typedef struct SSortPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function @@ -358,6 +382,14 @@ typedef struct SDataInserterNode { char* pData; } SDataInserterNode; +typedef struct SDataDeleterNode { + SDataSinkNode sink; + uint64_t tableId; + int8_t tableType; // table type + char tableFName[TSDB_TABLE_FNAME_LEN]; + STimeWindow deleteTimeRange; +} SDataDeleterNode; + typedef struct SSubplan { ENodeType type; SSubplanId id; // unique id of the subplan diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index e4af78892baaf3757ab58be41fec776e2cb7186f..c2aa86e89f731cb6fdadb2911e805b55454fe2b4 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -60,6 +60,7 @@ typedef struct SColumnNode { int8_t tableType; col_id_t colId; EColumnType colType; // column or tag + bool hasIndex; char dbName[TSDB_DB_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; char tableAlias[TSDB_TABLE_NAME_LEN]; @@ -258,6 +259,7 @@ typedef struct SSetOperator { SNodeList* pOrderByList; // SOrderByExprNode SNode* pLimit; char stmtName[TSDB_TABLE_NAME_LEN]; + uint8_t precision; } SSetOperator; typedef enum ESqlClause { @@ -272,6 +274,17 @@ typedef enum ESqlClause { SQL_CLAUSE_ORDER_BY } ESqlClause; +typedef struct SDeleteStmt { + ENodeType type; // QUERY_NODE_DELETE_STMT + SNode* pFromTable; // FROM clause + SNode* pWhere; // WHERE clause + SNode* pCountFunc; // count the number of rows affected + SNode* pTagIndexCond; // pWhere divided into pTagIndexCond and timeRange + STimeWindow timeRange; + uint8_t precision; + bool deleteZeroRows; +} SDeleteStmt; + typedef enum { PAYLOAD_TYPE_KV = 0, PAYLOAD_TYPE_RAW = 1, @@ -363,8 +376,11 @@ bool nodesIsRegularOp(const SOperatorNode* pOp); void* nodesGetValueFromNode(SValueNode* pNode); int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value); char* nodesGetStrValueFromNode(SValueNode* pNode); -char* getFillModeString(EFillMode mode); -void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal); +void nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal); + +char* nodesGetFillModeString(EFillMode mode); +int32_t nodesMergeConds(SNode** pDst, SNodeList** pSrc); +int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagCond, SNode** pOtherCond); #ifdef __cplusplus } diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 6e14445ac7abc12eb4ccca7a9f08f3e7ff43ddb8..af30ec4c6bf7d657dfdec1af49f871eed38b53d7 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -36,7 +36,7 @@ typedef struct SPlanContext { int64_t watermark; char* pMsg; int32_t msgLen; - double filesFactor; + double filesFactor; } SPlanContext; // Create the physical plan for the query, according to the AST. diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 45a7e9a29f3457a68e9998659237a9e0d70d39ab..a17be4484604206a133d95966b7c9d5e812a4956 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -222,7 +222,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t || (_type) == TDMT_VND_DROP_TABLE || (_type) == TDMT_VND_DROP_STB) #define NEED_SCHEDULER_RETRY_ERROR(_code) \ - ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) + ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) #define REQUEST_MAX_TRY_TIMES 1 diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 91cf975a56660cd13a9fac992cb59c79bd2362b4..aa20082fe02c75c74306adcbdbddc250384ac116 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -58,6 +58,8 @@ typedef struct { int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, const SMsgCb *pMsgCb); +int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg); + int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts); int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts); diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 331b78769029fb97764f81ce6bb646f28854918a..22706f0953298a95b40552d068189fffef24f116 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -100,10 +100,12 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryPlan *pDag, in */ int32_t schedulerFetchRows(int64_t job, void **data); -int32_t schedulerAsyncFetchRows(int64_t job, schedulerFetchCallback fp, void* param); +void schedulerAsyncFetchRows(int64_t job, schedulerFetchCallback fp, void* param); int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub); +void schedulerStopQueryHb(void *pTrans); + /** * Cancel query job @@ -121,8 +123,6 @@ void schedulerFreeJob(int64_t job); void schedulerDestroy(void); void schdExecCallback(SQueryResult* pResult, void* param, int32_t code); -void schdFetchCallback(void* pResult, void* param, int32_t code); - #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 7335d174a4cf67917f58d4219a5f576a4c3cff86..0b331abf8af5cb683fd287cc84b50b81391e8b08 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -40,9 +40,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error //common & util -#define TSDB_CODE_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) -#define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0002) -#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0003) +#define TSDB_CODE_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0003) +#define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0004) +#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) #define TSDB_CODE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0010) #define TSDB_CODE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0011) #define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x0012) @@ -235,6 +235,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x03AC) #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AD) #define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AE) +#define TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03AF) #define TSDB_CODE_MND_SINGLE_STB_MODE_DB TAOS_DEF_ERROR_CODE(0, 0x03B0) // mnode-infoSchema @@ -563,7 +564,9 @@ int32_t* taosGetErrno(); //scheduler&qworker #define TSDB_CODE_SCH_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2501) #define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502) -#define TSDB_CODE_QW_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x2503) +#define TSDB_CODE_SCH_IGNORE_ERROR TAOS_DEF_ERROR_CODE(0, 0x2503) +#define TSDB_CODE_SCH_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x2504) +#define TSDB_CODE_QW_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x2550) //parser #define TSDB_CODE_PAR_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x2600) @@ -647,6 +650,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_COL_JSON TAOS_DEF_ERROR_CODE(0, 0x2652) #define TSDB_CODE_PAR_VALUE_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x2653) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2654) +#define TSDB_CODE_PAR_INVALID_DELETE_WHERE TAOS_DEF_ERROR_CODE(0, 0x2655) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/include/util/tdef.h b/include/util/tdef.h index 0ae22d195395f6225dddf33c52a99046ad41354d..ebfb7b696aa5d156dbfa1f07f74470c37c4c63f9 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -129,7 +129,7 @@ typedef enum EOperatorType { OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV, - OP_TYPE_MOD, + OP_TYPE_REM, // unary arithmetic operator OP_TYPE_MINUS, OP_TYPE_ASSIGN, diff --git a/include/util/tlist.h b/include/util/tlist.h index 43833d7ecd84f09643546f3f3fa838edbd1dabf1..1954bda145a48f249875bda8ea3389b4fbed22be 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -229,7 +229,7 @@ int32_t tdListAppend(SList *list, void *data); SListNode *tdListPopHead(SList *list); SListNode *tdListPopTail(SList *list); SListNode *tdListGetHead(SList *list); -SListNode *tsListGetTail(SList *list); +SListNode *tdListGetTail(SList *list); SListNode *tdListPopNode(SList *list, SListNode *node); void tdListMove(SList *src, SList *dst); void tdListDiscard(SList *list); diff --git a/include/util/tutil.h b/include/util/tutil.h index 444a893a888abb5c28de0c70476af9beee32cc10..b29e1f7cfa889944b53b3603bd0c8fefc8ece1f0 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -57,11 +57,13 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar tMD5Init(&context); tMD5Update(&context, inBuf, (uint32_t)len); tMD5Final(&context); + char buf[TSDB_PASSWORD_LEN + 1]; - sprintf(target, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + sprintf(buf, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], context.digest[12], context.digest[13], context.digest[14], context.digest[15]); + memcpy(target, buf, TSDB_PASSWORD_LEN); } #ifdef __cplusplus diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 3ff61eca25fa54692569ff2db9a53813ce6b7a36..eaf18884c62caa0892a8aad6eeb22b4072567d4c 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -45,6 +45,7 @@ extern "C" { #define ERROR_MSG_BUF_DEFAULT_SIZE 512 #define HEARTBEAT_INTERVAL 1500 // ms +#define SYNC_ON_TOP_OF_ASYNC 0 enum { RES_TYPE__QUERY = 1, @@ -56,11 +57,6 @@ enum { typedef struct SAppInstInfo SAppInstInfo; -typedef struct { - void* param; - SClientHbReq* req; -} SHbConnInfo; - typedef struct { char* key; // statistics @@ -70,11 +66,8 @@ typedef struct { int64_t startTime; // ctl SRWLatch lock; // lock is used in serialization - // connection SAppInstInfo* pAppInstInfo; - // info SHashObj* activeInfo; // hash - SHashObj* connInfo; // hash } SAppHbMgr; typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); @@ -183,7 +176,9 @@ typedef struct SReqResultInfo { typedef struct SRequestSendRecvBody { tsem_t rspSem; // not used now - void* fp; + __taos_async_fn_t queryFp; + __taos_async_fn_t fetchFp; + void* param; SDataBuf requestMsg; int64_t queryJob; // query job, created according to sql query DAG. struct SQueryPlan* pDag; // the query dag, generated according to the sql statement. @@ -219,13 +214,21 @@ typedef struct SRequestObj { SRequestSendRecvBody body; } SRequestObj; +typedef struct SSyncQueryParam { + tsem_t sem; + SRequestObj* pRequest; +} SSyncQueryParam; + +void* doAsyncFetchRow(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); + void doSetOneRowPtr(SReqResultInfo* pResultInfo); void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, bool freeAfterUse); void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); void doFreeReqResultInfo(SReqResultInfo* pResInfo); +SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen); static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; @@ -271,7 +274,7 @@ int32_t releaseTscObj(int64_t rid); uint64_t generateRequestId(); -void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); +void* createRequest(STscObj* pObj, void* param, int32_t type); void destroyRequest(SRequestObj* pRequest); SRequestObj* acquireRequest(int64_t rid); int32_t releaseRequest(int64_t rid); @@ -314,14 +317,12 @@ void appHbMgrCleanup(void); int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); -int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen); - // --- mq void hbMgrInitMqHbRspHandle(); -SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery, void** res); -int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList); +SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res); int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList); +void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery); int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest); int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 19847d9aa2fd04bd8b3f7ff13c0d8462d3c2b9cf..be0a41ba40e417d2c7596ae985d056827c3f46f6 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -13,10 +13,11 @@ * along with this program. If not, see . */ +#include "os.h" #include "catalog.h" +#include "functionMgt.h" #include "clientInt.h" #include "clientLog.h" -#include "os.h" #include "query.h" #include "scheduler.h" #include "tcache.h" @@ -129,8 +130,13 @@ void destroyTscObj(void *pObj) { SClientHbKey connKey = {.tscRid = pTscObj->id, .connType = pTscObj->connType}; hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); - atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); closeAllRequests(pTscObj->pRequests); + schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); + if (0 == connNum) { + // TODO + //closeTransporter(pTscObj); + } tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); taosThreadMutexDestroy(&pTscObj->mutex); taosMemoryFreeClear(pTscObj); @@ -171,7 +177,7 @@ STscObj *acquireTscObj(int64_t rid) { return (STscObj *)taosAcquireRef(clientCon int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, rid); } -void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) { +void *createRequest(STscObj *pObj, void *param, int32_t type) { assert(pObj != NULL); SRequestObj *pRequest = (SRequestObj *)taosMemoryCalloc(1, sizeof(SRequestObj)); @@ -185,9 +191,10 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty pRequest->requestId = generateRequestId(); pRequest->metric.start = taosGetTimestampUs(); + pRequest->body.param = param; + pRequest->type = type; pRequest->pTscObj = pObj; - pRequest->body.fp = fp; // not used it yet pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); pRequest->msgBufLen = ERROR_MSG_BUF_DEFAULT_SIZE; tsem_init(&pRequest->body.rspSem, 0, 0); @@ -221,6 +228,10 @@ static void doDestroyRequest(void *p) { taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); + if (pRequest->body.queryJob != 0) { + schedulerFreeJob(pRequest->body.queryJob); + } + taosMemoryFreeClear(pRequest->msgBuf); taosMemoryFreeClear(pRequest->sqlstr); taosMemoryFreeClear(pRequest->pDb); @@ -228,10 +239,6 @@ static void doDestroyRequest(void *p) { doFreeReqResultInfo(&pRequest->body.resInfo); qDestroyQueryPlan(pRequest->body.pDag); - if (pRequest->body.queryJob != 0) { - schedulerFreeJob(pRequest->body.queryJob); - } - taosArrayDestroy(pRequest->tableList); taosArrayDestroy(pRequest->dbList); @@ -288,6 +295,7 @@ void taos_init_imp(void) { taosSetCoreDump(true); initTaskQueue(); + fmFuncMgtInit(); clientConnRefPool = taosOpenRef(200, destroyTscObj); clientReqRefPool = taosOpenRef(40960, doDestroyRequest); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 09c3d269c703d6e2dc78cbef49a7790c98f34245..a4c109ee17b584a41a322a2cb868e42c446fdf58 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -129,9 +129,9 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo } static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { - SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey)); - if (NULL == info) { - tscWarn("fail to get connInfo, may be dropped, refId:%" PRIx64 ", type:%d", pRsp->connKey.tscRid, + SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &pRsp->connKey, sizeof(SClientHbKey)); + if (NULL == pReq) { + tscWarn("pReq to get activeInfo, may be dropped, refId:%" PRIx64 ", type:%d", pRsp->connKey.tscRid, pRsp->connKey.connType); return TSDB_CODE_SUCCESS; } @@ -181,12 +181,11 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { break; } - int64_t *clusterId = (int64_t *)info->param; struct SCatalog *pCatalog = NULL; - int32_t code = catalogGetHandle(*clusterId, &pCatalog); + int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { - tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code)); break; } @@ -199,12 +198,11 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { break; } - int64_t *clusterId = (int64_t *)info->param; struct SCatalog *pCatalog = NULL; - int32_t code = catalogGetHandle(*clusterId, &pCatalog); + int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { - tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code)); break; } @@ -217,12 +215,11 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { break; } - int64_t *clusterId = (int64_t *)info->param; struct SCatalog *pCatalog = NULL; - int32_t code = catalogGetHandle(*clusterId, &pCatalog); + int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { - tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code)); break; } @@ -547,13 +544,10 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { pOneReq = taosArrayPush(pBatchReq->reqs, pOneReq); - SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pOneReq->connKey, sizeof(SClientHbKey)); - if (info) { - code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, info->param, pOneReq); - if (code) { - pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); - continue; - } + code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, &pOneReq->clusterId, pOneReq); + if (code) { + pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); + continue; } //hbClearClientHbReq(pOneReq); @@ -569,23 +563,6 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { return pBatchReq; } -void hbClearReqInfo(SAppHbMgr *pAppHbMgr) { - void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); - while (pIter != NULL) { - SClientHbReq *pOneReq = pIter; - - tFreeReqKvHash(pOneReq->info); - taosHashClear(pOneReq->info); - - if (pOneReq->query) { - taosArrayDestroy(pOneReq->query->queryDesc); - taosMemoryFreeClear(pOneReq->query); - } - - pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); - } -} - void hbThreadFuncUnexpectedStopped(void) { atomic_store_8(&clientHbMgr.threadStop, 2); } @@ -715,14 +692,6 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { } taosHashSetFreeFp(pAppHbMgr->activeInfo, tFreeClientHbReq); - // init getInfoFunc - pAppHbMgr->connInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); - - if (pAppHbMgr->connInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pAppHbMgr); - return NULL; - } taosThreadMutexLock(&clientHbMgr.lock); taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr); @@ -745,15 +714,6 @@ void appHbMgrCleanup(void) { taosHashCleanup(pTarget->activeInfo); pTarget->activeInfo = NULL; - pIter = taosHashIterate(pTarget->connInfo, NULL); - while (pIter != NULL) { - SHbConnInfo *info = pIter; - taosMemoryFree(info->param); - pIter = taosHashIterate(pTarget->connInfo, pIter); - } - taosHashCleanup(pTarget->connInfo); - pTarget->connInfo = NULL; - taosMemoryFree(pTarget->key); taosMemoryFree(pTarget); } @@ -791,7 +751,7 @@ void hbMgrCleanUp() { clientHbMgr.appHbMgrs = NULL; } -int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) { +int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, int64_t clusterId) { // init hash in activeinfo void *data = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); if (data != NULL) { @@ -799,17 +759,11 @@ int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo * } SClientHbReq hbReq = {0}; hbReq.connKey = connKey; + hbReq.clusterId = clusterId; //hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); taosHashPut(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey), &hbReq, sizeof(SClientHbReq)); - // init hash - if (info != NULL) { - SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); - info->req = pReq; - taosHashPut(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey), info, sizeof(SHbConnInfo)); - } - atomic_add_fetch_32(&pAppHbMgr->connKeyCnt, 1); return 0; } @@ -819,15 +773,10 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in .tscRid = tscRefId, .connType = connType, }; - SHbConnInfo info = {0}; switch (connType) { case CONN_TYPE__QUERY: { - int64_t *pClusterId = taosMemoryMalloc(sizeof(int64_t)); - *pClusterId = clusterId; - - info.param = pClusterId; - return hbRegisterConnImpl(pAppHbMgr, connKey, &info); + return hbRegisterConnImpl(pAppHbMgr, connKey, clusterId); } case CONN_TYPE__TMQ: { return 0; @@ -844,26 +793,10 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); } - SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); - if (info) { - taosMemoryFree(info->param); - taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); - } - - if (NULL == pReq || NULL == info) { + if (NULL == pReq) { return; } atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); } -int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *key, void *value, int32_t keyLen, - int32_t valueLen) { - // find req by connection id - SClientHbReq *pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); - ASSERT(pReq != NULL); - - taosHashPut(pReq->info, key, keyLen, value, valueLen); - - return 0; -} diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index e7b4fb06577d2f34a5ca9815754c00e904d873c8..916017543d6d951fa4838c92f37e80315f60767e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -133,7 +133,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, } int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest) { - *pRequest = createRequest(pTscObj, NULL, NULL, TSDB_SQL_SELECT); + *pRequest = createRequest(pTscObj, NULL, TSDB_SQL_SELECT); if (*pRequest == NULL) { tscError("failed to malloc sqlObj"); return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -192,6 +192,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC setResPrecision(&pRequest->body.resInfo, (*pQuery)->precision); } } + if (TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) { TSWAP(pRequest->dbList, (*pQuery)->pDbList); TSWAP(pRequest->tableList, (*pQuery)->pTableList); @@ -230,6 +231,29 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { return TSDB_CODE_SUCCESS; } +static SAppInstInfo* getAppInfo(SRequestObj* pRequest) { + return pRequest->pTscObj->pAppInfo; +} + +int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { + // drop table if exists not_exists_table + if (NULL == pQuery->pCmdMsg) { + return TSDB_CODE_SUCCESS; + } + + SCmdMsgInfo* pMsgInfo = pQuery->pCmdMsg; + pRequest->type = pMsgInfo->msgType; + pRequest->body.requestMsg = (SDataBuf){.pData = pMsgInfo->pMsg, .len = pMsgInfo->msgLen, .handle = NULL}; + pMsgInfo->pMsg = NULL; // pMsg transferred to SMsgSendInfo management + + SAppInstInfo* pAppInfo = getAppInfo(pRequest); + SMsgSendInfo* pSendMsg = buildMsgInfoImpl(pRequest); + + int64_t transporterId = 0; + asyncSendMsgToServer(pAppInfo->pTransporter, &pMsgInfo->epSet, &transporterId, pSendMsg); + return TSDB_CODE_SUCCESS; +} + int compareQueryNodeLoad(const void* elem1, const void* elem2) { SQueryNodeLoad* node1 = (SQueryNodeLoad*)elem1; SQueryNodeLoad* node2 = (SQueryNodeLoad*)elem2; @@ -286,9 +310,11 @@ int32_t getQnodeList(SRequestObj* pRequest, SArray** pNodeList) { int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray** pNodeList) { pRequest->type = pQuery->msgType; + SAppInstInfo* pAppInfo = getAppInfo(pRequest); + SPlanContext cxt = {.queryId = pRequest->requestId, .acctId = pRequest->pTscObj->acctId, - .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), + .mgmtEpSet = getEpSet_s(&pAppInfo->mgmtEp), .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, .pMsg = pRequest->msgBuf, @@ -298,6 +324,7 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra if (TSDB_CODE_SUCCESS == code) { code = qCreateQueryPlan(&cxt, pPlan, *pNodeList); } + return code; } @@ -352,7 +379,7 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod pRequest->body.resInfo.execRes = res.res; - while (true) { + while (true) { if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); @@ -388,7 +415,7 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) { void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; - SQueryResult res = {.code = 0, .numOfRows = 0}; + SQueryResult res = {0}; int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, pRequest->metric.start, &res); @@ -417,10 +444,6 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList return pRequest->code; } -int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList) { - return getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList); -} - int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet *epset) { int32_t code = 0; SArray* pArray = NULL; @@ -428,19 +451,19 @@ int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog if (pRsp->nBlocks <= 0) { return TSDB_CODE_SUCCESS; } - + pArray = taosArrayInit(pRsp->nBlocks, sizeof(STbSVersion)); if (NULL == pArray) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY; } - + for (int32_t i = 0; i < pRsp->nBlocks; ++i) { SSubmitBlkRsp* blk = pRsp->pBlocks + i; if (NULL == blk->tblFName || 0 == blk->tblFName[0]) { continue; } - + STbSVersion tbSver = {.tbFName = blk->tblFName, .sver = blk->sver}; taosArrayPush(pArray, &tbSver); } @@ -450,7 +473,7 @@ int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog _return: taosArrayDestroy(pArray); - return code; + return code; } int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet *epset) { @@ -461,13 +484,13 @@ int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, if (tbNum <= 0) { return TSDB_CODE_SUCCESS; } - + pArray = taosArrayInit(tbNum, sizeof(STbSVersion)); if (NULL == pArray) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY; } - + for (int32_t i = 0; i < tbNum; ++i) { STbVerInfo* tbInfo = taosArrayGet(pTbArray, i); STbSVersion tbSver = {.tbFName = tbInfo->tbFName, .sver = tbInfo->sversion, .tver = tbInfo->tversion}; @@ -479,7 +502,7 @@ int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, _return: taosArrayDestroy(pArray); - return code; + return code; } int32_t handleAlterTbExecRes(void* res, SCatalog* pCatalog) { @@ -490,19 +513,19 @@ int32_t handleExecRes(SRequestObj* pRequest) { if (NULL == pRequest->body.resInfo.execRes.res) { return TSDB_CODE_SUCCESS; } - + int32_t code = 0; SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code) { return code; } - + SEpSet epset = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); SQueryExecRes* pRes = &pRequest->body.resInfo.execRes; - + switch (pRes->msgType) { - case TDMT_VND_ALTER_TABLE: + case TDMT_VND_ALTER_TABLE: case TDMT_MND_ALTER_STB: { code = handleAlterTbExecRes(pRes->res, pCatalog); break; @@ -510,7 +533,7 @@ int32_t handleExecRes(SRequestObj* pRequest) { case TDMT_VND_SUBMIT: { code = handleSubmitExecRes(pRequest, pRes->res, pCatalog, &epset); break; - } + } case TDMT_VND_QUERY: { code = handleQueryExecRes(pRequest, pRes->res, pCatalog, &epset); break; @@ -523,30 +546,37 @@ int32_t handleExecRes(SRequestObj* pRequest) { return code; } -SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery, void** res) { - if (TSDB_CODE_SUCCESS == code) { - switch (pQuery->execMode) { - case QUERY_EXEC_MODE_LOCAL: - code = execLocalCmd(pRequest, pQuery); - break; - case QUERY_EXEC_MODE_RPC: - code = execDdlQuery(pRequest, pQuery); - break; - case QUERY_EXEC_MODE_SCHEDULE: { - SArray* pNodeList = NULL; - code = getPlan(pRequest, pQuery, &pRequest->body.pDag, &pNodeList); - if (TSDB_CODE_SUCCESS == code) { - code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList); - } - taosArrayDestroy(pNodeList); - break; +void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) { + SRequestObj* pRequest = (SRequestObj*) param; + + // return to client + pRequest->body.queryFp(pRequest->body.param, pRequest, code); +} + +SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res) { + int32_t code = 0; + + switch (pQuery->execMode) { + case QUERY_EXEC_MODE_LOCAL: + code = execLocalCmd(pRequest, pQuery); + break; + case QUERY_EXEC_MODE_RPC: + code = execDdlQuery(pRequest, pQuery); + break; + case QUERY_EXEC_MODE_SCHEDULE: { + SArray* pNodeList = NULL; + code = getPlan(pRequest, pQuery, &pRequest->body.pDag, &pNodeList); + if (TSDB_CODE_SUCCESS == code) { + code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList); } - case QUERY_EXEC_MODE_EMPTY_RESULT: - pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; - break; - default: - break; + taosArrayDestroy(pNodeList); + break; } + case QUERY_EXEC_MODE_EMPTY_RESULT: + pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + break; + default: + break; } if (!keepQuery) { @@ -583,7 +613,70 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { return pRequest; } - return launchQueryImpl(pRequest, pQuery, code, false, NULL); + return launchQueryImpl(pRequest, pQuery, false, NULL); +} + +void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) { + void* pRes = NULL; + + int32_t code = 0; + switch (pQuery->execMode) { + case QUERY_EXEC_MODE_LOCAL: + code = execLocalCmd(pRequest, pQuery); + break; + case QUERY_EXEC_MODE_RPC: + code = asyncExecDdlQuery(pRequest, pQuery); + break; + case QUERY_EXEC_MODE_SCHEDULE: { + SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); + + pRequest->type = pQuery->msgType; + + SPlanContext cxt = {.queryId = pRequest->requestId, + .acctId = pRequest->pTscObj->acctId, + .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), + .pAstRoot = pQuery->pRoot, + .showRewrite = pQuery->showRewrite, + .pMsg = pRequest->msgBuf, + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE}; + + SAppInstInfo* pAppInfo = getAppInfo(pRequest); + + if (TSDB_CODE_SUCCESS == code) { + code = qCreateQueryPlan(&cxt, &pRequest->body.pDag, pNodeList); + } + + if (TSDB_CODE_SUCCESS == code) { + schedulerAsyncExecJob(pAppInfo->pTransporter, pNodeList, pRequest->body.pDag, &pRequest->body.queryJob, + pRequest->sqlstr, pRequest->metric.start, schedulerExecCb, pRequest); + // if (NULL != pRes) { + // code = validateSversion(pRequest, pRes); + // } + } + + taosArrayDestroy(pNodeList); + break; + } + case QUERY_EXEC_MODE_EMPTY_RESULT: + pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + break; + default: + break; + } + + // if (!keepQuery) { + // qDestroyQuery(pQuery); + // } + + if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { + pRequest->code = terrno; + } + + // if (res) { + // *res = pRes; + // } else { +// freeRequestRes(pRequest, pRes); +// pRes = NULL; } int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { @@ -666,17 +759,6 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) { return pRequest; } -TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) { - STscObj* pTscObj = (STscObj*)taos; - if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { - tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); - terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - return NULL; - } - - return execQuery(pTscObj, sql, sqlLen); -} - int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) { pEpSet->version = 0; @@ -726,7 +808,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t return pTscObj; } - SRequestObj* pRequest = createRequest(pTscObj, fp, param, TDMT_MND_CONNECT); + SRequestObj* pRequest = createRequest(pTscObj, param, TDMT_MND_CONNECT); if (pRequest == NULL) { destroyTscObj(pTscObj); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -843,8 +925,7 @@ void updateTargetEpSet(SMsgSendInfo* pSendInfo, STscObj* pTscObj, SRpcMsg* pMsg, void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; assert(pMsg->info.ahandle != NULL); - SRequestObj* pRequest = NULL; - STscObj* pTscObj = NULL; + STscObj* pTscObj = NULL; if (pSendInfo->requestObjRefId != 0) { SRequestObj* pRequest = (SRequestObj*)taosAcquireRef(clientReqRefPool, pSendInfo->requestObjRefId); @@ -923,7 +1004,7 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) { int32_t bytes = pResultInfo->fields[i].bytes; if (IS_VAR_DATA_TYPE(type)) { - if (pCol->offset[pResultInfo->current] != -1) { + if (!IS_VAR_NULL_TYPE(type, bytes) && pCol->offset[pResultInfo->current] != -1) { char* pStart = pResultInfo->pCol[i].offset[pResultInfo->current] + pResultInfo->pCol[i].pData; pResultInfo->length[i] = varDataLen(pStart); @@ -944,7 +1025,7 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) { } } -void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) { +void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) { assert(pRequest != NULL); SReqResultInfo* pResultInfo = &pRequest->body.resInfo; @@ -955,17 +1036,8 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU return NULL; } - tsem_init(&schdRspSem, 0, 0); - SReqResultInfo* pResInfo = &pRequest->body.resInfo; - SSchdFetchParam param = {.pData = (void**)&pResInfo->pData, .code = &pRequest->code}; - pRequest->code = schedulerAsyncFetchRows(pRequest->body.queryJob, schdFetchCallback, ¶m); - if (pRequest->code != TSDB_CODE_SUCCESS) { - pResultInfo->numOfRows = 0; - return NULL; - } - - tsem_wait(&schdRspSem); + pRequest->code = schedulerFetchRows(pRequest->body.queryJob, (void**)&pResInfo->pData); if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; return NULL; @@ -994,8 +1066,12 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU return pResultInfo->row; } -void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) { - // return doAsyncFetchRows(pRequest, setupOneRowPtr, convertUcs4); +static void syncFetchFn(void* param, TAOS_RES* res, int32_t numOfRows) { + SSyncQueryParam* pParam = param; + tsem_post(&pParam->sem); +} + +void* doAsyncFetchRow(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) { assert(pRequest != NULL); SReqResultInfo* pResultInfo = &pRequest->body.resInfo; @@ -1006,29 +1082,12 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) return NULL; } - SReqResultInfo* pResInfo = &pRequest->body.resInfo; - pRequest->code = schedulerFetchRows(pRequest->body.queryJob, (void**)&pResInfo->pData); - if (pRequest->code != TSDB_CODE_SUCCESS) { - pResultInfo->numOfRows = 0; - return NULL; - } - - pRequest->code = - setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResInfo->pData, convertUcs4, true); - if (pRequest->code != TSDB_CODE_SUCCESS) { - pResultInfo->numOfRows = 0; - return NULL; - } - - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, - pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId); - - if (pResultInfo->numOfRows == 0) { - return NULL; - } + SSyncQueryParam* pParam = pRequest->body.param; + taos_fetch_rows_a(pRequest, syncFetchFn, pParam); + tsem_wait(&pParam->sem); } - if (setupOneRowPtr) { + if (pRequest->code == TSDB_CODE_SUCCESS && pResultInfo->numOfRows > 0 && setupOneRowPtr) { doSetOneRowPtr(pResultInfo); pResultInfo->current += 1; } @@ -1175,7 +1234,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); varDataSetLen(dst, strlen(varDataVal(dst))); } else if (jsonInnerType == TD_TAG_JSON) { - char* jsonString = parseTagDatatoJson(jsonInnerData); + char* jsonString = parseTagDatatoJson(pStart); STR_TO_VARSTR(dst, jsonString); taosMemoryFree(jsonString); } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index e144885e9efc4b3eca7c806996b77ad416d70161..bddc32ea207ce52802e0d3e01fa20223d1c71916 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -66,10 +66,11 @@ void taos_cleanup(void) { hbMgrCleanUp(); - rpcCleanup(); catalogDestroy(); schedulerDestroy(); + rpcCleanup(); + tscInfo("all local resources released"); taosCleanupCfg(); taosCloseLog(); @@ -175,12 +176,39 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { return pResInfo->userFields; } +static void syncQueryFn(void* param, void* res, int32_t code) { + SSyncQueryParam* pParam = param; + pParam->pRequest = res; + pParam->pRequest->code = code; + + tsem_post(&pParam->sem); +} + TAOS_RES *taos_query(TAOS *taos, const char *sql) { if (taos == NULL || sql == NULL) { return NULL; } - return taos_query_l(taos, sql, (int32_t)strlen(sql)); + STscObj* pTscObj = (STscObj*)taos; + +#if SYNC_ON_TOP_OF_ASYNC + SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(struct SSyncQueryParam)); + tsem_init(¶m->sem, 0, 0); + + taos_query_a(pTscObj, sql, syncQueryFn, param); + tsem_wait(¶m->sem); + + return param->pRequest; +#else + size_t sqlLen = strlen(sql); + if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + return NULL; + } + + return execQuery(pTscObj, sql, sqlLen); +#endif } TAOS_ROW taos_fetch_row(TAOS_RES *res) { @@ -195,7 +223,11 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { return NULL; } +#if SYNC_ON_TOP_OF_ASYNC + return doAsyncFetchRow(pRequest, true, true); +#else return doFetchRows(pRequest, true, true); +#endif } else if (TD_RES_TMQ(res)) { SMqRspObj *msg = ((SMqRspObj *)res); @@ -205,6 +237,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { } else { pResultInfo = tmqGetCurResInfo(res); } + if (pResultInfo->current < pResultInfo->numOfRows) { doSetOneRowPtr(pResultInfo); pResultInfo->current += 1; @@ -563,38 +596,195 @@ const char *taos_get_server_info(TAOS *taos) { return pTscObj->ver; } +typedef struct SqlParseWrapper { + SParseContext* pCtx; + SCatalogReq catalogReq; + SRequestObj* pRequest; + SQuery* pQuery; +} SqlParseWrapper; + +void retrieveMetaCallback(SMetaData* pResultMeta, void* param, int32_t code) { + SqlParseWrapper *pWrapper = (SqlParseWrapper*) param; + SQuery* pQuery = pWrapper->pQuery; + SRequestObj* pRequest = pWrapper->pRequest; + + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + if (pQuery->haveResultSet) { + setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, (pQuery)->numOfResCols); + setResPrecision(&pRequest->body.resInfo, (pQuery)->precision); + } + + TSWAP(pRequest->dbList, (pQuery)->pDbList); + TSWAP(pRequest->tableList, (pQuery)->pTableList); + + taosMemoryFree(pWrapper); + launchAsyncQuery(pRequest, pQuery); + return; + + _error: + taosMemoryFree(pWrapper); + tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, reqId:%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); + pRequest->code = code; + pRequest->body.queryFp(pRequest->body.param, pRequest, code); +} + +// todo add retry before return user's callback void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { + ASSERT(fp != NULL); + if (taos == NULL || sql == NULL) { - fp(param, NULL, TSDB_CODE_INVALID_PARA); + terrno = TSDB_CODE_INVALID_PARA; + fp(param, NULL, terrno); return; } - SRequestObj* pRequest = NULL; + size_t sqlLen = strlen(sql); + if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + + fp(param, NULL, terrno); + return; + } + + SRequestObj *pRequest = NULL; int32_t retryNum = 0; int32_t code = 0; - size_t sqlLen = strlen(sql); + // while (retryNum++ < REQUEST_MAX_TRY_TIMES) { + code = buildRequest(taos, sql, sqlLen, &pRequest); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + fp(param, NULL, code); + return; + } - while (retryNum++ < REQUEST_MAX_TRY_TIMES) { - pRequest = launchQuery(taos, sql, sqlLen); - if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { - break; - } + pRequest->body.queryFp = fp; + pRequest->body.param = param; - code = refreshMeta(taos, pRequest); - if (code) { - pRequest->code = code; - break; - } + STscObj *pTscObj = pRequest->pTscObj; - destroyRequest(pRequest); + SParseContext* pCxt = taosMemoryCalloc(1, sizeof(SParseContext)); + *pCxt = (SParseContext){.requestId = pRequest->requestId, + .acctId = pTscObj->acctId, + .db = pRequest->pDb, + .topicQuery = false, + .pSql = pRequest->sqlstr, + .sqlLen = pRequest->sqlLen, + .pMsg = pRequest->msgBuf, + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, + .pTransporter = pTscObj->pAppInfo->pTransporter, + .pStmtCb = NULL, + .pUser = pTscObj->user, + .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), + .async = true,}; + + pCxt->mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCxt->pCatalog); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + SQuery * pQuery = NULL; + SCatalogReq catalogReq = {0}; + code = qParseSqlSyntax(pCxt, &pQuery, &catalogReq); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + SqlParseWrapper *pWrapper = taosMemoryCalloc(1, sizeof(SqlParseWrapper)); + pWrapper->pCtx = pCxt; + pWrapper->pQuery = pQuery; + pWrapper->pRequest = pRequest; + pWrapper->catalogReq = catalogReq; + + code = catalogAsyncGetAllMeta(pCxt->pCatalog, pCxt->pTransporter, &pCxt->mgmtEpSet, pRequest->requestId, + &catalogReq, retrieveMetaCallback, pWrapper, &pRequest->body.queryJob); + + if (code != TSDB_CODE_SUCCESS) { + goto _error; } + return; + + // todo handle the retry process + + // if (TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) { + // TSWAP(pRequest->dbList, (pQuery)->pDbList); + // TSWAP(pRequest->tableList, (pQuery)->pTableList); + // } + + _error: + terrno = code; + pRequest->code = code; fp(param, pRequest, code); } +static void fetchCallback(void* pResult, void* param, int32_t code) { + SRequestObj* pRequest = (SRequestObj*) param; + + SReqResultInfo* pResultInfo = &pRequest->body.resInfo; + + pResultInfo->pData = pResult; + pResultInfo->numOfRows = 0; + + if (code != TSDB_CODE_SUCCESS) { + pRequest->code = code; + pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); + return; + } + + if (pRequest->code != TSDB_CODE_SUCCESS) { + pRequest->code = code; + pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); + } + + pRequest->code = setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResultInfo->pData, true, false); + if (pRequest->code != TSDB_CODE_SUCCESS) { + pResultInfo->numOfRows = 0; + + pRequest->code = code; + pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); + } + + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->requestId); + + pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); +} + void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { - // TODO + ASSERT (res != NULL && fp != NULL); + + SRequestObj *pRequest = res; + pRequest->body.fetchFp = fp; + + SReqResultInfo *pResultInfo = &pRequest->body.resInfo; + if (taos_num_fields(pRequest) == 0) { + pResultInfo->numOfRows = 0; + pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows); + return; + } + + if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { + // All data has returned to App already, no need to try again + if (pResultInfo->completed) { + pResultInfo->numOfRows = 0; + pRequest->body.fetchFp(param, pRequest, pResultInfo->numOfRows); + return; + } + } + + schedulerAsyncFetchRows(pRequest->body.queryJob, fetchCallback, pRequest); } TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 9de3ee1d0f1a8a529c7177329543e3379cdc6cbb..2d5199f18171364d8b8a80b0a0b3a76caace9d40 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -33,7 +33,11 @@ int32_t genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) { setErrno(pRequest, code); taosMemoryFree(pMsg->pData); - tsem_post(&pRequest->body.rspSem); + if (pRequest->body.queryFp != NULL) { + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + } else { + tsem_post(&pRequest->body.rspSem); + } return code; } @@ -117,7 +121,12 @@ int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); } - tsem_post(&pRequest->body.rspSem); + + if (pRequest->body.queryFp) { + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + } else { + tsem_post(&pRequest->body.rspSem); + } return code; } @@ -146,7 +155,13 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(pMsg->pData); setErrno(pRequest, code); - tsem_post(&pRequest->body.rspSem); + + if (pRequest->body.queryFp != NULL) { + pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code); + } else { + tsem_post(&pRequest->body.rspSem); + } + return code; } @@ -165,7 +180,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { taosMemoryFreeClear(output.dbVgroup); tscError("0x%" PRIx64 " failed to build use db output since %s", pRequest->requestId, terrstr()); - } else if (output.dbVgroup) { + } else if (output.dbVgroup && output.dbVgroup->vgHash) { struct SCatalog* pCatalog = NULL; int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); @@ -185,7 +200,12 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { setConnectionDB(pRequest->pTscObj, db); taosMemoryFree(pMsg->pData); - tsem_post(&pRequest->body.rspSem); + + if (pRequest->body.queryFp != NULL) { + pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code); + } else { + tsem_post(&pRequest->body.rspSem); + } return 0; } @@ -196,11 +216,13 @@ int32_t processCreateTableRsp(void* param, const SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); - tsem_post(&pRequest->body.rspSem); - return code; } - tsem_post(&pRequest->body.rspSem); + if (pRequest->body.queryFp != NULL) { + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + } else { + tsem_post(&pRequest->body.rspSem); + } return code; } @@ -208,18 +230,20 @@ int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); - tsem_post(&pRequest->body.rspSem); - return code; - } - - SDropDbRsp dropdbRsp = {0}; - tDeserializeSDropDbRsp(pMsg->pData, pMsg->len, &dropdbRsp); + } else { + SDropDbRsp dropdbRsp = {0}; + tDeserializeSDropDbRsp(pMsg->pData, pMsg->len, &dropdbRsp); - struct SCatalog* pCatalog = NULL; - catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); - catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid); + struct SCatalog* pCatalog = NULL; + catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid); + } - tsem_post(&pRequest->body.rspSem); + if (pRequest->body.queryFp != NULL) { + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + } else { + tsem_post(&pRequest->body.rspSem); + } return code; } @@ -245,6 +269,7 @@ int32_t processAlterStbRsp(void* param, const SDataBuf* pMsg, int32_t code) { } +// todo refactor: this arraylist is too large void initMsgHandleFp() { handleRequestRspFp[TMSG_INDEX(TDMT_MND_CONNECT)] = processConnectRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = processCreateDbRsp; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index bcdba696925a0cef4838fa24a563852385044c22..5643af746dd29cb2c3e479266ae62bf452bf2971 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2241,7 +2241,7 @@ static int32_t smlInsertData(SSmlHandle* info) { } info->cost.insertRpcTime = taosGetTimestampUs(); - launchQueryImpl(info->pRequest, info->pQuery, TSDB_CODE_SUCCESS, true, NULL); + launchQueryImpl(info->pRequest, info->pQuery, true, NULL); info->affectedRows = taos_affected_rows(info->pRequest); return info->pRequest->code; @@ -2332,6 +2332,8 @@ static int32_t isSchemalessDb(SSmlHandle* info){ smlBuildInvalidDataMsg(&info->msgBuf, "catalogGetDBCfg error, code:", tstrerror(code)); return code; } + taosArrayDestroy(pInfo.pRetensions); + if (!pInfo.schemaless){ info->pRequest->code = TSDB_CODE_SML_INVALID_DB_CONF; smlBuildInvalidDataMsg(&info->msgBuf, "can not insert into schemaless db:", dbFname); @@ -2362,7 +2364,7 @@ static int32_t isSchemalessDb(SSmlHandle* info){ */ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { - SRequestObj* request = (SRequestObj*)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj* request = (SRequestObj*)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT); if(!request){ uError("SML:taos_schemaless_insert error request is null"); return NULL; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 3adb3684da1164363a1ffda4c26130643efc5f78..2dbba801279e3a7657d93b39607e112d4ca9f727 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -771,11 +771,10 @@ int stmtExec(TAOS_STMT* stmt) { STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); if (STMT_TYPE_QUERY == pStmt->sql.type) { - launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true, NULL); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } else { STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); - launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true, - (autoCreateTbl ? (void**)&pRsp : NULL)); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, (autoCreateTbl ? (void**)&pRsp : NULL)); } if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) { diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index b5b6ea65e0bab73e3d7801fc81fbcd29c013cc25..a0f33627c4669424d6ead15d06fb1e3839411645 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -13,10 +13,12 @@ * along with this program. If not, see . */ -#include -#include -#include #include +#include +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" +#include "clientInt.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -24,7 +26,6 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -#include "../inc/clientInt.h" #include "taos.h" namespace { @@ -41,6 +42,76 @@ void showDB(TAOS* pConn) { printf("%s\n", str); } } + +void fetchCallback(void* param, void* res, int32_t numOfRow) { + printf("numOfRow = %d \n", numOfRow); + int numFields = taos_num_fields(res); + TAOS_FIELD *fields = taos_fetch_fields(res); + TAOS *_taos = (TAOS *)param; + if (numOfRow > 0) { + for (int i = 0; i < numOfRow; ++i) { + TAOS_ROW row = taos_fetch_row(res); + + char temp[256] = {0}; + taos_print_row(temp, row, fields, numFields); + printf("%s\n", temp); + } + taos_fetch_rows_a(res, fetchCallback, _taos); + } else { + printf("no more data, close the connection.\n"); +// taos_free_result(res); +// taos_close(_taos); +// taos_cleanup(); + } +} + +void queryCallback(void* param, void* res, int32_t code) { + if (code != TSDB_CODE_SUCCESS) { + printf("failed to execute, reason:%s\n", taos_errstr(res)); + } + printf("start to fetch data\n"); + taos_fetch_rows_a(res, fetchCallback, param); +} + +void queryCallback1(void* param, void* res, int32_t code) { + if (code != TSDB_CODE_SUCCESS) { + printf("failed to execute, reason:%s\n", taos_errstr(res)); + } + + taos_free_result(res); + + printf("exec query:\n"); + taos_query_a(param, "select * from tm1", queryCallback, param); +} + +void createNewTable(TAOS* pConn, int32_t index) { + char str[1024] = {0}; + sprintf(str, "create table tu%d using st2 tags(%d)", index, index); + + TAOS_RES* pRes = taos_query(pConn, str); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + for(int32_t i = 0; i < 1000; i += 20) { + char sql[1024] = {0}; + sprintf(sql, + "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", index, + i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7, + i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14, + i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); + TAOS_RES* p = taos_query(pConn, sql); + if (taos_errno(p) != 0) { + printf("failed to insert data, reason:%s\n", taos_errstr(p)); + } + + taos_free_result(p); + } +} } // namespace int main(int argc, char** argv) { @@ -52,7 +123,7 @@ TEST(testCase, driverInit_Test) { // taosInitGlobalCfg(); // taos_init(); } -#if 0 + TEST(testCase, connect_Test) { // taos_options(TSDB_OPTION_CONFIGDIR, "/home/ubuntu/first/cfg"); @@ -62,7 +133,7 @@ TEST(testCase, connect_Test) { } taos_close(pConn); } - +#if 0 TEST(testCase, create_user_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -432,7 +503,7 @@ TEST(testCase, create_multiple_tables) { taos_free_result(pRes); - pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); + pRes = taos_query(pConn, "create table if not exists t_2 using st1 tags(1)"); if (taos_errno(pRes) != 0) { printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); @@ -440,7 +511,7 @@ TEST(testCase, create_multiple_tables) { } taos_free_result(pRes); - pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); + pRes = taos_query(pConn, "create table if not exists t_3 using st1 tags(2)"); if (taos_errno(pRes) != 0) { printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); @@ -480,9 +551,7 @@ TEST(testCase, show_table_Test) { TAOS_RES* pRes = taos_query(pConn, "show tables"); if (taos_errno(pRes) != 0) { printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); } - taos_free_result(pRes); taos_query(pConn, "use abc1"); @@ -599,50 +668,10 @@ TEST(testCase, projection_query_tables) { } taos_free_result(pRes); - pRes = taos_query(pConn, "create table tu2 using st2 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - for(int32_t i = 0; i < 1000000; i += 20) { - char sql[1024] = {0}; - sprintf(sql, - "insert into tu values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", - i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7, - i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14, - i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); - TAOS_RES* p = taos_query(pConn, sql); - if (taos_errno(p) != 0) { - printf("failed to insert data, reason:%s\n", taos_errstr(p)); - } - - taos_free_result(p); + for(int32_t i = 0; i < 100; ++i) { + printf("create table :%d\n", i); + createNewTable(pConn, i); } - - printf("start to insert next table\n"); - - for(int32_t i = 0; i < 1000000; i += 20) { - char sql[1024] = {0}; - sprintf(sql, - "insert into tu2 values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", - i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7, - i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14, - i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); - TAOS_RES* p = taos_query(pConn, sql); - if (taos_errno(p) != 0) { - printf("failed to insert data, reason:%s\n", taos_errstr(p)); - } - - taos_free_result(p); - } - // pRes = taos_query(pConn, "select * from tu"); // if (taos_errno(pRes) != 0) { // printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); @@ -664,74 +693,93 @@ TEST(testCase, projection_query_tables) { taos_close(pConn); } -TEST(testCase, projection_query_stables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "select ts from st1"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); +//TEST(testCase, projection_query_stables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "select ts from st1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } +//TEST(testCase, agg_query_tables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show stables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// int32_t n = 0; +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t* length = taos_fetch_lengths(pRes); +// for(int32_t i = 0; i < numOfFields; ++i) { +// printf("(%d):%d " , i, length[i]); +// } +// printf("\n"); +// +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// memset(str, 0, sizeof(str)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +#endif - taos_free_result(pRes); - taos_close(pConn); -} +/* +--- copy the following script in the shell to setup the environment --- -#endif +create database test; +use test; +create table m1(ts timestamp, k int) tags(a int); +create table tm0 using m1 tags(1); +create table tm1 using m1 tags(2); +insert into tm0 values('2021-1-1 1:1:1.120', 1) ('2021-1-1 1:1:2.9', 2) tm1 values('2021-1-1 1:1:1.120', 11) ('2021-1-1 1:1:2.99', 22); -TEST(testCase, agg_query_tables) { + */ +TEST(testCase, async_api_test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "explain analyze select count(*) from tu interval(1s)"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - int32_t n = 0; - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t* length = taos_fetch_lengths(pRes); - for(int32_t i = 0; i < numOfFields; ++i) { - printf("(%d):%d " , i, length[i]); - } - printf("\n"); - - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - memset(str, 0, sizeof(str)); - } - - taos_free_result(pRes); + taos_query_a(pConn, "drop table test.tm0", queryCallback, pConn); + getchar(); taos_close(pConn); } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 6dc8088cd1ab9470fefe35666fa186b5acd5f3f6..1358d45f3063a256300728764efedbecc49d0419 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -486,7 +486,7 @@ TEST(testCase, smlProcess_influx_Test) { pRes = taos_query(taos, "use inflx_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -607,7 +607,7 @@ TEST(testCase, smlParseLine_error_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -656,7 +656,7 @@ TEST(testCase, smlProcess_telnet_Test) { pRes = taos_query(taos, "use telnet_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -710,7 +710,7 @@ TEST(testCase, smlProcess_json1_Test) { pRes = taos_query(taos, "use json_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -779,7 +779,7 @@ TEST(testCase, smlProcess_json2_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -823,7 +823,7 @@ TEST(testCase, smlProcess_json3_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -895,7 +895,7 @@ TEST(testCase, smlProcess_json4_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -957,7 +957,7 @@ TEST(testCase, smlParseTelnetLine_error_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -1006,7 +1006,7 @@ TEST(testCase, smlParseTelnetLine_diff_type_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -1033,7 +1033,7 @@ TEST(testCase, smlParseTelnetLine_json_error_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -1101,7 +1101,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -1146,7 +1146,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -1191,7 +1191,7 @@ TEST(testCase, sml_TD15662_Test) { pRes = taos_query(taos, "use db_15662"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); @@ -1218,7 +1218,7 @@ TEST(testCase, sml_TD15735_Test) { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -1244,7 +1244,7 @@ TEST(testCase, sml_TD15742_Test) { pRes = taos_query(taos, "use TD15742"); taos_free_result(pRes); - SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, NULL, TSDB_SQL_INSERT); + SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT); ASSERT_NE(request, nullptr); SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 948b50c01a6a0d583fdea5b0d412e9e8440092cb..2b59354d609775c6db7b2b1383aadbce4718d7c6 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -78,7 +78,7 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "replica", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "strict", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "duration", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "keep", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "buffer", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "pages", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, @@ -87,11 +87,12 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "wal", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "cache_model", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "single_stable_model", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "schemaless", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "schemaless", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL}, + {.name = "retension", .bytes = 60 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, // {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // disable update }; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 9d16de43b392ebd49699739990e8d94aa7386a1e..c615af705a5b278b0893450d12185570c30d5648 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -99,6 +99,24 @@ void colDataTrim(SColumnInfoData* pColumnInfoData) { // TODO } +int32_t getJsonValueLen(const char *data) { + int32_t dataLen = 0; + if (*data == TSDB_DATA_TYPE_NULL) { + dataLen = CHAR_BYTES; + } else if (*data == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(data + CHAR_BYTES) + CHAR_BYTES; + } else if (*data == TSDB_DATA_TYPE_DOUBLE) { + dataLen = DOUBLE_BYTES + CHAR_BYTES; + } else if (*data == TSDB_DATA_TYPE_BOOL) { + dataLen = CHAR_BYTES + CHAR_BYTES; + } else if (*data & TD_TAG_JSON) { // json string + dataLen = ((STag*)(data))->len; + } else { + ASSERT(0); + } + return dataLen; +} + int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull) { ASSERT(pColumnInfoData != NULL); @@ -118,19 +136,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con if (IS_VAR_DATA_TYPE(type)) { int32_t dataLen = 0; if (type == TSDB_DATA_TYPE_JSON) { - if (*pData == TSDB_DATA_TYPE_NULL) { - dataLen = CHAR_BYTES; - } else if (*pData == TSDB_DATA_TYPE_NCHAR) { - dataLen = varDataTLen(pData + CHAR_BYTES) + CHAR_BYTES; - } else if (*pData == TSDB_DATA_TYPE_DOUBLE) { - dataLen = DOUBLE_BYTES + CHAR_BYTES; - } else if (*pData == TSDB_DATA_TYPE_BOOL) { - dataLen = CHAR_BYTES + CHAR_BYTES; - } else if (*pData == TD_TAG_JSON) { // json string - dataLen = ((STag*)(pData))->len; - } else { - ASSERT(0); - } + dataLen = getJsonValueLen(pData); }else { dataLen = varDataTLen(pData); } @@ -355,14 +361,19 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) return -1; } - int32_t index = (tsColumnIndex == -1) ? 0 : tsColumnIndex; + int32_t index = (tsColumnIndex == -1) ? 0 : tsColumnIndex; + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, index); if (pColInfoData->info.type != TSDB_DATA_TYPE_TIMESTAMP) { return 0; } - pDataBlock->info.window.skey = *(TSKEY*)colDataGetData(pColInfoData, 0); - pDataBlock->info.window.ekey = *(TSKEY*)colDataGetData(pColInfoData, (pDataBlock->info.rows - 1)); + TSKEY skey = *(TSKEY*)colDataGetData(pColInfoData, 0); + TSKEY ekey = *(TSKEY*)colDataGetData(pColInfoData, (pDataBlock->info.rows - 1)); + + pDataBlock->info.window.skey = TMIN(skey, ekey); + pDataBlock->info.window.ekey = TMAX(skey, ekey); + return 0; } @@ -1273,25 +1284,39 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { memmove(nullBitmap, nullBitmap + n / 8, newLen); } else { int32_t tail = n % 8; - int32_t i = 0; - + int32_t i = 0; uint8_t* p = (uint8_t*)nullBitmap; - while (i < len) { - uint8_t v = p[i]; - p[i] = 0; - p[i] = (v << tail); + if (n < 8) { + while (i < len) { + uint8_t v = p[i]; // source bitmap value + p[i] = (v << tail); + + if (i < len - 1) { + uint8_t next = p[i + 1]; + p[i] |= (next >> (8 - tail)); + } - if (i < len - 1) { - uint8_t next = p[i + 1]; - p[i] |= (next >> (8 - tail)); + i += 1; } + } else if (n > 8) { + int32_t gap = len - newLen; + while(i < newLen) { + uint8_t v = p[i + gap]; + p[i] = (v << tail); + + if (i < newLen - 1) { + uint8_t next = p[i + gap + 1]; + p[i] |= (next >> (8 - tail)); + } - i += 1; + i += 1; + } } } } + static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[n], (total - n) * sizeof(int32_t)); @@ -1463,7 +1488,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { return buf; } -void blockDebugShowData(const SArray* dataBlocks) { +void blockDebugShowData(const SArray* dataBlocks, const char* flag) { char pBuf[128] = {0}; int32_t sz = taosArrayGetSize(dataBlocks); for (int32_t i = 0; i < sz; i++) { @@ -1471,7 +1496,7 @@ void blockDebugShowData(const SArray* dataBlocks) { int32_t colNum = pDataBlock->info.numOfCols; int32_t rows = pDataBlock->info.rows; for (int32_t j = 0; j < rows; j++) { - printf("|"); + printf("%s |", flag); for (int32_t k = 0; k < colNum; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); @@ -1496,8 +1521,11 @@ void blockDebugShowData(const SArray* dataBlocks) { case TSDB_DATA_TYPE_UBIGINT: printf(" %15lu |", *(uint64_t*)var); break; + case TSDB_DATA_TYPE_FLOAT: + printf(" %15f |", *(float*)var); + break; case TSDB_DATA_TYPE_DOUBLE: - printf(" %15f |", *(double*)var); + printf(" %15lf |", *(double*)var); break; } } @@ -1525,8 +1553,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks bufSize += sizeof(SSubmitBlk); } - ASSERT(bufSize < 3 * 1024 * 1024); - *pReq = taosMemoryCalloc(1, bufSize); if (!(*pReq)) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1537,7 +1563,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks int32_t msgLen = sizeof(SSubmitReq); int32_t numOfBlks = 0; SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); // TODO: use the latest version + tdSRowInit(&rb, pTSchema->version); for (int32_t i = 0; i < sz; ++i) { SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); @@ -1555,18 +1581,17 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks pSubmitBlk->uid = pDataBlock->info.groupId; pSubmitBlk->numOfRows = rows; - ++numOfBlks; - msgLen += sizeof(SSubmitBlk); int32_t dataLen = 0; for (int32_t j = 0; j < rows; ++j) { // iterate by row tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf - printf("|"); bool isStartKey = false; int32_t offset = 0; for (int32_t k = 0; k < colNum; ++k) { // iterate by column SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + STColumn* pCol = &pTSchema->columns[k]; + ASSERT(pCol->type == pColInfoData->info.type); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: if (!isStartKey) { @@ -1575,29 +1600,29 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks offset, k); } else { - tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, offset, k); + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, offset, k); } break; case TSDB_DATA_TYPE_NCHAR: { - tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, offset, k); + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, offset, k); break; } case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY - tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, offset, k); + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, offset, k); break; } case TSDB_DATA_TYPE_VARBINARY: case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_BLOB: case TSDB_DATA_TYPE_MEDIUMBLOB: - printf("the column type %" PRIi16 " is defined but not implemented yet\n", pColInfoData->info.type); + uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); TASSERT(0); break; default: if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { - tdAppendColValToRow(&rb, 2, pColInfoData->info.type, TD_VTYPE_NORM, var, true, offset, k); + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pColInfoData->info.type, TD_VTYPE_NORM, var, true, offset, k); } else { - printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); TASSERT(0); } break; @@ -1605,7 +1630,13 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks offset += TYPE_BYTES[pColInfoData->info.type]; } dataLen += TD_ROW_LEN(rb.pBuf); +#ifdef TD_DEBUG_PRINT_ROW + tdSRowPrint(rb.pBuf, pTSchema, __func__); +#endif } + + ++numOfBlks; + pSubmitBlk->dataLen = dataLen; msgLen += pSubmitBlk->dataLen; } @@ -1803,3 +1834,99 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo taosArrayDestroy(tagArray); return ret; } + +void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { + int32_t* actualLen = (int32_t*)data; + data += sizeof(int32_t); + + uint64_t* groupId = (uint64_t*)data; + data += sizeof(uint64_t); + + int32_t* colSizes = (int32_t*)data; + data += numOfCols * sizeof(int32_t); + + *dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t)); + + int32_t numOfRows = pBlock->info.rows; + for (int32_t col = 0; col < numOfCols; ++col) { + SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, col); + + // copy the null bitmap + if (IS_VAR_DATA_TYPE(pColRes->info.type)) { + size_t metaSize = numOfRows * sizeof(int32_t); + memcpy(data, pColRes->varmeta.offset, metaSize); + data += metaSize; + (*dataLen) += metaSize; + } else { + int32_t len = BitmapLen(numOfRows); + memcpy(data, pColRes->nullbitmap, len); + data += len; + (*dataLen) += len; + } + + if (needCompress) { + colSizes[col] = blockCompressColData(pColRes, numOfRows, data, needCompress); + data += colSizes[col]; + (*dataLen) += colSizes[col]; + } else { + colSizes[col] = colDataGetLength(pColRes, numOfRows); + (*dataLen) += colSizes[col]; + memmove(data, pColRes->pData, colSizes[col]); + data += colSizes[col]; + } + + colSizes[col] = htonl(colSizes[col]); + } + + *actualLen = *dataLen; + *groupId = pBlock->info.groupId; +} + +const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { + blockDataEnsureCapacity(pBlock, numOfRows); + const char* pStart = pData; + + int32_t dataLen = *(int32_t*)pStart; + pStart += sizeof(int32_t); + + pBlock->info.groupId = *(uint64_t*)pStart; + pStart += sizeof(uint64_t); + + int32_t* colLen = (int32_t*)pStart; + pStart += sizeof(int32_t) * numOfCols; + + for (int32_t i = 0; i < numOfCols; ++i) { + colLen[i] = htonl(colLen[i]); + ASSERT(colLen[i] >= 0); + + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + pColInfoData->varmeta.length = colLen[i]; + pColInfoData->varmeta.allocLen = colLen[i]; + + memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows); + pStart += sizeof(int32_t) * numOfRows; + + if (colLen[i] > 0) { + taosMemoryFreeClear(pColInfoData->pData); + pColInfoData->pData = taosMemoryMalloc(colLen[i]); + } + } else { + memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); + pStart += BitmapLen(numOfRows); + } + + if (colLen[i] > 0) { + memcpy(pColInfoData->pData, pStart, colLen[i]); + } + + // TODO + // setting this flag to true temporarily so aggregate function on stable will + // examine NULL value for non-primary key column + pColInfoData->hasNull = true; + pStart += colLen[i]; + } + + ASSERT(pStart - pData == dataLen); + return pStart; +} \ No newline at end of file diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 65daee650840a5bea69b2c0dcb687ea76b4dc705..9aa94b0216ee219e0bc294ae3988a11b14f3ffe4 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -36,8 +36,6 @@ typedef struct { #define GET_BIT1(p, i) (((p)[(i) / 8] >> ((i) % 8)) & ((uint8_t)1)) #define GET_BIT2(p, i) (((p)[(i) / 4] >> ((i) % 4)) & ((uint8_t)3)) -static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2); - // SValue static FORCE_INLINE int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t n = 0; @@ -141,203 +139,320 @@ static FORCE_INLINE int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { return n; } +int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type) { + // TODO + return 0; +} + // STSRow2 ======================================================================== -static void tTupleTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { - int32_t nColVal = taosArrayGetSize(pArray); +static void setBitMap(uint8_t *pb, uint8_t v, int32_t idx, uint8_t flags) { + if (pb) { + switch (flags & 0xf) { + case TSROW_HAS_NULL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NONE: + if (v) { + SET_BIT1(pb, idx, (uint8_t)1); + } else { + SET_BIT1(pb, idx, (uint8_t)0); + } + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL: + v = v - 1; + SET_BIT1(pb, idx, v); + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + SET_BIT2(pb, idx, v); + break; + + default: + ASSERT(0); + } + } +} +#define SET_IDX(p, i, n, f) \ + do { \ + if ((f)&TSROW_KV_SMALL) { \ + ((uint8_t *)(p))[i] = (n); \ + } else if ((f)&TSROW_KV_MID) { \ + ((uint16_t *)(p))[i] = (n); \ + } else { \ + ((uint32_t *)(p))[i] = (n); \ + } \ + } while (0) + +int32_t tTSRowNew(STSRowBuilder *pBuilder, SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { + int32_t code = 0; STColumn *pTColumn; SColVal *pColVal; + int32_t nColVal = taosArrayGetSize(pArray); + int32_t iColVal; ASSERT(nColVal > 0); - pRow->sver = pTSchema->version; - - // ts - pTColumn = &pTSchema->columns[0]; - pColVal = (SColVal *)taosArrayGet(pArray, 0); - - ASSERT(pTColumn->colId == 0 && pColVal->cid == 0); - ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); - - pRow->ts = pColVal->value.ts; - - // other fields - int32_t iColVal = 1; - int32_t bidx; - uint32_t nv = 0; - uint8_t *pb = NULL; - uint8_t *pf = NULL; - uint8_t *pv = NULL; + // try uint8_t flags = 0; - for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { - bidx = iColumn - 1; - pTColumn = &pTSchema->columns[iColumn]; + uint32_t ntv = 0; + uint32_t nkv = 0; + int16_t nTag = 0; + uint32_t maxIdx = 0; + iColVal = 0; + for (int32_t iColumn = 0; iColumn < pTSchema->numOfCols; iColumn++) { + pTColumn = &pTSchema->columns[iColumn]; if (iColVal < nColVal) { pColVal = (SColVal *)taosArrayGet(pArray, iColVal); } else { pColVal = NULL; } - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - iColVal++; - if (pColVal->isNone) { - goto _set_none; - } else if (pColVal->isNull) { - goto _set_null; + if (iColumn == 0) { + ASSERT(pColVal->cid == pTColumn->colId); + ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(pTColumn->colId == 0); + + iColVal++; + } else { + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + iColVal++; + + if (pColVal->isNone) { + flags |= TSROW_HAS_NONE; + } else if (pColVal->isNull) { + flags |= TSROW_HAS_NULL; + maxIdx = nkv; + nTag++; + nkv += tPutI16v(NULL, -pTColumn->colId); + } else { + flags |= TSROW_HAS_VAL; + maxIdx = nkv; + nTag++; + nkv += tPutI16v(NULL, pTColumn->colId); + nkv += tPutValue(NULL, &pColVal->value, pTColumn->type); + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + ntv += tPutValue(NULL, &pColVal->value, pTColumn->type); + } + } + } else if (pColVal->cid > pTColumn->colId) { + flags |= TSROW_HAS_NONE; } else { - goto _set_value; + ASSERT(0); } - } else if (pColVal->cid > pTColumn->colId) { - goto _set_none; } else { - ASSERT(0); + flags |= TSROW_HAS_NONE; } - } else { - goto _set_none; } + } - _set_none: - flags |= TSROW_HAS_NONE; - // SET_BIT2(pb, bidx, 0); (todo) - continue; + ASSERT(flags); - _set_null: - flags != TSROW_HAS_NULL; - // SET_BIT2(pb, bidx, 1); (todo) - continue; + // decide + uint32_t nData = 0; + uint32_t nDataT = 0; + uint32_t nDataK = 0; + if (flags == TSROW_HAS_NONE || flags == TSROW_HAS_NULL) { + nData = 0; + } else { + switch (flags) { + case TSROW_HAS_VAL: + nDataT = pTSchema->flen + ntv; + break; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + nDataT = BIT1_SIZE(pTSchema->numOfCols - 1); + break; + case TSROW_HAS_VAL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL: + nDataT = BIT1_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv; + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + nDataT = BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntv; + break; + default: + ASSERT(0); + } - _set_value: - flags != TSROW_HAS_VAL; - // SET_BIT2(pb, bidx, 2); (todo) - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - // nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 1); + uint8_t tflags = 0; + if (maxIdx <= UINT8_MAX) { + nDataK = sizeof(STSKVRow) + sizeof(uint8_t) * nTag + nkv; + tflags |= TSROW_KV_SMALL; + } else if (maxIdx <= UINT16_MAX) { + nDataK = sizeof(STSKVRow) + sizeof(uint16_t) * nTag + nkv; + tflags |= TSROW_KV_MID; } else { - // tPutColVal(pf ? pf + pTColumn->offset : pf, pColVal, pTColumn->type, 1); + nDataK = sizeof(STSKVRow) + sizeof(uint32_t) * nTag + nkv; + tflags |= TSROW_KV_BIG; } - continue; - } - ASSERT(flags); - switch (flags & 0xf) { - case TSROW_HAS_NONE: - case TSROW_HAS_NULL: - pRow->nData = 0; - break; - case TSROW_HAS_VAL: - pRow->nData = pTSchema->flen + nv; - break; - case TSROW_HAS_NULL | TSROW_HAS_NONE: - pRow->nData = BIT1_SIZE(pTSchema->numOfCols - 1); - break; - case TSROW_HAS_VAL | TSROW_HAS_NONE: - case TSROW_HAS_VAL | TSROW_HAS_NULL: - pRow->nData = BIT1_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + nv; - break; - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - pRow->nData = BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + nv; - break; - default: - break; + if (nDataT < nDataK) { + nData = nDataT; + } else { + nData = nDataK; + flags |= tflags; + } } -} -static void tMapTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 *pRow) { - int32_t nColVal = taosArrayGetSize(pArray); - STColumn *pTColumn; - SColVal *pColVal; - - ASSERT(nColVal > 0); - - pRow->sver = pTSchema->version; - - // ts - pTColumn = &pTSchema->columns[0]; - pColVal = (SColVal *)taosArrayGet(pArray, 0); + // alloc + if (pBuilder) { + // create from a builder + if (nData == 0) { + pBuilder->tsRow.nData = 0; + pBuilder->tsRow.pData = NULL; + } else { + if (pBuilder->szBuf < nData) { + uint8_t *p = taosMemoryRealloc(pBuilder->pBuf, nData); + if (p == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pBuilder->pBuf = p; + pBuilder->szBuf = nData; + } - ASSERT(pTColumn->colId == 0 && pColVal->cid == 0); - ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); + pBuilder->tsRow.nData = nData; + pBuilder->tsRow.pData = pBuilder->pBuf; + } - pRow->ts = pColVal->value.ts; + *ppRow = &pBuilder->tsRow; + } else { + // create a new one + *ppRow = (STSRow2 *)taosMemoryMalloc(sizeof(STSRow2)); + if (*ppRow == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + if (nData == 0) { + (*ppRow)->nData = 0; + (*ppRow)->pData = NULL; + } else { + (*ppRow)->nData = nData; + (*ppRow)->pData = taosMemoryMalloc(nData); + if ((*ppRow)->pData == NULL) { + taosMemoryFree(*ppRow); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + } + } - // other fields - int32_t iColVal = 1; - uint32_t nv = 0; - uint8_t *pv = NULL; - uint8_t *pidx = NULL; - uint8_t flags = 0; - int16_t nCol = 0; - for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { - pTColumn = &pTSchema->columns[iColumn]; + // build + (*ppRow)->flags = flags; + (*ppRow)->sver = pTSchema->version; - if (iColVal < nColVal) { - pColVal = (SColVal *)taosArrayGet(pArray, iColVal); + pColVal = (SColVal *)taosArrayGet(pArray, 0); + (*ppRow)->ts = pColVal->value.ts; + + if ((*ppRow)->pData) { + STSKVRow *pTSKVRow = NULL; + uint8_t *pidx = NULL; + uint8_t *pkv = NULL; + uint8_t *pb = NULL; + uint8_t *pf = NULL; + uint8_t *ptv = NULL; + nkv = 0; + ntv = 0; + iColVal = 1; + + if (flags & 0xf0 == 0) { + switch (flags & 0xf) { + case TSROW_HAS_VAL: + pf = (*ppRow)->pData; + ptv = pf + pTSchema->flen; + break; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + pb = (*ppRow)->pData; + break; + case TSROW_HAS_VAL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL: + pb = (*ppRow)->pData; + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + ptv = pf + pTSchema->flen; + break; + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + pb = (*ppRow)->pData; + pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); + ptv = pf + pTSchema->flen; + break; + default: + ASSERT(0); + } } else { - pColVal = NULL; + pTSKVRow = (STSKVRow *)(*ppRow)->pData; + pTSKVRow->nCols = 0; + pidx = pTSKVRow->idx; + if (flags & TSROW_KV_SMALL) { + pkv = pidx + sizeof(uint8_t) * nTag; + } else if (flags & TSROW_KV_MID) { + pkv = pidx + sizeof(uint16_t) * nTag; + } else { + pkv = pidx + sizeof(uint32_t) * nTag; + } } - if (pColVal) { - if (pColVal->cid == pTColumn->colId) { - iColVal++; - if (pColVal->isNone) { + for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { + pTColumn = &pTSchema->columns[iColumn]; + if (iColVal < nColVal) { + pColVal = (SColVal *)taosArrayGet(pArray, iColVal); + } else { + pColVal = NULL; + } + + if (pColVal) { + if (pColVal->cid == pTColumn->colId) { + iColVal++; + + if (pColVal->isNone) { + goto _set_none; + } else if (pColVal->isNull) { + goto _set_null; + } else { + goto _set_value; + } + } else if (pColVal->cid > pTColumn->colId) { goto _set_none; - } else if (pColVal->isNull) { - goto _set_null; } else { - goto _set_value; + ASSERT(0); } - } else if (pColVal->cid > pTColumn->colId) { - goto _set_none; } else { - ASSERT(0); + goto _set_none; } - } else { - goto _set_none; - } - _set_none: - flags |= TSROW_HAS_NONE; - continue; - - _set_null: - flags != TSROW_HAS_NULL; - pidx[nCol++] = nv; - // nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); - continue; - - _set_value: - flags != TSROW_HAS_VAL; - pidx[nCol++] = nv; - // nv += tPutColVal(pv ? pv + nv : pv, pColVal, pTColumn->type, 0); - continue; - } + _set_none: + if (flags & 0xf0 == 0) { + setBitMap(pb, 0, iColumn - 1, flags); + } + continue; - if (nv <= UINT8_MAX) { - // small - } else if (nv <= UINT16_MAX) { - // mid - } else { - // large - } -} + _set_null: + if (flags & 0xf0 == 0) { + setBitMap(pb, 1, iColumn - 1, flags); + } else { + SET_IDX(pidx, pTSKVRow->nCols, nkv, flags); + pTSKVRow->nCols++; + nkv += tPutI16v(pkv + nkv, -pTColumn->colId); + } + continue; -// try-decide-build -int32_t tTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow2 **ppRow) { - int32_t code = 0; - STSRow2 rowT = {0}; - STSRow2 rowM = {0}; + _set_value: + if (flags & 0xf0 == 0) { + setBitMap(pb, 2, iColumn - 1, flags); - // try - tTupleTSRowNew(pArray, pTSchema, &rowT); - tMapTSRowNew(pArray, pTSchema, &rowM); - - // decide & build - if (rowT.nData <= rowM.nData) { - tTupleTSRowNew(pArray, pTSchema, &rowT); - } else { - tMapTSRowNew(pArray, pTSchema, &rowM); + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + *(VarDataOffsetT *)(pf + pTColumn->offset) = ntv; + ntv += tPutValue(ptv + ntv, &pColVal->value, pTColumn->type); + } else { + tPutValue(pf + pTColumn->offset, &pColVal->value, pTColumn->type); + } + } else { + SET_IDX(pidx, pTSKVRow->nCols, nkv, flags); + pTSKVRow->nCols++; + nkv += tPutI16v(pkv + nkv, pColVal->cid); + nkv += tPutValue(pkv + nkv, &pColVal->value, pTColumn->type); + } + continue; + } } +_exit: return code; } @@ -626,266 +741,8 @@ void tTSchemaDestroy(STSchema *pTSchema) { } // STSRowBuilder -#if 0 -int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, int32_t nCols, SSchema *pSchema) { - if (tTSchemaCreate(sver, pSchema, nCols, &pBuilder->pTSchema) < 0) return -1; - - pBuilder->szBitMap1 = BIT1_SIZE(nCols - 1); - pBuilder->szBitMap2 = BIT2_SIZE(nCols - 1); - pBuilder->szKVBuf = - sizeof(STSKVRow) + sizeof(SKVIdx) * (nCols - 1) + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; - pBuilder->szTPBuf = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->pTSchema->vlen; - pBuilder->pKVBuf = taosMemoryMalloc(pBuilder->szKVBuf); - if (pBuilder->pKVBuf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tTSchemaDestroy(pBuilder->pTSchema); - return -1; - } - pBuilder->pTPBuf = taosMemoryMalloc(pBuilder->szTPBuf); - if (pBuilder->pTPBuf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pBuilder->pKVBuf); - tTSchemaDestroy(pBuilder->pTSchema); - return -1; - } - - return 0; -} - -void tTSRowBuilderClear(STSRowBuilder *pBuilder) { - if (pBuilder->pTPBuf) { - taosMemoryFree(pBuilder->pTPBuf); - pBuilder->pTPBuf = NULL; - } - if (pBuilder->pKVBuf) { - taosMemoryFree(pBuilder->pKVBuf); - pBuilder->pKVBuf = NULL; - } - tTSchemaDestroy(pBuilder->pTSchema); - pBuilder->pTSchema = NULL; -} - -void tTSRowBuilderReset(STSRowBuilder *pBuilder) { - for (int32_t iCol = pBuilder->pTSchema->numOfCols - 1; iCol >= 0; iCol--) { - STColumn *pTColumn = &pBuilder->pTSchema->columns[iCol]; - COL_CLR_SET(pTColumn->flags); - } - - pBuilder->iCol = 0; - ((STSKVRow *)pBuilder->pKVBuf)->nCols = 0; - pBuilder->vlenKV = 0; - pBuilder->vlenTP = 0; - pBuilder->row.flags = 0; -} - -int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData) { - STColumn *pTColumn = &pBuilder->pTSchema->columns[pBuilder->iCol]; - uint8_t *p; - int32_t iCol; - STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf; - - // use interp search - if (pTColumn->colId < cid) { // right search - for (iCol = pBuilder->iCol + 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) { - pTColumn = &pBuilder->pTSchema->columns[iCol]; - if (pTColumn->colId >= cid) break; - } - } else if (pTColumn->colId > cid) { // left search - for (iCol = pBuilder->iCol - 1; iCol >= 0; iCol--) { - pTColumn = &pBuilder->pTSchema->columns[iCol]; - if (pTColumn->colId <= cid) break; - } - } - - if (pTColumn->colId != cid || COL_IS_SET(pTColumn->flags)) { - return -1; - } - - pBuilder->iCol = iCol; - - // set value - if (cid == 0) { - ASSERT(pData && nData == sizeof(TSKEY) && iCol == 0); - pBuilder->row.ts = *(TSKEY *)pData; - pTColumn->flags |= COL_SET_VAL; - } else { - if (pData) { - // set VAL - - pBuilder->row.flags |= TSROW_HAS_VAL; - pTColumn->flags |= COL_SET_VAL; - - /* KV */ - if (1) { // avoid KV at some threshold (todo) - pTSKVRow->idx[pTSKVRow->nCols].cid = cid; - pTSKVRow->idx[pTSKVRow->nCols].offset = pBuilder->vlenKV; - - p = pBuilder->pKVBuf + sizeof(STSKVRow) + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + - pBuilder->vlenKV; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - ASSERT(nData <= pTColumn->bytes); - pBuilder->vlenKV += tPutBinary(p, pData, nData); - } else { - ASSERT(nData == pTColumn->bytes); - memcpy(p, pData, nData); - pBuilder->vlenKV += nData; - } - } - - /* TUPLE */ - p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pTColumn->offset; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - ASSERT(nData <= pTColumn->bytes); - *(int32_t *)p = pBuilder->vlenTP; - - p = pBuilder->pTPBuf + pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->vlenTP; - pBuilder->vlenTP += tPutBinary(p, pData, nData); - } else { - ASSERT(nData == pTColumn->bytes); - memcpy(p, pData, nData); - } - } else { - // set NULL - - pBuilder->row.flags |= TSROW_HAS_NULL; - pTColumn->flags |= COL_SET_NULL; - - pTSKVRow->idx[pTSKVRow->nCols].cid = cid; - pTSKVRow->idx[pTSKVRow->nCols].offset = -1; - } - - pTSKVRow->nCols++; - } - - return 0; -} - -static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2) { - SKVIdx *pKVIdx1 = (SKVIdx *)p1; - SKVIdx *pKVIdx2 = (SKVIdx *)p2; - if (pKVIdx1->cid > pKVIdx2->cid) { - return 1; - } else if (pKVIdx1->cid < pKVIdx2->cid) { - return -1; - } - return 0; -} -static void setBitMap(uint8_t *p, STSchema *pTSchema, uint8_t flags) { - int32_t bidx; - STColumn *pTColumn; - - for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { - pTColumn = &pTSchema->columns[iCol]; - bidx = iCol - 1; - - switch (flags) { - case TSROW_HAS_NULL | TSROW_HAS_NONE: - if (pTColumn->flags & COL_SET_NULL) { - SET_BIT1(p, bidx, (uint8_t)1); - } else { - SET_BIT1(p, bidx, (uint8_t)0); - } - break; - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - if (pTColumn->flags & COL_SET_NULL) { - SET_BIT2(p, bidx, (uint8_t)1); - } else if (pTColumn->flags & COL_SET_VAL) { - SET_BIT2(p, bidx, (uint8_t)2); - } else { - SET_BIT2(p, bidx, (uint8_t)0); - } - break; - default: - if (pTColumn->flags & COL_SET_VAL) { - SET_BIT1(p, bidx, (uint8_t)1); - } else { - SET_BIT1(p, bidx, (uint8_t)0); - } - - break; - } - } -} -int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { - int32_t nDataTP, nDataKV; - STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf; - int32_t nCols = pBuilder->pTSchema->numOfCols; - - // error not set ts - if (!COL_IS_SET(pBuilder->pTSchema->columns->flags)) { - return -1; - } - - ASSERT(pTSKVRow->nCols < nCols); - if (pTSKVRow->nCols < nCols - 1) { - pBuilder->row.flags |= TSROW_HAS_NONE; - } - - ASSERT((pBuilder->row.flags & 0xf) != 0); - *(ppRow) = &pBuilder->row; - switch (pBuilder->row.flags & 0xf) { - case TSROW_HAS_NONE: - case TSROW_HAS_NULL: - pBuilder->row.nData = 0; - pBuilder->row.pData = NULL; - return 0; - case TSROW_HAS_NULL | TSROW_HAS_NONE: - nDataTP = pBuilder->szBitMap1; - break; - case TSROW_HAS_VAL: - nDataTP = pBuilder->pTSchema->flen + pBuilder->vlenTP; - break; - case TSROW_HAS_VAL | TSROW_HAS_NONE: - case TSROW_HAS_VAL | TSROW_HAS_NULL: - nDataTP = pBuilder->szBitMap1 + pBuilder->pTSchema->flen + pBuilder->vlenTP; - break; - case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: - nDataTP = pBuilder->szBitMap2 + pBuilder->pTSchema->flen + pBuilder->vlenTP; - break; - default: - ASSERT(0); - } - - nDataKV = sizeof(STSKVRow) + sizeof(SKVIdx) * pTSKVRow->nCols + pBuilder->vlenKV; - pBuilder->row.sver = pBuilder->pTSchema->version; - if (nDataKV < nDataTP) { - // generate KV row - - ASSERT((pBuilder->row.flags & 0xf) != TSROW_HAS_VAL); - - pBuilder->row.flags |= TSROW_KV_ROW; - pBuilder->row.nData = nDataKV; - pBuilder->row.pData = pBuilder->pKVBuf; - - qsort(pTSKVRow->idx, pTSKVRow->nCols, sizeof(SKVIdx), tSKVIdxCmprFn); - if (pTSKVRow->nCols < nCols - 1) { - memmove(&pTSKVRow->idx[pTSKVRow->nCols], &pTSKVRow->idx[nCols - 1], pBuilder->vlenKV); - } - } else { - // generate TUPLE row - - pBuilder->row.nData = nDataTP; - - uint8_t *p; - uint8_t flags = (pBuilder->row.flags & 0xf); - - if (flags == TSROW_HAS_VAL) { - pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2; - } else { - if (flags == (TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE)) { - pBuilder->row.pData = pBuilder->pTPBuf; - } else { - pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1; - } - - setBitMap(pBuilder->row.pData, pBuilder->pTSchema, flags); - } - } - - return 0; -} -#endif +// STag static int tTagValCmprFn(const void *p1, const void *p2) { if (((STagVal *)p1)->cid < ((STagVal *)p2)->cid) { return -1; @@ -953,25 +810,6 @@ static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const c } } -// if (isLarge) { -// p = (uint8_t *)&((int16_t *)pTag->idx)[pTag->nTag]; -// } else { -// p = (uint8_t *)&pTag->idx[pTag->nTag]; -// } - -// (*ppArray) = taosArrayInit(pTag->nTag + 1, sizeof(STagVal)); -// if (*ppArray == NULL) { -// code = TSDB_CODE_OUT_OF_MEMORY; -// goto _err; -// } - -// for (int16_t iTag = 0; iTag < pTag->nTag; iTag++) { -// if (isLarge) { -// offset = ((int16_t *)pTag->idx)[iTag]; -// } else { -// offset = pTag->idx[iTag]; -// } - void debugPrintSTag(STag *pTag, const char *tag, int32_t ln) { int8_t isJson = pTag->flags & TD_TAG_JSON; int8_t isLarge = pTag->flags & TD_TAG_LARGE; @@ -1020,7 +858,7 @@ void debugCheckTags(STag *pTag) { } ASSERT(pTag->nTag <= 128 && pTag->nTag >= 0); - ASSERT(pTag->ver <= 512 && pTag->ver >= 0); // temp condition for pTag->ver + ASSERT(pTag->ver <= 512 && pTag->ver >= 0); // temp condition for pTag->ver } static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { @@ -1133,7 +971,7 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) { debugPrintSTag(*ppTag, __func__, __LINE__); #endif - debugCheckTags(*ppTag); // TODO: remove this line after debug + debugCheckTags(*ppTag); // TODO: remove this line after debug return code; _err: @@ -1215,9 +1053,7 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag) { return tEncodeBinary(pEncoder, (const uint8_t *)pTag, pTag->len); } -int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag) { - return tDecodeBinary(pDecoder, (uint8_t **)ppTag, NULL); -} +int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag) { return tDecodeBinary(pDecoder, (uint8_t **)ppTag, NULL); } int32_t tTagToValArray(const STag *pTag, SArray **ppArray) { int32_t code = 0; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 141ec4f03b76238d6c15695c7ea3a8ea112d9e4b..e77c462e5ae0fe81521f34cbd1475669747e0ee6 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -109,8 +109,11 @@ int32_t tsCompressColData = -1; */ int32_t tsCompatibleModel = 1; +// count/hyperloglog function always return values in case of all NULL data or Empty data set. +int32_t tsCountAlwaysReturnValue = 1; + // 10 ms for sliding time, the value will changed in case of time precision changed -int32_t tsMinSlidingTime = 10; +int32_t tsMinSlidingTime = 10; // the maxinum number of distict query result int32_t tsMaxNumOfDistinctResults = 1000 * 10000; @@ -130,7 +133,6 @@ int32_t tsRetryStreamCompDelay = 10 * 1000; // The delayed computing ration. 10% of the whole computing time window by default. float tsStreamComputDelayRatio = 0.1f; -int32_t tsProjectExecInterval = 10000; // every 10sec, the projection will be executed once int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance // the maximum allowed query buffer size during query processing for each data node. @@ -374,6 +376,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxStreamCompDelay", tsMaxStreamComputDelay, 10, 1000000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxFirstStreamCompDelay", tsStreamCompStartDelay, 1000, 1000000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "retryStreamCompDelay", tsRetryStreamCompDelay, 10, 1000000000, 0) != 0) return -1; @@ -567,6 +570,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32; tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; + tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32; tsMaxStreamComputDelay = cfgGetItem(pCfg, "maxStreamCompDelay")->i32; tsStreamCompStartDelay = cfgGetItem(pCfg, "maxFirstStreamCompDelay")->i32; tsRetryStreamCompDelay = cfgGetItem(pCfg, "retryStreamCompDelay")->i32; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 9c6c532bcd071bf8799e28076a9ea147d5b81443..7041c9478e27c3d8421040da225aafeeef177ba3 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -35,6 +35,7 @@ int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { } pIter->totalLen = htonl(pMsg->length); + pIter->numOfBlocks = htonl(pMsg->numOfBlocks); ASSERT(pIter->totalLen > 0); pIter->len = 0; pIter->pMsg = pMsg; @@ -693,7 +694,6 @@ void tFreeSMAltertbReq(SMAlterStbReq *pReq) { pReq->pFields = NULL; } - int32_t tSerializeSEpSet(void *buf, int32_t bufLen, const SEpSet *pEpset) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3660,6 +3660,7 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { if (tEncodeCStr(pCoder, pSma->indexName) < 0) return -1; if (tEncodeI32(pCoder, pSma->exprLen) < 0) return -1; if (tEncodeI32(pCoder, pSma->tagsFilterLen) < 0) return -1; + if (tEncodeI32(pCoder, pSma->numOfVgroups) < 0) return -1; if (tEncodeI64(pCoder, pSma->indexUid) < 0) return -1; if (tEncodeI64(pCoder, pSma->tableUid) < 0) return -1; if (tEncodeI64(pCoder, pSma->interval) < 0) return -1; @@ -3671,7 +3672,17 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) { if (pSma->tagsFilterLen > 0) { if (tEncodeCStr(pCoder, pSma->tagsFilter) < 0) return -1; } - + for (int32_t v = 0; v < pSma->numOfVgroups; ++v) { + if (tEncodeI32(pCoder, pSma->pVgEpSet[v].vgId) < 0) return -1; + if (tEncodeI8(pCoder, pSma->pVgEpSet[v].epSet.inUse) < 0) return -1; + int8_t numOfEps = pSma->pVgEpSet[v].epSet.numOfEps; + if (tEncodeI8(pCoder, numOfEps) < 0) return -1; + for (int32_t n = 0; n < numOfEps; ++n) { + const SEp *pEp = &pSma->pVgEpSet[v].epSet.eps[n]; + if (tEncodeCStr(pCoder, pEp->fqdn) < 0) return -1; + if (tEncodeU16(pCoder, pEp->port) < 0) return -1; + } + } return 0; } @@ -3684,6 +3695,7 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) { if (tDecodeCStrTo(pCoder, pSma->indexName) < 0) return -1; if (tDecodeI32(pCoder, &pSma->exprLen) < 0) return -1; if (tDecodeI32(pCoder, &pSma->tagsFilterLen) < 0) return -1; + if (tDecodeI32(pCoder, &pSma->numOfVgroups) < 0) return -1; if (tDecodeI64(pCoder, &pSma->indexUid) < 0) return -1; if (tDecodeI64(pCoder, &pSma->tableUid) < 0) return -1; if (tDecodeI64(pCoder, &pSma->interval) < 0) return -1; @@ -3699,6 +3711,27 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) { } else { pSma->tagsFilter = NULL; } + if (pSma->numOfVgroups > 0) { + pSma->pVgEpSet = (SVgEpSet *)tDecoderMalloc(pCoder, pSma->numOfVgroups * sizeof(SVgEpSet)); + if (!pSma->pVgEpSet) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + memset(pSma->pVgEpSet, 0, pSma->numOfVgroups * sizeof(SVgEpSet)); + + for (int32_t v = 0; v < pSma->numOfVgroups; ++v) { + if (tDecodeI32(pCoder, &pSma->pVgEpSet[v].vgId) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->pVgEpSet[v].epSet.inUse) < 0) return -1; + if (tDecodeI8(pCoder, &pSma->pVgEpSet[v].epSet.numOfEps) < 0) return -1; + int8_t numOfEps = pSma->pVgEpSet[v].epSet.numOfEps; + for (int32_t n = 0; n < numOfEps; ++n) { + SEp *pEp = &pSma->pVgEpSet[v].epSet.eps[n]; + if (tDecodeCStrTo(pCoder, pEp->fqdn) < 0) return -1; + if (tDecodeU16(pCoder, &pEp->port) < 0) return -1; + } + } + } return 0; } @@ -3741,6 +3774,111 @@ int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) { return 0; } +int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder *pCoder, const SVGetTsmaExpWndsReq *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1; + if (tEncodeI64(pCoder, pReq->queryWindow.skey) < 0) return -1; + if (tEncodeI64(pCoder, pReq->queryWindow.ekey) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder *pCoder, SVGetTsmaExpWndsReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->queryWindow.skey) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->queryWindow.ekey) < 0) return -1; + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVGetTSmaExpWndsRsp(SEncoder *pCoder, const SVGetTsmaExpWndsRsp *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1; + if (tEncodeI8(pCoder, pReq->flags) < 0) return -1; + if (tEncodeI32(pCoder, pReq->numExpWnds) < 0) return -1; + for (int32_t i = 0; i < pReq->numExpWnds; ++i) { + if (tEncodeI64(pCoder, pReq->wndSKeys[i]) < 0) return -1; + } + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVGetTsmaExpWndsRsp(SDecoder *pCoder, SVGetTsmaExpWndsRsp *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1; + if (tDecodeI8(pCoder, &pReq->flags) < 0) return -1; + if (tDecodeI32(pCoder, &pReq->numExpWnds) < 0) return -1; + for (int32_t i = 0; i < pReq->numExpWnds; ++i) { + if (tDecodeI64(pCoder, &pReq->wndSKeys[i]) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVDeleteReq(SEncoder *pCoder, const SVDeleteReq *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI64(pCoder, pReq->delUid) < 0) return -1; + if (tEncodeI64(pCoder, pReq->tbUid) < 0) return -1; + if (tEncodeI8(pCoder, pReq->type) < 0) return -1; + if (tEncodeI16v(pCoder, pReq->nWnds) < 0) return -1; + if (tEncodeCStr(pCoder, pReq->tbFullName) < 0) return -1; + if (tEncodeCStr(pCoder, pReq->subPlan) < 0) return -1; + for (int16_t i = 0; i < pReq->nWnds; ++i) { + if (tEncodeI64(pCoder, pReq->wnds[i].skey) < 0) return -1; + if (tEncodeI64(pCoder, pReq->wnds[i].ekey) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVDeleteReq(SDecoder *pCoder, SVDeleteReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pReq->delUid) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->tbUid) < 0) return -1; + if (tDecodeI8(pCoder, &pReq->type) < 0) return -1; + if (tDecodeI16v(pCoder, &pReq->nWnds) < 0) return -1; + if (tDecodeCStr(pCoder, &pReq->tbFullName) < 0) return -1; + if (tDecodeCStr(pCoder, &pReq->subPlan) < 0) return -1; + for (int16_t i = 0; i < pReq->nWnds; ++i) { + if (tDecodeI64(pCoder, &pReq->wnds[i].skey) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->wnds[i].ekey) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVDeleteRsp(SEncoder *pCoder, const SVDeleteRsp *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32(pCoder, pReq->code) < 0) return -1; + if (tEncodeI64(pCoder, pReq->affectedRows) < 0) return -1; + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVDeleteRsp(SDecoder *pCoder, SVDeleteRsp *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32(pCoder, &pReq->code) < 0) return -1; + if (tDecodeI64(pCoder, &pReq->affectedRows) < 0) return -1; + + tEndDecode(pCoder); + return 0; +} + int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateStreamReq *pReq) { int32_t sqlLen = 0; int32_t astLen = 0; @@ -4373,7 +4511,7 @@ int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) { } int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) { - int32_t meta = 0; + int32_t meta = 0; SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -4414,7 +4552,7 @@ int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp) { } int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) { - int32_t meta = 0; + int32_t meta = 0; SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -4430,7 +4568,7 @@ int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp return 0; } -void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp) { +void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) { if (NULL == pRsp) { return; } @@ -4440,6 +4578,3 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp) { taosMemoryFree(pRsp->pMeta); } } - - - diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c index 126a4c023a09505c8b93174c622e14654aa71b0f..b8eec655b125eadf3a8e4f199168ef4bf96109a0 100644 --- a/source/common/src/tmsgcb.c +++ b/source/common/src/tmsgcb.c @@ -22,41 +22,21 @@ static SMsgCb defaultMsgCb; void tmsgSetDefault(const SMsgCb* msgcb) { defaultMsgCb = *msgcb; } int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg) { - PutToQueueFp fp = msgcb->queueFps[qtype]; - return (*fp)(msgcb->mgmt, pMsg); + return (*msgcb->putToQueueFp)(msgcb->mgmt, qtype, pMsg); } int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype) { - GetQueueSizeFp fp = msgcb->qsizeFp; - return (*fp)(msgcb->mgmt, vgId, qtype); + return (*msgcb->qsizeFp)(msgcb->mgmt, vgId, qtype); } -int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { - SendReqFp fp = defaultMsgCb.sendReqFp; - return (*fp)(epSet, pMsg); -} +int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg) { return (*defaultMsgCb.sendReqFp)(epSet, pMsg); } -void tmsgSendRsp(SRpcMsg* pMsg) { - SendRspFp fp = defaultMsgCb.sendRspFp; - return (*fp)(pMsg); -} +void tmsgSendRsp(SRpcMsg* pMsg) { return (*defaultMsgCb.sendRspFp)(pMsg); } -void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) { - SendRedirectRspFp fp = defaultMsgCb.sendRedirectRspFp; - (*fp)(pMsg, pNewEpSet); -} +void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) { (*defaultMsgCb.sendRedirectRspFp)(pMsg, pNewEpSet); } -void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) { - RegisterBrokenLinkArgFp fp = defaultMsgCb.registerBrokenLinkArgFp; - (*fp)(pMsg); -} +void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) { (*defaultMsgCb.registerBrokenLinkArgFp)(pMsg); } -void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { - ReleaseHandleFp fp = defaultMsgCb.releaseHandleFp; - (*fp)(pHandle, type); -} +void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { (*defaultMsgCb.releaseHandleFp)(pHandle, type); } -void tmsgReportStartup(const char* name, const char* desc) { - ReportStartup fp = defaultMsgCb.reportStartupFp; - (*fp)(name, desc); -} \ No newline at end of file +void tmsgReportStartup(const char* name, const char* desc) { (*defaultMsgCb.reportStartupFp)(name, desc); } \ No newline at end of file diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 1a04e83f8127c0c71570c2cef4f300f05da11e67..2b0f6a01a0b87cdee8d071d1e53bad398ea90f97 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -216,7 +216,7 @@ int main(int argc, char const *argv[]) { return -1; } - dInfo("start to run dnode"); + dInfo("start to open dnode"); dmSetSignalHandle(); int32_t code = dmRun(); dInfo("shutting down the service"); diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index bd034fe7d6c21dcf31e0ca4e9e83d7a23fa28fb8..22691300bcc86a77d0ff83ed1880728ccb92bf2c 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -56,6 +56,7 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmProcessGetLoadsReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mndPreprocessQueryMsg(SMnode * pMnode, SRpcMsg * pMsg); // mmWorker.c int32_t mmStartWorker(SMnodeMgmt *pMgmt); @@ -65,10 +66,7 @@ int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutRpcMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutRpcMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutRpcMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutRpcMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 5c5316e3a3ba5f51a59e51b10bdc7663970dd71d..27f5e6f7af0fbbdee169b6ab252798ffa3f4449a 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -129,12 +129,7 @@ SArray *mmGetMsgHandles() { SArray *pArray = taosArrayInit(64, sizeof(SMgmtHandle)); if (pArray == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MON_MM_INFO, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MON_MM_LOAD, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; - - // Requests handled by DNODE if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_QNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_QNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -146,7 +141,6 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - // Requests handled by MNODE if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_ACCT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -159,6 +153,8 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_QNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_QNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -172,66 +168,66 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TABLE_META, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SMA, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SMA, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_TABLE_META, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_QUERY, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STREAM, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SUBSCRIBE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_ASK_EP, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STREAM, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DEPLOY_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_QUERY, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - // Requests handled by VNODE + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - + if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DEPLOY_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_TIMEOUT, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_PING, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_PING_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_CLIENT_REQUEST, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_REQUEST_VOTE, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_REQUEST_VOTE_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_APPEND_ENTRIES, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_MON_MM_INFO, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MON_MM_LOAD, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 1b973f3045d5dd4e2f6e5fcc4e25413068af6af5..0f3c06cb3a909d2c936895026425883f225376f1 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -105,10 +105,7 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->path = pInput->path; pMgmt->name = pInput->name; pMgmt->msgCb = pInput->msgCb; - pMgmt->msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)mmPutRpcMsgToQueryQueue; - pMgmt->msgCb.queueFps[READ_QUEUE] = (PutToQueueFp)mmPutRpcMsgToReadQueue; - pMgmt->msgCb.queueFps[WRITE_QUEUE] = (PutToQueueFp)mmPutRpcMsgToWriteQueue; - pMgmt->msgCb.queueFps[SYNC_QUEUE] = (PutToQueueFp)mmPutRpcMsgToSyncQueue; + pMgmt->msgCb.putToQueueFp = (PutToQueueFp)mmPutRpcMsgToQueue; pMgmt->msgCb.mgmt = pMgmt; taosThreadRwlockInit(&pMgmt->lock, NULL); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 1de9875d063933fe1f35bb5b5770c1aabc6b8fc3..42fa7b718e333b9e146e66fbc58c9e25f5235ea5 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -89,6 +89,8 @@ int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + mndPreprocessQueryMsg(pMgmt->pMnode, pMsg); + return mmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); } @@ -96,40 +98,38 @@ int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutNodeMsgToWorker(&pMgmt->monitorWorker, pMsg); } -static inline int32_t mmPutRpcMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pRpc) { +int32_t mmPutRpcMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); if (pMsg == NULL) return -1; - - dTrace("msg:%p, create and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); memcpy(pMsg, pRpc, sizeof(SRpcMsg)); - taosWriteQitem(pWorker->queue, pMsg); - return 0; -} - -int32_t mmPutRpcMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutRpcMsgToWorker(&pMgmt->queryWorker, pMsg); -} -int32_t mmPutRpcMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutRpcMsgToWorker(&pMgmt->writeWorker, pMsg); -} - -int32_t mmPutRpcMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutRpcMsgToWorker(&pMgmt->readWorker, pMsg); -} - -int32_t mmPutRpcMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - int32_t code = -1; - if (mmAcquire(pMgmt) == 0) { - code = mmPutRpcMsgToWorker(&pMgmt->syncWorker, pMsg); - mmRelease(pMgmt); - } - - if (code != 0) { - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; + switch (qtype) { + case WRITE_QUEUE: + dTrace("msg:%p, is created and will put into vnode-write queue", pMsg); + taosWriteQitem(pMgmt->writeWorker.queue, pMsg); + return 0; + case QUERY_QUEUE: + dTrace("msg:%p, is created and will put into vnode-query queue", pMsg); + taosWriteQitem(pMgmt->queryWorker.queue, pMsg); + return 0; + + case READ_QUEUE: + dTrace("msg:%p, is created and will put into vnode-read queue", pMsg); + taosWriteQitem(pMgmt->readWorker.queue, pMsg); + return 0; + case SYNC_QUEUE: + if (mmAcquire(pMgmt) == 0) { + dTrace("msg:%p, is created and will put into vnode-sync queue", pMsg); + taosWriteQitem(pMgmt->syncWorker.queue, pMsg); + mmRelease(pMgmt); + return 0; + } else { + return -1; + } + default: + terrno = TSDB_CODE_INVALID_PARA; + return -1; } - return code; } int32_t mmStartWorker(SMnodeMgmt *pMgmt) { diff --git a/source/dnode/mgmt/mgmt_qnode/inc/qmInt.h b/source/dnode/mgmt/mgmt_qnode/inc/qmInt.h index 9738fb0c454a460a80fa0516b6e2e0ff1e8b05ff..acc101386bd112862ea243bea89c2e3e0c867fbc 100644 --- a/source/dnode/mgmt/mgmt_qnode/inc/qmInt.h +++ b/source/dnode/mgmt/mgmt_qnode/inc/qmInt.h @@ -42,8 +42,7 @@ int32_t qmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); int32_t qmProcessGetMonitorInfoReq(SQnodeMgmt *pMgmt, SRpcMsg *pMsg); // qmWorker.c -int32_t qmPutRpcMsgToQueryQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t qmPutRpcMsgToFetchQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pMsg); int32_t qmGetQueueSize(SQnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype); int32_t qmStartWorker(SQnodeMgmt *pMgmt); @@ -52,6 +51,8 @@ int32_t qmPutNodeMsgToQueryQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t qmPutNodeMsgToFetchQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t qmPutNodeMsgToMonitorQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t qndPreprocessQueryMsg(SQnode *pQnode, SRpcMsg * pMsg); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmInt.c b/source/dnode/mgmt/mgmt_qnode/src/qmInt.c index 06c18ab2889c05c910ad933338dc608a1d15d676..3b425a0b4923b3384bd620fc86e421cdc211ba4a 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmInt.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmInt.c @@ -43,8 +43,7 @@ static int32_t qmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->path = pInput->path; pMgmt->name = pInput->name; pMgmt->msgCb = pInput->msgCb; - pMgmt->msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qmPutRpcMsgToQueryQueue; - pMgmt->msgCb.queueFps[FETCH_QUEUE] = (PutToQueueFp)qmPutRpcMsgToFetchQueue; + pMgmt->msgCb.putToQueueFp = (PutToQueueFp)qmPutRpcMsgToQueue; pMgmt->msgCb.qsizeFp = (GetQueueSizeFp)qmGetQueueSize; pMgmt->msgCb.mgmt = pMgmt; diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index e7fc261b67a8a6416cdbafae07552a5c9576bc22..6814643b59d9a17735a69cb3d345c97ca1b5c7d0 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -57,6 +57,8 @@ static int32_t qmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) { } int32_t qmPutNodeMsgToQueryQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) { + qndPreprocessQueryMsg(pMgmt->pQnode, pMsg); + return qmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); } @@ -68,22 +70,24 @@ int32_t qmPutNodeMsgToMonitorQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) { return qmPutNodeMsgToWorker(&pMgmt->monitorWorker, pMsg); } -static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pRpc) { +int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); if (pMsg == NULL) return -1; - - dTrace("msg:%p, create and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); memcpy(pMsg, pRpc, sizeof(SRpcMsg)); - taosWriteQitem(pWorker->queue, pMsg); - return 0; -} -int32_t qmPutRpcMsgToQueryQueue(SQnodeMgmt *pMgmt, SRpcMsg *pRpc) { - return qmPutRpcMsgToWorker(pMgmt, &pMgmt->queryWorker, pRpc); -} - -int32_t qmPutRpcMsgToFetchQueue(SQnodeMgmt *pMgmt, SRpcMsg *pRpc) { - return qmPutRpcMsgToWorker(pMgmt, &pMgmt->fetchWorker, pRpc); + switch (qtype) { + case QUERY_QUEUE: + dTrace("msg:%p, is created and will put into qnode-query queue", pMsg); + taosWriteQitem(pMgmt->queryWorker.queue, pMsg); + return 0; + case READ_QUEUE: + dTrace("msg:%p, is created and will put into qnode-fetch queue", pMsg); + taosWriteQitem(pMgmt->fetchWorker.queue, pMsg); + return 0; + default: + terrno = TSDB_CODE_INVALID_PARA; + return -1; + } } int32_t qmGetQueueSize(SQnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index 5ec33fe810a777e654a9b64160169003f983ab77..adc18fdced4777fcfdc731993ce758e301c34ac9 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -44,26 +44,26 @@ typedef struct SVnodeMgmt { } SVnodeMgmt; typedef struct { - int32_t vgId; - int32_t vgVersion; - int8_t dropped; - char path[PATH_MAX + 20]; + int32_t vgId; + int32_t vgVersion; + int8_t dropped; + char path[PATH_MAX + 20]; } SWrapperCfg; typedef struct { - int32_t vgId; - int32_t refCount; - int32_t vgVersion; - int8_t dropped; - int8_t accessState; - char *path; - SVnode *pImpl; - STaosQueue *pWriteQ; - STaosQueue *pSyncQ; - STaosQueue *pApplyQ; - STaosQueue *pQueryQ; - STaosQueue *pFetchQ; - STaosQueue *pMergeQ; + int32_t vgId; + int32_t refCount; + int32_t vgVersion; + int8_t dropped; + int8_t accessState; + char *path; + SVnode *pImpl; + STaosQueue *pWriteQ; + STaosQueue *pSyncQ; + STaosQueue *pApplyQ; + STaosQueue *pQueryQ; + STaosQueue *pFetchQ; + STaosQueue *pMergeQ; } SVnodeObj; typedef struct { @@ -100,21 +100,16 @@ void vmStopWorker(SVnodeMgmt *pMgmt); int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode); void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode); -int32_t vmPutRpcMsgToWriteQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t vmPutRpcMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t vmPutRpcMsgToApplyQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t vmPutRpcMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t vmPutRpcMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t vmPutRpcMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype); - -int32_t vmPutNodeMsgToWriteQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t vmPutNodeMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t vmPutNodeMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t vmPutNodeMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t vmPutNodeMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t vmPutNodeMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc); + +int32_t vmPutMsgToWriteQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmPutMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c index cf5a7ad88544bad3e9fbe21e5605b621148183fe..613f3fb994da4fca90842b971ca65b3e6db7fbc7 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmFile.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmFile.c @@ -16,6 +16,8 @@ #define _DEFAULT_SOURCE #include "vmInt.h" +#define MAX_CONTENT_LEN 1024 * 1024 + SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) { taosThreadRwlockRdlock(&pMgmt->lock); @@ -29,7 +31,7 @@ SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) { SVnodeObj *pVnode = *ppVnode; if (pVnode && num < size) { int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); + // dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); pVnodes[num] = (*ppVnode); num++; pIter = taosHashIterate(pMgmt->hash, pIter); @@ -47,7 +49,7 @@ SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) { int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; int32_t len = 0; - int32_t maxLen = 1024 * 1024; + int32_t maxLen = MAX_CONTENT_LEN; char *content = taosMemoryCalloc(1, maxLen + 1); cJSON *root = NULL; FILE *fp = NULL; @@ -128,7 +130,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes = vnodesNum; code = 0; - dDebug("succcessed to read file %s", file); + dDebug("succcessed to read file %s, numOfVnodes:%d", file, vnodesNum); _OVER: if (content != NULL) taosMemoryFree(content); @@ -156,7 +158,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { SVnodeObj **pVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); int32_t len = 0; - int32_t maxLen = 1024 * 1024; + int32_t maxLen = MAX_CONTENT_LEN; char *content = taosMemoryCalloc(1, maxLen + 1); if (content == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -195,6 +197,6 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) { taosMemoryFree(pVnodes); } - dDebug("successed to write %s", realfile); + dDebug("successed to write %s, numOfVnodes:%d", realfile, numOfVnodes); return taosRenameFile(file, realfile); } \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 018d7a607c146243019b085ab858e7c7a670eda6..77109ab52f1f9e6067ff48ac6b70a40a221b37ab 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -180,6 +180,24 @@ static void vmGenerateWrapperCfg(SVnodeMgmt *pMgmt, SCreateVnodeReq *pCreate, SW snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCreate->vgId); } +static int32_t vmTsmaAdjustDays(SVnodeCfg *pCfg, SCreateVnodeReq *pReq) { + if (pReq->isTsma) { + SMsgHead *smaMsg = pReq->pTsma; + uint32_t contLen = (uint32_t)(htonl(smaMsg->contLen) - sizeof(SMsgHead)); + return smaGetTSmaDays(pCfg, POINTER_SHIFT(smaMsg, sizeof(SMsgHead)), contLen, &pCfg->tsdbCfg.days); + } + return 0; +} + +static int32_t vmTsmaProcessCreate(SVnode *pVnode, SCreateVnodeReq *pReq) { + if (pReq->isTsma) { + SMsgHead *smaMsg = pReq->pTsma; + uint32_t contLen = (uint32_t)(htonl(smaMsg->contLen) - sizeof(SMsgHead)); + return vnodeProcessCreateTSma(pVnode, POINTER_SHIFT(smaMsg, sizeof(SMsgHead)), contLen); + } + return 0; +} + int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SCreateVnodeReq createReq = {0}; SVnodeCfg vnodeCfg = {0}; @@ -192,9 +210,16 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dDebug("vgId:%d, create vnode req is received, tsma:%d standby:%d", createReq.vgId, createReq.isTsma, + dDebug("vgId:%d, start to create vnode, tsma:%d standby:%d", createReq.vgId, createReq.isTsma, createReq.standby); vmGenerateVnodeCfg(&createReq, &vnodeCfg); + + if (vmTsmaAdjustDays(&vnodeCfg, &createReq) < 0) { + dError("vgId:%d, failed to adjust tsma days since %s", createReq.vgId, terrstr()); + code = terrno; + goto _OVER; + } + vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId); @@ -203,14 +228,16 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { tFreeSCreateVnodeReq(&createReq); vmReleaseVnode(pMgmt, pVnode); terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; - return -1; + code = terrno; + goto _OVER; } snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId); if (vnodeCreate(path, &vnodeCfg, pMgmt->pTfs) < 0) { tFreeSCreateVnodeReq(&createReq); dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); - return -1; + code = terrno; + goto _OVER; } SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); @@ -227,14 +254,11 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { goto _OVER; } - if (createReq.isTsma) { - SMsgHead *smaMsg = createReq.pTsma; - uint32_t contLen = (uint32_t)(htonl(smaMsg->contLen) - sizeof(SMsgHead)); - if (vnodeProcessCreateTSma(pImpl, POINTER_SHIFT(smaMsg, sizeof(SMsgHead)), contLen) < 0) { - dError("vgId:%d, failed to create tsma since %s", createReq.vgId, terrstr()); - code = terrno; - goto _OVER; - }; + code = vmTsmaProcessCreate(pImpl, &createReq); + if (code != 0) { + dError("vgId:%d, failed to create tsma since %s", createReq.vgId, terrstr()); + code = terrno; + goto _OVER; } code = vnodeStart(pImpl); @@ -253,6 +277,8 @@ _OVER: if (code != 0) { vnodeClose(pImpl); vnodeDestroy(path, pMgmt->pTfs); + } else { + dInfo("vgId:%d, vnode is created", createReq.vgId); } tFreeSCreateVnodeReq(&createReq); @@ -268,7 +294,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t vgId = dropReq.vgId; - dDebug("vgId:%d, drop vnode req is received", vgId); + dDebug("vgId:%d, start to drop vnode", vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); if (pVnode == NULL) { @@ -295,59 +321,60 @@ SArray *vmGetMsgHandles() { SArray *pArray = taosArrayInit(32, sizeof(SMgmtHandle)); if (pArray == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MON_VM_INFO, vmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MON_VM_LOAD, vmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MON_VM_INFO, vmPutMsgToMonitorQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MON_VM_LOAD, vmPutMsgToMonitorQueue, 0) == NULL) goto _OVER; // Requests handled by VNODE - if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, vmPutNodeMsgToQueryQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, vmPutNodeMsgToQueryQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_TABLE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_UPDATE_TAG_VAL, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_TABLE_META, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_TABLES_META, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_CONSUME, vmPutNodeMsgToQueryQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_QUERY, vmPutNodeMsgToQueryQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_CONNECT, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_DISCONNECT, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - // if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_SET_CUR, vmPutNodeMsgToWriteQueue, 0)== NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_CANCEL_TASK, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_TABLE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TABLE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_CANCEL_SMA, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT_RSMA, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DEPLOY, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_RUN, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DISPATCH, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_RECOVER, vmPutNodeMsgToFetchQueue, 0) == NULL) goto _OVER; - - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; - - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_TIMEOUT, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_PING, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_PING_REPLY, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_CLIENT_REQUEST, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_REQUEST_VOTE, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_REQUEST_VOTE_REPLY, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_APPEND_ENTRIES, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, vmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_UPDATE_TAG_VAL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TABLE_META, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TABLES_META, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_CONSUME, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_QUERY, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_CONNECT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_DISCONNECT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + // if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_SET_CUR, vmPutMsgToWriteQueue, 0)== NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CANCEL_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CANCEL_SMA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT_RSMA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_RUN, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_DISPATCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TASK_RECOVER, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 0c8d492ef449624e7462b736fcdd9c2ffb9c2ac2..23927255bb8c920bfaf9de12e91bbffe4bd1de39 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -18,21 +18,17 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { SVnodeObj *pVnode = NULL; - int32_t refCount = 0; taosThreadRwlockRdlock(&pMgmt->lock); taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode); if (pVnode == NULL) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; } else { - refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + // dTrace("vgId:%d, acquire vnode, ref:%d", pVnode->vgId, refCount); } taosThreadRwlockUnlock(&pMgmt->lock); - if (pVnode != NULL) { - dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); - } - return pVnode; } @@ -41,8 +37,8 @@ void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { taosThreadRwlockRdlock(&pMgmt->lock); int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); + // dTrace("vgId:%d, release vnode, ref:%d", pVnode->vgId, refCount); taosThreadRwlockUnlock(&pMgmt->lock); - dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); } int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { @@ -138,7 +134,7 @@ static void *vmOpenVnodeInThread(void *param) { } } - dDebug("thread:%d, total vnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, + dDebug("thread:%d, numOfVnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened, pThread->failed); return NULL; } @@ -160,7 +156,7 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) { pMgmt->state.totalVnodes = numOfVnodes; - int32_t threadNum = 1; // tsNumOfCores; + int32_t threadNum = 1; int32_t vnodesPerThread = numOfVnodes / threadNum + 1; SVnodeThread *threads = taosMemoryCalloc(threadNum, sizeof(SVnodeThread)); @@ -253,12 +249,7 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->path = pInput->path; pMgmt->name = pInput->name; pMgmt->msgCb = pInput->msgCb; - pMgmt->msgCb.queueFps[WRITE_QUEUE] = (PutToQueueFp)vmPutRpcMsgToWriteQueue; - pMgmt->msgCb.queueFps[SYNC_QUEUE] = (PutToQueueFp)vmPutRpcMsgToSyncQueue; - pMgmt->msgCb.queueFps[APPLY_QUEUE] = (PutToQueueFp)vmPutRpcMsgToApplyQueue; - pMgmt->msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)vmPutRpcMsgToQueryQueue; - pMgmt->msgCb.queueFps[FETCH_QUEUE] = (PutToQueueFp)vmPutRpcMsgToFetchQueue; - pMgmt->msgCb.queueFps[MERGE_QUEUE] = (PutToQueueFp)vmPutRpcMsgToMergeQueue; + pMgmt->msgCb.putToQueueFp = (PutToQueueFp)vmPutRpcMsgToQueue; pMgmt->msgCb.qsizeFp = (GetQueueSizeFp)vmGetQueueSize; pMgmt->msgCb.mgmt = pMgmt; taosThreadRwlockInit(&pMgmt->lock, NULL); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index a945358d342f373f1ec4bda4659ea3008c9a2383..427b3d5c94e49883f7349c133d6d8bfc6111a59a 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -29,11 +29,11 @@ static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) { tmsgSendRsp(&rsp); } -static void vmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { +static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SVnodeMgmt *pMgmt = pInfo->ahandle; int32_t code = -1; - dTrace("msg:%p, get from vnode queue, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + dTrace("msg:%p, get from vnode-mgmt queue", pMsg); switch (pMsg->msgType) { case TDMT_MON_VM_INFO: code = vmProcessGetMonitorInfoReq(pMgmt, pMsg); @@ -49,11 +49,14 @@ static void vmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; - dError("msg:%p, not processed in vnode queue", pMsg); + dError("msg:%p, not processed in vnode-mgmt queue", pMsg); } if (IsReq(pMsg)) { - if (code != 0 && terrno != 0) code = terrno; + if (code != 0 && terrno != 0) { + dError("msg:%p failed to process since %s", pMsg, terrstr()); + code = terrno; + } vmSendRsp(pMsg, code); } @@ -65,13 +68,15 @@ static void vmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SVnodeObj *pVnode = pInfo->ahandle; - dTrace("msg:%p, get from vnode-query queue", pMsg); + dTrace("vgId:%d, msg:%p get from vnode-query queue", pVnode->vgId, pMsg); int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, pMsg); if (code != 0) { if (terrno != 0) code = terrno; + dError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, terrstr()); vmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); + + dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -79,63 +84,77 @@ static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { SVnodeObj *pVnode = pInfo->ahandle; - dTrace("msg:%p, get from vnode-fetch queue", pMsg); + dTrace("vgId:%d, msg:%p get from vnode-fetch queue", pVnode->vgId, pMsg); int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; + dError("vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr()); vmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); + + dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { + int32_t code = 0; + SRpcMsg *pMsg = NULL; SVnodeObj *pVnode = pInfo->ahandle; - SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); - if (pArray == NULL) { - dError("failed to process %d msgs in write-queue since %s", numOfMsgs, terrstr()); - return; - } + int64_t sync = vnodeGetSyncHandle(pVnode->pImpl); + SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg **)); - for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; + for (int32_t m = 0; m < numOfMsgs; m++) { if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; + dTrace("vgId:%d, msg:%p get from vnode-write queue", pVnode->vgId, pMsg); - dTrace("msg:%p, get from vnode-write queue", pMsg); if (taosArrayPush(pArray, &pMsg) == NULL) { - dTrace("msg:%p, failed to push to array since %s", pMsg, terrstr()); + dError("vgId:%d, failed to push msg:%p to vnode-write array", pVnode->vgId, pMsg); vmSendRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY); } } - for (int i = 0; i < taosArrayGetSize(pArray); i++) { - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - SRpcMsg rsp = {.info = pMsg->info}; + for (int32_t m = 0; m < taosArrayGetSize(pArray); m++) { + pMsg = *(SRpcMsg **)taosArrayGet(pArray, m); + code = vnodePreprocessReq(pVnode->pImpl, pMsg); + + if (code == TSDB_CODE_ACTION_IN_PROGRESS) { + dTrace("vgId:%d, msg:%p in progress and no rsp", pVnode->vgId, pMsg); + continue; + } + + if (pMsg->msgType != TDMT_VND_ALTER_REPLICA) { + code = syncPropose(sync, pMsg, false); + } + + if (code == TAOS_SYNC_PROPOSE_SUCCESS) { + dTrace("vgId:%d, msg:%p is proposed and no rsp", pVnode->vgId, pMsg); + continue; + } else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) { + SEpSet newEpSet = {0}; + syncGetEpSet(sync, &newEpSet); + SEp *pEp = &newEpSet.eps[newEpSet.inUse]; + if (pEp->port == tsServerPort && strcmp(pEp->fqdn, tsLocalFqdn) == 0) { + newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps; + } - vnodePreprocessReq(pVnode->pImpl, pMsg); + dTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", pVnode->vgId, pMsg, + newEpSet.numOfEps, newEpSet.inUse); + for (int32_t i = 0; i < newEpSet.numOfEps; ++i) { + dTrace("vgId:%d, msg:%p ep:%s:%u", pVnode->vgId, pMsg, newEpSet.eps[i].fqdn, newEpSet.eps[i].port); + } - int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pMsg, false); - if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { - dTrace("msg:%p, is redirect since not leader, vgId:%d ", pMsg, pVnode->vgId); - rsp.code = TSDB_CODE_RPC_REDIRECT; - SEpSet newEpSet; - syncGetEpSet(vnodeGetSyncHandle(pVnode->pImpl), &newEpSet); - newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps; + SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info}; tmsgSendRedirectRsp(&rsp, &newEpSet); - } else if (ret == TAOS_SYNC_PROPOSE_OTHER_ERROR) { - rsp.code = TSDB_CODE_SYN_INTERNAL_ERROR; - tmsgSendRsp(&rsp); - } else if (ret == TAOS_SYNC_PROPOSE_SUCCESS) { - // send response in applyQ } else { - assert(0); + dError("vgId:%d, msg:%p failed to propose write since %s, code:0x%x", pVnode->vgId, pMsg, tstrerror(code), code); + vmSendRsp(pMsg, code); } } for (int32_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - dTrace("msg:%p, is freed", pMsg); + pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); + dTrace("vgId:%d, msg:%p is freed", pVnode->vgId, pMsg); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -145,16 +164,17 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; + SRpcMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); + if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; + dTrace("vgId:%d, msg:%p get from vnode-apply queue", pVnode->vgId, pMsg); // init response rpc msg SRpcMsg rsp = {0}; // get original rpc msg - assert(pMsg->msgType == TDMT_VND_SYNC_APPLY_MSG); + assert(pMsg->msgType == TDMT_SYNC_APPLY_MSG); SyncApplyMsg *pSyncApplyMsg = syncApplyMsgFromRpcMsg2(pMsg); syncApplyMsgLog2("==vmProcessApplyQueue==", pSyncApplyMsg); SRpcMsg originalRpcMsg; @@ -163,7 +183,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO // apply data into tsdb if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) { rsp.code = terrno; - dTrace("msg:%p, process write error since %s", pMsg, terrstr()); + dError("vgId:%d, msg:%p failed to apply since %s", pVnode->vgId, pMsg, terrstr()); } syncApplyMsgDestroy(pSyncApplyMsg); @@ -175,6 +195,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO tmsgSendRsp(&rsp); } + dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, rsp.code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -182,23 +203,22 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; + SRpcMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); + if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; + dTrace("vgId:%d, msg:%p get from vnode-sync queue", pVnode->vgId, pMsg); int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); if (code != 0) { + dError("vgId:%d, msg:%p failed to sync since %s", pVnode->vgId, pMsg, terrstr()); if (pMsg->info.handle != NULL) { - SRpcMsg rsp = { - .code = (terrno < 0) ? terrno : code, - .info = pMsg->info, - }; - dTrace("msg:%p, failed to process sync queue since %s", pMsg, terrstr()); - tmsgSendRsp(&rsp); + if (terrno != 0) code = terrno; + vmSendRsp(pMsg, code); } } + dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -206,24 +226,26 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; + SRpcMsg *pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); + if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; + dTrace("vgId:%d, msg:%p get from vnode-merge queue", pVnode->vgId, pMsg); - dTrace("msg:%p, get from vnode-merge queue", pMsg); int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { + dError("vgId:%d, msg:%p failed to merge since %s", pVnode->vgId, pMsg, terrstr()); if (terrno != 0) code = terrno; vmSendRsp(pMsg, code); } + dTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } } -static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) { +static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) { SMsgHead *pHead = pMsg->pCont; int32_t code = 0; @@ -232,31 +254,37 @@ static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { - dError("vgId:%d, failed to put msg:%p into vnode-queue since %s", pHead->vgId, pMsg, terrstr()); + dError("vgId:%d, failed to put msg:%p into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(), + TMSG_INFO(pMsg->msgType)); return terrno != 0 ? terrno : -1; } switch (qtype) { case QUERY_QUEUE: - dTrace("msg:%p, put into vnode-query worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); + dTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pQueryQ, pMsg); break; case FETCH_QUEUE: - dTrace("msg:%p, put into vnode-fetch worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + dTrace("vgId:%d, msg:%p put into vnode-fetch queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - dTrace("msg:%p, put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + dTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pWriteQ, pMsg); break; case SYNC_QUEUE: - dTrace("msg:%p, put into vnode-sync worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + dTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pSyncQ, pMsg); break; case MERGE_QUEUE: - dTrace("msg:%p, put into vnode-merge worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + dTrace("vgId:%d, msg:%p put into vnode-merge queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pMergeQ, pMsg); break; + case APPLY_QUEUE: + dTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pApplyQ, pMsg); + break; default: code = -1; terrno = TSDB_CODE_INVALID_PARA; @@ -267,110 +295,39 @@ static int32_t vmPutNodeMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType return code; } -int32_t vmPutNodeMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return vmPutNodeMsgToQueue(pMgmt, pMsg, SYNC_QUEUE); -} +int32_t vmPutMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_QUEUE); } -int32_t vmPutNodeMsgToWriteQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return vmPutNodeMsgToQueue(pMgmt, pMsg, WRITE_QUEUE); -} +int32_t vmPutMsgToWriteQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, WRITE_QUEUE); } -int32_t vmPutNodeMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return vmPutNodeMsgToQueue(pMgmt, pMsg, QUERY_QUEUE); -} +int32_t vmPutMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, QUERY_QUEUE); } -int32_t vmPutNodeMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return vmPutNodeMsgToQueue(pMgmt, pMsg, FETCH_QUEUE); -} +int32_t vmPutMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, FETCH_QUEUE); } -int32_t vmPutNodeMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return vmPutNodeMsgToQueue(pMgmt, pMsg, MERGE_QUEUE); -} +int32_t vmPutMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, MERGE_QUEUE); } -int32_t vmPutNodeMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { - SSingleWorker *pWorker = &pMgmt->mgmtWorker; - dTrace("msg:%p, put into vnode-mgmt worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - taosWriteQitem(pWorker->queue, pMsg); +int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { + dTrace("msg:%p, put into vnode-mgmt queue", pMsg); + taosWriteQitem(pMgmt->mgmtWorker.queue, pMsg); return 0; } -int32_t vmPutNodeMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { - SSingleWorker *pWorker = &pMgmt->monitorWorker; - dTrace("msg:%p, put into vnode-monitor worker, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - taosWriteQitem(pWorker->queue, pMsg); +int32_t vmPutMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { + dTrace("msg:%p, put into vnode-monitor queue", pMsg); + taosWriteQitem(pMgmt->monitorWorker.queue, pMsg); return 0; } -static int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc, EQueueType qtype) { - SMsgHead *pHead = pRpc->pCont; - SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); - if (pVnode == NULL) return -1; - +int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); - int32_t code = 0; - - if (pMsg == NULL) { - rpcFreeCont(pRpc->pCont); - pRpc->pCont = NULL; - code = -1; - } else { - memcpy(pMsg, pRpc, sizeof(SRpcMsg)); - switch (qtype) { - case WRITE_QUEUE: - dTrace("msg:%p, create and put into vnode-write worker, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); - taosWriteQitem(pVnode->pWriteQ, pMsg); - break; - case QUERY_QUEUE: - dTrace("msg:%p, create and put into vnode-query queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); - taosWriteQitem(pVnode->pQueryQ, pMsg); - break; - case FETCH_QUEUE: - dTrace("msg:%p, create and put into vnode-fetch queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); - taosWriteQitem(pVnode->pFetchQ, pMsg); - break; - case APPLY_QUEUE: - dTrace("msg:%p, create and put into vnode-apply queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); - taosWriteQitem(pVnode->pApplyQ, pMsg); - break; - case MERGE_QUEUE: - dTrace("msg:%p, create and put into vnode-merge queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); - taosWriteQitem(pVnode->pMergeQ, pMsg); - break; - case SYNC_QUEUE: - dTrace("msg:%p, create and put into vnode-sync queue, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); - taosWriteQitem(pVnode->pSyncQ, pMsg); - break; - default: - code = -1; - terrno = TSDB_CODE_INVALID_PARA; - break; - } - } - - vmReleaseVnode(pMgmt, pVnode); - return code; -} - -int32_t vmPutRpcMsgToWriteQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc) { - return vmPutRpcMsgToQueue(pMgmt, pRpc, WRITE_QUEUE); -} - -int32_t vmPutRpcMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc) { return vmPutRpcMsgToQueue(pMgmt, pRpc, SYNC_QUEUE); } + if (pMsg == NULL) return -1; -int32_t vmPutRpcMsgToApplyQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc) { - return vmPutRpcMsgToQueue(pMgmt, pRpc, APPLY_QUEUE); -} + SMsgHead *pHead = pRpc->pCont; + dTrace("vgId:%d, msg:%p is created, type:%s", pHead->vgId, pMsg, TMSG_INFO(pMsg->msgType)); -int32_t vmPutRpcMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc) { - return vmPutRpcMsgToQueue(pMgmt, pRpc, QUERY_QUEUE); -} - -int32_t vmPutRpcMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc) { - return vmPutRpcMsgToQueue(pMgmt, pRpc, FETCH_QUEUE); -} - -int32_t vmPutRpcMsgToMergeQueue(SVnodeMgmt *pMgmt, SRpcMsg *pRpc) { - return vmPutRpcMsgToQueue(pMgmt, pRpc, MERGE_QUEUE); + pHead->contLen = htonl(pHead->contLen); + pHead->vgId = htonl(pHead->vgId); + memcpy(pMsg, pRpc, sizeof(SRpcMsg)); + return vmPutMsgToQueue(pMgmt, pMsg, qtype); } int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { @@ -466,29 +423,23 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) { pMPool->max = tsNumOfVnodeMergeThreads; if (tWWorkerInit(pMPool) != 0) return -1; - SSingleWorkerCfg cfg = { + SSingleWorkerCfg mgmtCfg = { .min = 1, .max = 1, .name = "vnode-mgmt", - .fp = (FItem)vmProcessQueue, + .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt, }; - if (tSingleWorkerInit(&pMgmt->mgmtWorker, &cfg) != 0) { - dError("failed to start vnode-mgmt worker since %s", terrstr()); - return -1; - } + if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) return -1; - SSingleWorkerCfg mCfg = { + SSingleWorkerCfg monitorCfg = { .min = 1, .max = 1, .name = "vnode-monitor", - .fp = (FItem)vmProcessQueue, + .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt, }; - if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { - dError("failed to start vnode-monitor worker since %s", terrstr()); - return -1; - } + if (tSingleWorkerInit(&pMgmt->monitorWorker, &monitorCfg) != 0) return -1; dDebug("vnode workers are initialized"); return 0; diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 5f1bf30523c25cb8c2bad6755ecddc4769ea108d..528beb280bfd05aa4030a3351aaf278f31b96e17 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -50,26 +50,26 @@ static int32_t dmInitMonitor() { } int32_t dmInit(int8_t rtype) { - dInfo("start to init env"); + dInfo("start to init dnode env"); if (dmCheckRepeatInit(dmInstance()) != 0) return -1; if (dmInitSystem() != 0) return -1; if (dmInitMonitor() != 0) return -1; if (dmInitDnode(dmInstance(), rtype) != 0) return -1; - dInfo("env is initialized"); + dInfo("dnode env is initialized"); return 0; } static int32_t dmCheckRepeatCleanup(SDnode *pDnode) { if (atomic_val_compare_exchange_8(&pDnode->once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { - dError("env is already cleaned up"); + dError("dnode env is already cleaned up"); return -1; } return 0; } void dmCleanup() { - dDebug("start to cleanup env"); + dDebug("start to cleanup dnode env"); SDnode *pDnode = dmInstance(); if (dmCheckRepeatCleanup(pDnode) != 0) return; dmCleanupDnode(pDnode); @@ -79,7 +79,7 @@ void dmCleanup() { udfcClose(); udfStopUdfd(); taosStopCacheRefreshWorker(); - dInfo("env is cleaned up"); + dInfo("dnode env is cleaned up"); taosCloseLog(); taosCleanupCfg(); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 787f5e50190fb099d5998d4f604decd8b17feb68..ee27f27f06fe0ce502fdfd729a2b573ddb221c37 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -229,7 +229,7 @@ SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType ntype) { taosThreadRwlockRdlock(&pWrapper->lock); if (pWrapper->deployed) { int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); - dTrace("node:%s, is acquired, ref:%d", pWrapper->name, refCount); + // dTrace("node:%s, is acquired, ref:%d", pWrapper->name, refCount); } else { terrno = TSDB_CODE_NODE_NOT_DEPLOYED; pRetWrapper = NULL; @@ -245,7 +245,7 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) { taosThreadRwlockRdlock(&pWrapper->lock); if (pWrapper->deployed || (InParentProc(pWrapper) && pWrapper->required)) { int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); - dTrace("node:%s, is marked, ref:%d", pWrapper->name, refCount); + // dTrace("node:%s, is marked, ref:%d", pWrapper->name, refCount); } else { terrno = TSDB_CODE_NODE_NOT_DEPLOYED; code = -1; @@ -261,7 +261,7 @@ void dmReleaseWrapper(SMgmtWrapper *pWrapper) { taosThreadRwlockRdlock(&pWrapper->lock); int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1); taosThreadRwlockUnlock(&pWrapper->lock); - dTrace("node:%s, is released, ref:%d", pWrapper->name, refCount); + // dTrace("node:%s, is released, ref:%d", pWrapper->name, refCount); } static void dmGetServerStartupStatus(SDnode *pDnode, SServerStatusRsp *pStatus) { diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index e5893fd94740fa20fa244bd1957a02a50e39bf08..8b939d15ce35e4a8ccb9f032025250476a3b8a82 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -117,6 +117,7 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { if (pMsg == NULL) { goto _OVER; } + dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); if (dmBuildNodeMsg(pMsg, pRpc) != 0) { goto _OVER; diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 332dd9a58a6e14f0e0eca104142b9d1258f51cb6..937c1ab7faa29191ea65f8770d0ffb2c531b3c35 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -148,7 +148,6 @@ int32_t dmReadEps(SDnodeData *pData) { code = 0; dDebug("succcessed to read file %s", file); - dmPrintEps(pData); _OVER: if (content != NULL) taosMemoryFree(content); @@ -162,6 +161,7 @@ _OVER: taosArrayPush(pData->dnodeEps, &dnodeEp); } + dDebug("reset dnode list on startup"); dmResetEps(pData, pData->dnodeEps); if (dmIsEpChanged(pData, pData->dnodeId, tsLocalEp)) { @@ -236,11 +236,13 @@ void dmUpdateEps(SDnodeData *pData, SArray *eps) { int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pData->dnodeEps); if (numOfEps != numOfEpsOld) { + dDebug("new dnode list get from mnode"); dmResetEps(pData, eps); dmWriteEps(pData); } else { int32_t size = numOfEps * sizeof(SDnodeEp); if (memcmp(pData->dnodeEps->pData, eps->pData, size) != 0) { + dDebug("new dnode list get from mnode"); dmResetEps(pData, eps); dmWriteEps(pData); } @@ -282,7 +284,7 @@ static void dmResetEps(SDnodeData *pData, SArray *dnodeEps) { static void dmPrintEps(SDnodeData *pData) { int32_t numOfEps = (int32_t)taosArrayGetSize(pData->dnodeEps); - dDebug("print dnode ep list, num:%d", numOfEps); + dDebug("print dnode list, num:%d", numOfEps); for (int32_t i = 0; i < numOfEps; i++) { SDnodeEp *pEp = taosArrayGet(pData->dnodeEps, i); dDebug("dnode:%d, fqdn:%s port:%u is_mnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode); diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 7ac6fc129d2bb591706d6ed722878359c4993515..78e706f90814950287aed067103690f9c215e8e3 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -135,7 +135,7 @@ TdFilePtr dmCheckRunning(const char *dataDir) { return NULL; } - dDebug("file:%s is locked", filepath); + dDebug("lock file:%s to prevent repeated starts", filepath); return pFile; } diff --git a/source/dnode/mgmt/test/mnode/dmnode.cpp b/source/dnode/mgmt/test/mnode/dmnode.cpp index 8c945b50ac48b4b1e290875c58a98f168971bc37..857f58befce1898be621a37ea7cb33feb692d58a 100644 --- a/source/dnode/mgmt/test/mnode/dmnode.cpp +++ b/source/dnode/mgmt/test/mnode/dmnode.cpp @@ -94,7 +94,7 @@ TEST_F(DndTestMnode, 02_Alter_Mnode) { void* pReq = rpcMallocCont(contLen); tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); - SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION); } @@ -111,7 +111,7 @@ TEST_F(DndTestMnode, 02_Alter_Mnode) { void* pReq = rpcMallocCont(contLen); tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); - SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION); } @@ -128,7 +128,7 @@ TEST_F(DndTestMnode, 02_Alter_Mnode) { void* pReq = rpcMallocCont(contLen); tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); - SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); } @@ -186,7 +186,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { void* pReq = rpcMallocCont(contLen); tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); - SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 83a36f4b0d5509884b2e99e7bd0eb4663a564959..8ea172ef94e082424b8e987eba95d9513e8f7090 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -298,28 +298,30 @@ typedef struct { } SVgObj; typedef struct { - char name[TSDB_TABLE_FNAME_LEN]; - char stb[TSDB_TABLE_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createdTime; - int64_t uid; - int64_t stbUid; - int64_t dbUid; - int8_t intervalUnit; - int8_t slidingUnit; - int8_t timezone; - int32_t dstVgId; // for stream - int64_t interval; - int64_t offset; - int64_t sliding; - int32_t exprLen; // strlen + 1 - int32_t tagsFilterLen; - int32_t sqlLen; - int32_t astLen; - char* expr; - char* tagsFilter; - char* sql; - char* ast; + char name[TSDB_TABLE_FNAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createdTime; + int64_t uid; + int64_t stbUid; + int64_t dbUid; + int8_t intervalUnit; + int8_t slidingUnit; + int8_t timezone; + int32_t dstVgId; // for stream + int64_t interval; + int64_t offset; + int64_t sliding; + int32_t exprLen; // strlen + 1 + int32_t tagsFilterLen; + int32_t sqlLen; + int32_t astLen; + int32_t numOfVgroups; + char* expr; + char* tagsFilter; + char* sql; + char* ast; + SVgEpSet* pVgEpSet; } SSmaObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index c5c4800e0295fa48ee4bf9669200f7ce7a31eff8..4becad6da28a85ad1c3249135f64aff8fbba36e5 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -37,6 +37,8 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]); int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); +int32_t mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, col_id_t colId); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 6d1f3710830563e24fe124a3a95582b316ef4e00..736fa3b36108e2e840b1f24a25c302262f83e6ca 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -29,12 +29,18 @@ typedef enum { TRANS_STOP_FUNC_MQ_REB = 4, } ETrnFunc; +typedef enum { + TRANS_ACTION_NULL = 0, + TRANS_ACTION_MSG = 1, + TRANS_ACTION_RAW = 2, +} ETrnAct; + typedef struct { int32_t id; int32_t errCode; int32_t acceptableCode; int8_t stage; - int8_t actionType; // 0-msg, 1-raw + ETrnAct actionType; int8_t rawWritten; int8_t msgSent; int8_t msgReceived; @@ -57,6 +63,7 @@ void mndTransDrop(STrans *pTrans); int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); +int32_t mndTransAppendNullLog(STrans *pTrans); int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction); int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); @@ -65,7 +72,7 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname); void mndTransSetSerial(STrans *pTrans); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); -void mndTransProcessRsp(SRpcMsg *pRsp); +int32_t mndTransProcessRsp(SRpcMsg *pRsp); void mndTransPullup(SMnode *pMnode); int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans); diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 3f4f3f2053bd4fd633488eaf4a4fac71d642df51..2119320de5cf97e620f36a1d8c806e5d6efa2d4d 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -34,7 +34,7 @@ int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup); int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); SArray *mndBuildDnodesArray(SMnode *pMnode); int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray); -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2); +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pVgId); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen, bool standby); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 801f335a8056757c2cbe2d7f1ca6d65a4501003f..38a166900da9e1f18c3b3afeb665e736f3a4a2c4 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -30,25 +30,25 @@ static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj); static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOld, SBnodeObj *pNew); static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj); static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq); -static int32_t mndProcessCreateBnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq); -static int32_t mndProcessDropBnodeRsp(SRpcMsg *pRsp); static int32_t mndRetrieveBnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter); int32_t mndInitBnode(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_BNODE, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndBnodeActionEncode, - .decodeFp = (SdbDecodeFp)mndBnodeActionDecode, - .insertFp = (SdbInsertFp)mndBnodeActionInsert, - .updateFp = (SdbUpdateFp)mndBnodeActionUpdate, - .deleteFp = (SdbDeleteFp)mndBnodeActionDelete}; + SSdbTable table = { + .sdbType = SDB_BNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndBnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndBnodeActionDecode, + .insertFp = (SdbInsertFp)mndBnodeActionInsert, + .updateFp = (SdbUpdateFp)mndBnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndBnodeActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_BNODE, mndProcessCreateBnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_BNODE, mndProcessDropBnodeReq); - mndSetMsgHandle(pMnode, TDMT_DND_CREATE_BNODE_RSP, mndProcessCreateBnodeRsp); - mndSetMsgHandle(pMnode, TDMT_DND_DROP_BNODE_RSP, mndProcessDropBnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_BNODE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_BNODE_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndRetrieveBnodes); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndCancelGetNextBnode); @@ -427,16 +427,6 @@ _OVER: return code; } -static int32_t mndProcessCreateBnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessDropBnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndRetrieveBnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 6d7a638c30818d64e436f2de989c7d2fe1d7b9c5..1ce1a2590a2a8948b996fab059e5fb7b76b08314 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -263,7 +263,8 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool standby) { +static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, + bool standby) { STransAction action = {0}; SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); @@ -288,6 +289,32 @@ static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } +static int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + + int32_t contLen = sizeof(SMsgHead); + SMsgHead *pHead = taosMemoryMalloc(contLen); + if (pHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + action.pCont = pHead; + action.contLen = contLen; + action.msgType = TDMT_VND_ALTER_CONFIRM; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pHead); + return -1; + } + + return 0; +} + static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType) { STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); @@ -415,7 +442,6 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0; if (pCfg->schemaless < 0) pCfg->schemaless = TSDB_DB_SCHEMALESS_OFF; - } static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { @@ -726,30 +752,32 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj mndTransSetSerial(pTrans); if (newVgroup.replica < pDb->cfg.replications) { - mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, + mInfo("db:%s, vgId:%d, vn:0 dnode:%d, will add 2 vnodes", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); - if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) { - mError("db:%s, failed to add vnode to vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); - return -1; - } - newVgroup.replica = pDb->cfg.replications; + if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[1], true) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + + if (mndAddVnodeToVgroup(pMnode, &newVgroup, pArray) != 0) return -1; if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVgroup, &newVgroup.vnodeGid[2], true) != 0) return -1; if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; } else { mInfo("db:%s, vgId:%d, will remove 2 vnodes", pVgroup->dbName, pVgroup->vgId); SVnodeGid del1 = {0}; - SVnodeGid del2 = {0}; - if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1, &del2) != 0) { - mError("db:%s, failed to remove vnode from vgId:%d since %s", pDb->name, newVgroup.vgId, terrstr()); - return -1; - } - newVgroup.replica = pDb->cfg.replications; + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1; if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; + + SVnodeGid del2 = {0}; + if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1; + if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; } SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup); @@ -1341,9 +1369,12 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable) { - mDebug("db:%s, version & numOfTable not changed", pDbVgVersion->dbFName); + mDebug("db:%s, version and numOfTable not changed", pDbVgVersion->dbFName); mndReleaseDb(pMnode, pDb); continue; + } else { + mDebug("db:%s, vgroup version changed from %d to %d", pDbVgVersion->dbFName, pDbVgVersion->vgVersion, + pDb->vgVersion); } usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); @@ -1428,6 +1459,62 @@ const char *mndGetDbStr(const char *src) { return pos; } +int64_t getValOfDiffPrecision(int8_t unit, int64_t val) { + int64_t v = 0; + switch (unit) { + case 's': + v = val / 1000; + break; + case 'm': + v = val / tsTickPerMin[TSDB_TIME_PRECISION_MILLI]; + break; + case 'h': + v = val / (tsTickPerMin[TSDB_TIME_PRECISION_MILLI] * 60); + break; + case 'd': + v = val / (tsTickPerMin[TSDB_TIME_PRECISION_MILLI] * 24 * 60); + break; + case 'w': + v = val / (tsTickPerMin[TSDB_TIME_PRECISION_MILLI] * 24 * 60 * 7); + break; + default: + break; + } + + return v; +} + +char *buildRetension(SArray *pRetension) { + size_t size = taosArrayGetSize(pRetension); + if (size == 0) { + return NULL; + } + + char *p1 = taosMemoryCalloc(1, 100); + SRetention *p = taosArrayGet(pRetension, 0); + + int32_t len = 2; + + int64_t v1 = getValOfDiffPrecision(p->freqUnit, p->freq); + int64_t v2 = getValOfDiffPrecision(p->keepUnit, p->keep); + len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c,", v1, p->freqUnit, v2, p->keepUnit); + + p = taosArrayGet(pRetension, 1); + + v1 = getValOfDiffPrecision(p->freqUnit, p->freq); + v2 = getValOfDiffPrecision(p->keepUnit, p->keep); + len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c,", v1, p->freqUnit, v2, p->keepUnit); + + p = taosArrayGet(pRetension, 2); + + v1 = getValOfDiffPrecision(p->freqUnit, p->freq); + v2 = getValOfDiffPrecision(p->keepUnit, p->keep); + len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit); + + varDataSetLen(p1, len); + return p1; +} + static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables, bool sysDb) { int32_t cols = 0; @@ -1475,7 +1562,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.replications, false); - const char *src = pDb->cfg.strict ? "strict" : "nostrict"; + const char *src = pDb->cfg.strict ? "strict" : "no_strict"; char strict[24] = {0}; STR_WITH_SIZE_TO_VARSTR(strict, src, strlen(src)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -1523,6 +1610,20 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + + STR_WITH_SIZE_TO_VARSTR(strict, src, strlen(src)); +#if 0 + char cacheModel[24] = {0}; + bool null = false; + if (pDb->cfg.cacheLastRow == 0) { + STR_TO_VARSTR(cacheModel, "no_cache"); + } else if (pDb->cfg.cacheLastRow == 1) { + STR_TO_VARSTR(cacheModel, "last_row_cache") + } else { + null = true; + } + colDataAppend(pColInfo, rows, cacheModel, null); +#endif colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheLastRow, false); char *prec = NULL; @@ -1552,8 +1653,18 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)statusB, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); + + char *p = buildRetension(pDb->cfg.pRetensions); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + if (p == NULL) { + colDataAppendNULL(pColInfo, rows); + } else { + colDataAppend(pColInfo, rows, (const char *)p, false); + taosMemoryFree(p); + } } } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index bf4baebd8584bd8324f3e4e53836bbd8a2002fad..7e5dbb95660dd3ec89ffc8b6dbdf93a4c3b9f619 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -538,7 +538,7 @@ static int32_t mndRetrieveFuncs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&isAgg, false); - char b3[TSDB_TYPE_STR_MAX_LEN] = {0}; + char b3[TSDB_TYPE_STR_MAX_LEN + 1] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(b3, mnodeGenTypeStr(buf, TSDB_TYPE_STR_MAX_LEN, pFunc->outputType, pFunc->outputLen), pShow->pMeta->pSchemas[cols].bytes); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 3a3fd7ebdb5ac8f56a64ea5b0169dfeda8cd3b97..d989ba6ec39f64b4660b1469c817909ad96f4ee7 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -376,35 +376,35 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { syncRpcMsgLog2(logBuf, pMsg); taosMemoryFree(syncNodeStr); - if (pMsg->msgType == TDMT_VND_SYNC_TIMEOUT) { + if (pMsg->msgType == TDMT_SYNC_TIMEOUT) { SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg); code = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); syncTimeoutDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_PING) { + } else if (pMsg->msgType == TDMT_SYNC_PING) { SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg); code = syncNodeOnPingCb(pSyncNode, pSyncMsg); syncPingDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_PING_REPLY) { + } else if (pMsg->msgType == TDMT_SYNC_PING_REPLY) { SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg); code = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); syncPingReplyDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) { + } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); code = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); syncClientRequestDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) { + } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); code = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); syncRequestVoteDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) { + } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg); code = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); syncRequestVoteReplyDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) { + } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pMsg); code = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); syncAppendEntriesDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) { + } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg); code = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); @@ -625,7 +625,7 @@ int32_t mndAcquireRpcRef(SMnode *pMnode) { code = -1; } else { int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1); - mTrace("mnode rpc is acquired, ref:%d", ref); + // mTrace("mnode rpc is acquired, ref:%d", ref); } taosThreadRwlockUnlock(&pMnode->lock); return code; @@ -634,7 +634,7 @@ int32_t mndAcquireRpcRef(SMnode *pMnode) { void mndReleaseRpcRef(SMnode *pMnode) { taosThreadRwlockRdlock(&pMnode->lock); int32_t ref = atomic_sub_fetch_32(&pMnode->rpcRef, 1); - mTrace("mnode rpc is released, ref:%d", ref); + // mTrace("mnode rpc is released, ref:%d", ref); taosThreadRwlockUnlock(&pMnode->lock); } @@ -675,7 +675,7 @@ int32_t mndAcquireSyncRef(SMnode *pMnode) { code = -1; } else { int32_t ref = atomic_add_fetch_32(&pMnode->syncRef, 1); - mTrace("mnode sync is acquired, ref:%d", ref); + // mTrace("mnode sync is acquired, ref:%d", ref); } taosThreadRwlockUnlock(&pMnode->lock); return code; @@ -684,6 +684,6 @@ int32_t mndAcquireSyncRef(SMnode *pMnode) { void mndReleaseSyncRef(SMnode *pMnode) { taosThreadRwlockRdlock(&pMnode->lock); int32_t ref = atomic_sub_fetch_32(&pMnode->syncRef, 1); - mTrace("mnode sync is released, ref:%d", ref); + // mTrace("mnode sync is released, ref:%d", ref); taosThreadRwlockUnlock(&pMnode->lock); } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 7c94a33ffe10f93a6c809089f04bf704bcb6f676..1cc832b3af9d6f09b0c99f3157cfabc111ef71cb 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -34,9 +34,6 @@ static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNe static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq); static int32_t mndProcessAlterMnodeReq(SRpcMsg *pReq); static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq); -static int32_t mndProcessCreateMnodeRsp(SRpcMsg *pRsp); -static int32_t mndProcessAlterMnodeRsp(SRpcMsg *pRsp); -static int32_t mndProcessDropMnodeRsp(SRpcMsg *pRsp); static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter); @@ -53,11 +50,11 @@ int32_t mndInitMnode(SMnode *pMnode) { }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_MNODE, mndProcessCreateMnodeReq); - mndSetMsgHandle(pMnode, TDMT_DND_ALTER_MNODE, mndProcessAlterMnodeReq); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_MNODE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_MND_ALTER_MNODE, mndProcessAlterMnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_ALTER_MNODE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_DROP_MNODE, mndProcessDropMnodeReq); - mndSetMsgHandle(pMnode, TDMT_DND_CREATE_MNODE_RSP, mndProcessCreateMnodeRsp); - mndSetMsgHandle(pMnode, TDMT_DND_ALTER_MNODE_RSP, mndProcessAlterMnodeRsp); - mndSetMsgHandle(pMnode, TDMT_DND_DROP_MNODE_RSP, mndProcessDropMnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_MNODE_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndRetrieveMnodes); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndCancelGetNextMnode); @@ -338,7 +335,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno .epSet = alterEpset, .pCont = pReq, .contLen = contLen, - .msgType = TDMT_DND_ALTER_MNODE, + .msgType = TDMT_MND_ALTER_MNODE, .acceptableCode = 0, }; @@ -367,7 +364,7 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; - + if (mndTransAppendNullLog(pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -506,7 +503,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode .epSet = alterEpset, .pCont = pReq, .contLen = contLen, - .msgType = TDMT_DND_ALTER_MNODE, + .msgType = TDMT_MND_ALTER_MNODE, .acceptableCode = 0, }; @@ -549,6 +546,7 @@ static int32_t mndDropMnode(SMnode *pMnode, SRpcMsg *pReq, SMnodeObj *pObj) { if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) goto _OVER; if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) goto _OVER; if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) goto _OVER; + if (mndTransAppendNullLog(pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -616,21 +614,6 @@ _OVER: return code; } -static int32_t mndProcessCreateMnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessAlterMnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessDropMnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -655,7 +638,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, b1, false); - const char *roles = "OFFLINE"; + const char *roles = "offline"; if (pObj->id == pMnode->selfDnodeId) { roles = syncStr(TAOS_SYNC_STATE_LEADER); } @@ -667,9 +650,9 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)b2, false); - const char *status = "READY"; - if (objStatus == SDB_STATUS_CREATING) status = "CREATING"; - if (objStatus == SDB_STATUS_DROPPING) status = "DROPPING"; + const char *status = "ready"; + if (objStatus == SDB_STATUS_CREATING) status = "creating"; + if (objStatus == SDB_STATUS_DROPPING) status = "dropping"; char b3[9 + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(b3, status, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 27881865af11913b4a04c4fc84df115e98823fd1..4fb9226144a16f1d44bdddb8826bc91888647f0c 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -30,26 +30,26 @@ static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj); static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOld, SQnodeObj *pNew); static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj); static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq); -static int32_t mndProcessCreateQnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq); -static int32_t mndProcessDropQnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessQnodeListReq(SRpcMsg *pReq); static int32_t mndRetrieveQnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextQnode(SMnode *pMnode, void *pIter); int32_t mndInitQnode(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_QNODE, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndQnodeActionEncode, - .decodeFp = (SdbDecodeFp)mndQnodeActionDecode, - .insertFp = (SdbInsertFp)mndQnodeActionInsert, - .updateFp = (SdbUpdateFp)mndQnodeActionUpdate, - .deleteFp = (SdbDeleteFp)mndQnodeActionDelete}; + SSdbTable table = { + .sdbType = SDB_QNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndQnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndQnodeActionDecode, + .insertFp = (SdbInsertFp)mndQnodeActionInsert, + .updateFp = (SdbUpdateFp)mndQnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndQnodeActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_QNODE, mndProcessCreateQnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_QNODE, mndProcessDropQnodeReq); - mndSetMsgHandle(pMnode, TDMT_DND_CREATE_QNODE_RSP, mndProcessCreateQnodeRsp); - mndSetMsgHandle(pMnode, TDMT_DND_DROP_QNODE_RSP, mndProcessDropQnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_QNODE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_QNODE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_QNODE_LIST, mndProcessQnodeListReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QNODE, mndRetrieveQnodes); @@ -503,16 +503,6 @@ _OVER: return code; } -static int32_t mndProcessCreateQnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessDropQnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndRetrieveQnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 97594f2b913334ac17e2bd5e6c8fc95e19a03e9e..12b39e5b78a10c22fee3d953fd102f62b01330a1 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -18,6 +18,14 @@ #include "mndMnode.h" #include "qworker.h" +int32_t mndPreprocessQueryMsg(SMnode * pMnode, SRpcMsg * pMsg) { + if (TDMT_VND_QUERY != pMsg->msgType) { + return 0; + } + + return qWorkerPreprocessQueryMsg(pMnode->pQuery, pMsg); +} + int32_t mndProcessQueryMsg(SRpcMsg *pMsg) { int32_t code = -1; SMnode *pMnode = pMsg->info.node; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 6cb70d1f27895cd64f08fdb383f4072739e03f52..f9cb63dc74cab51d21766b971298b6d390c29c23 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -36,27 +36,28 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw); static int32_t mndSmaActionInsert(SSdb *pSdb, SSmaObj *pSma); static int32_t mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSpSmatb); static int32_t mndSmaActionUpdate(SSdb *pSdb, SSmaObj *pOld, SSmaObj *pNew); +static int32_t mndSmaGetVgEpSet(SMnode *pMnode, SDbObj *pDb, SVgEpSet **ppVgEpSet, int32_t *numOfVgroups); static int32_t mndProcessMCreateSmaReq(SRpcMsg *pReq); static int32_t mndProcessMDropSmaReq(SRpcMsg *pReq); -static int32_t mndProcessVCreateSmaRsp(SRpcMsg *pRsp); -static int32_t mndProcessVDropSmaRsp(SRpcMsg *pRsp); static int32_t mndProcessGetSmaReq(SRpcMsg *pReq); static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); int32_t mndInitSma(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_SMA, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndSmaActionEncode, - .decodeFp = (SdbDecodeFp)mndSmaActionDecode, - .insertFp = (SdbInsertFp)mndSmaActionInsert, - .updateFp = (SdbUpdateFp)mndSmaActionUpdate, - .deleteFp = (SdbDeleteFp)mndSmaActionDelete}; + SSdbTable table = { + .sdbType = SDB_SMA, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndSmaActionEncode, + .decodeFp = (SdbDecodeFp)mndSmaActionDecode, + .insertFp = (SdbInsertFp)mndSmaActionInsert, + .updateFp = (SdbUpdateFp)mndSmaActionUpdate, + .deleteFp = (SdbDeleteFp)mndSmaActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SMA, mndProcessMCreateSmaReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessMDropSmaReq); - mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndProcessVCreateSmaRsp); - mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndProcessVDropSmaRsp); + mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetSmaReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveSma); @@ -262,7 +263,9 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm req.sliding = pSma->sliding; req.expr = pSma->expr; req.tagsFilter = pSma->tagsFilter; - + req.numOfVgroups = pSma->numOfVgroups; + req.pVgEpSet = pSma->pVgEpSet; + // get length int32_t ret = 0; tEncodeSize(tEncodeSVCreateTSmaReq, &req, contLen, ret); @@ -420,6 +423,15 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, mndReleaseDnode(pMnode, pDnode); // todo add sma info here + SVgEpSet *pVgEpSet = NULL; + int32_t numOfVgroups = 0; + if (mndSmaGetVgEpSet(pMnode, pDb, &pVgEpSet, &numOfVgroups) != 0) { + return -1; + } + + pSma->pVgEpSet = pVgEpSet; + pSma->numOfVgroups = numOfVgroups; + int32_t smaContLen = 0; void *pSmaReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &smaContLen); if (pSmaReq == NULL) return -1; @@ -637,11 +649,6 @@ _OVER: return code; } -static int32_t mndProcessVCreateSmaRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndSetDropSmaRedoLogs(SMnode *pMnode, STrans *pTrans, SSmaObj *pSma) { SSdbRaw *pRedoRaw = mndSmaActionEncode(pSma); if (pRedoRaw == NULL) return -1; @@ -910,11 +917,6 @@ _OVER: return code; } -static int32_t mndProcessVDropSmaRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -974,3 +976,52 @@ static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +static int32_t mndSmaGetVgEpSet(SMnode *pMnode, SDbObj *pDb, SVgEpSet **ppVgEpSet, int32_t *numOfVgroups) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + SVgEpSet *pVgEpSet = NULL; + int32_t nAllocVgs = 16; + int32_t nVgs = 0; + + pVgEpSet = taosMemoryCalloc(nAllocVgs, sizeof(SVgEpSet)); + if (!pVgEpSet) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) { + sdbRelease(pSdb, pVgroup); + continue; + } + + if (nVgs >= nAllocVgs) { + void *p = taosMemoryRealloc(pVgEpSet, nAllocVgs * 2 * sizeof(SVgEpSet)); + if (!p) { + taosMemoryFree(pVgEpSet); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pVgEpSet = (SVgEpSet *)p; + nAllocVgs *= 2; + } + + (pVgEpSet + nVgs)->vgId = pVgroup->vgId; + (pVgEpSet + nVgs)->epSet = mndGetVgroupEpset(pMnode, pVgroup); + + ++nVgs; + + sdbRelease(pSdb, pVgroup); + } + + *ppVgEpSet = pVgEpSet; + *numOfVgroups = nVgs; + + return 0; +} diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index c6acb4fef4a09ef78c561178f11428cb3004b4f3..f5a1e76723f11623644b30a093a0fea0cd7fd1b7 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -30,25 +30,25 @@ static int32_t mndSnodeActionInsert(SSdb *pSdb, SSnodeObj *pObj); static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOld, SSnodeObj *pNew); static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj); static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq); -static int32_t mndProcessCreateSnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq); -static int32_t mndProcessDropSnodeRsp(SRpcMsg *pRsp); static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSnode(SMnode *pMnode, void *pIter); int32_t mndInitSnode(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_SNODE, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndSnodeActionEncode, - .decodeFp = (SdbDecodeFp)mndSnodeActionDecode, - .insertFp = (SdbInsertFp)mndSnodeActionInsert, - .updateFp = (SdbUpdateFp)mndSnodeActionUpdate, - .deleteFp = (SdbDeleteFp)mndSnodeActionDelete}; + SSdbTable table = { + .sdbType = SDB_SNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndSnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndSnodeActionDecode, + .insertFp = (SdbInsertFp)mndSnodeActionInsert, + .updateFp = (SdbUpdateFp)mndSnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndSnodeActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SNODE, mndProcessCreateSnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_SNODE, mndProcessDropSnodeReq); - mndSetMsgHandle(pMnode, TDMT_DND_CREATE_SNODE_RSP, mndProcessCreateSnodeRsp); - mndSetMsgHandle(pMnode, TDMT_DND_DROP_SNODE_RSP, mndProcessDropSnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_SNODE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_SNODE_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_SNODE, mndRetrieveSnodes); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_SNODE, mndCancelGetNextSnode); @@ -437,16 +437,6 @@ _OVER: return code; } -static int32_t mndProcessCreateSnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessDropSnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 556837f397e0d9c09546d5fa9400654b44f39401..388172f6ae30a476f8c7c95068eba07d3df28cef 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -23,6 +23,7 @@ #include "mndPerfSchema.h" #include "mndScheduler.h" #include "mndShow.h" +#include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" @@ -38,9 +39,6 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew); static int32_t mndProcessMCreateStbReq(SRpcMsg *pReq); static int32_t mndProcessMAlterStbReq(SRpcMsg *pReq); static int32_t mndProcessMDropStbReq(SRpcMsg *pReq); -static int32_t mndProcessVCreateStbRsp(SRpcMsg *pRsp); -static int32_t mndProcessVAlterStbRsp(SRpcMsg *pRsp); -static int32_t mndProcessVDropStbRsp(SRpcMsg *pRsp); static int32_t mndProcessTableMetaReq(SRpcMsg *pReq); static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); @@ -59,9 +57,9 @@ int32_t mndInitStb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STB, mndProcessMCreateStbReq); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_STB, mndProcessMAlterStbReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_STB, mndProcessMDropStbReq); - mndSetMsgHandle(pMnode, TDMT_VND_CREATE_STB_RSP, mndProcessVCreateStbRsp); - mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndProcessVAlterStbRsp); - mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndProcessVDropStbRsp); + mndSetMsgHandle(pMnode, TDMT_VND_CREATE_STB_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb); @@ -397,14 +395,14 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.pRSmaParam.xFilesFactor = pStb->xFilesFactor; req.pRSmaParam.delay = pStb->delay; if (pStb->ast1Len > 0) { - if (mndConvertRSmaTask(pStb->pAst1, pStb->uid, 0, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len, req.pRSmaParam.xFilesFactor) != - TSDB_CODE_SUCCESS) { + if (mndConvertRSmaTask(pStb->pAst1, pStb->uid, 0, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len, + req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { return NULL; } } if (pStb->ast2Len > 0) { - if (mndConvertRSmaTask(pStb->pAst2, pStb->uid, 0, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len, req.pRSmaParam.xFilesFactor) != - TSDB_CODE_SUCCESS) { + if (mndConvertRSmaTask(pStb->pAst2, pStb->uid, 0, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len, + req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { return NULL; } } @@ -837,11 +835,6 @@ _OVER: return code; } -static int32_t mndProcessVCreateStbRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) { if (pAlter->commentLen != 0 || pAlter->ttl != 0) return 0; @@ -957,13 +950,18 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p return 0; } -static int32_t mndDropSuperTableTag(const SStbObj *pOld, SStbObj *pNew, const char *tagName) { +static int32_t mndDropSuperTableTag(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const char *tagName) { int32_t tag = mndFindSuperTableTagIndex(pOld, tagName); if (tag < 0) { terrno = TSDB_CODE_MND_TAG_NOT_EXIST; return -1; } + col_id_t colId = pOld->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } @@ -976,7 +974,7 @@ static int32_t mndDropSuperTableTag(const SStbObj *pOld, SStbObj *pNew, const ch return 0; } -static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pFields) { +static int32_t mndAlterStbTagName(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, SArray *pFields) { if ((int32_t)taosArrayGetSize(pFields) != 2) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; @@ -994,6 +992,11 @@ static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pF return -1; } + col_id_t colId = pOld->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndFindSuperTableTagIndex(pOld, newTagName) >= 0) { terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; return -1; @@ -1016,13 +1019,18 @@ static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pF return 0; } -static int32_t mndAlterStbTagBytes(const SStbObj *pOld, SStbObj *pNew, const SField *pField) { +static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const SField *pField) { int32_t tag = mndFindSuperTableTagIndex(pOld, pField->name); if (tag < 0) { terrno = TSDB_CODE_MND_TAG_NOT_EXIST; return -1; } + col_id_t colId = pOld->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } @@ -1083,7 +1091,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray return 0; } -static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const char *colName) { +static int32_t mndDropSuperTableColumn(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const char *colName) { int32_t col = mndFindSuperTableColumnIndex(pOld, colName); if (col < 0) { terrno = TSDB_CODE_MND_COLUMN_NOT_EXIST; @@ -1100,6 +1108,11 @@ static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const return -1; } + col_id_t colId = pOld->pTags[col].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } @@ -1112,7 +1125,7 @@ static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const return 0; } -static int32_t mndAlterStbColumnBytes(const SStbObj *pOld, SStbObj *pNew, const SField *pField) { +static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const SField *pField) { int32_t col = mndFindSuperTableColumnIndex(pOld, pField->name); if (col < 0) { terrno = TSDB_CODE_MND_COLUMN_NOT_EXIST; @@ -1129,6 +1142,11 @@ static int32_t mndAlterStbColumnBytes(const SStbObj *pOld, SStbObj *pNew, const return -1; } + col_id_t colId = pOld->pTags[col].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; } @@ -1207,7 +1225,6 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } - static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, STableMetaRsp *pRsp) { taosRLockLatch(&pStb->lock); @@ -1277,13 +1294,13 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char return code; } - -static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont, int32_t *pLen) { +static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont, + int32_t *pLen) { int ret; SEncoder ec = {0}; - uint32_t contLen = 0; + uint32_t contLen = 0; SMAlterStbRsp alterRsp = {0}; - SName name = {0}; + SName name = {0}; tNameFromString(&name, pAlter->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); alterRsp.pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); @@ -1291,20 +1308,20 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, S terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - + ret = mndBuildStbSchemaImp(pDb, pObj, name.tname, alterRsp.pMeta); if (ret) { tFreeSMAlterStbRsp(&alterRsp); return ret; } - + tEncodeSize(tEncodeSMAlterStbRsp, &alterRsp, contLen, ret); if (ret) { tFreeSMAlterStbRsp(&alterRsp); return ret; } - void* cont = taosMemoryMalloc(contLen); + void *cont = taosMemoryMalloc(contLen); tEncoderInit(&ec, cont, contLen); tEncodeSMAlterStbRsp(&ec, &alterRsp); tEncoderClear(&ec); @@ -1313,24 +1330,24 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, S *pCont = cont; *pLen = contLen; - + return 0; } static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { - bool needRsp = true; + bool needRsp = true; + int32_t code = -1; + STrans *pTrans = NULL; + SField *pField0 = NULL; + SStbObj stbObj = {0}; taosRLockLatch(&pOld->lock); memcpy(&stbObj, pOld, sizeof(SStbObj)); + taosRUnLockLatch(&pOld->lock); stbObj.pColumns = NULL; stbObj.pTags = NULL; stbObj.updateTime = taosGetTimestampMs(); stbObj.lock = 0; - taosRUnLockLatch(&pOld->lock); - - int32_t code = -1; - STrans *pTrans = NULL; - SField *pField0 = NULL; switch (pAlter->alterType) { case TSDB_ALTER_TABLE_ADD_TAG: @@ -1338,25 +1355,25 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p break; case TSDB_ALTER_TABLE_DROP_TAG: pField0 = taosArrayGet(pAlter->pFields, 0); - code = mndDropSuperTableTag(pOld, &stbObj, pField0->name); + code = mndDropSuperTableTag(pMnode, pOld, &stbObj, pField0->name); break; case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: - code = mndAlterStbTagName(pOld, &stbObj, pAlter->pFields); + code = mndAlterStbTagName(pMnode, pOld, &stbObj, pAlter->pFields); break; case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: pField0 = taosArrayGet(pAlter->pFields, 0); - code = mndAlterStbTagBytes(pOld, &stbObj, pField0); + code = mndAlterStbTagBytes(pMnode, pOld, &stbObj, pField0); break; case TSDB_ALTER_TABLE_ADD_COLUMN: code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields); break; case TSDB_ALTER_TABLE_DROP_COLUMN: pField0 = taosArrayGet(pAlter->pFields, 0); - code = mndDropSuperTableColumn(pOld, &stbObj, pField0->name); + code = mndDropSuperTableColumn(pMnode, pOld, &stbObj, pField0->name); break; case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: pField0 = taosArrayGet(pAlter->pFields, 0); - code = mndAlterStbColumnBytes(pOld, &stbObj, pField0); + code = mndAlterStbColumnBytes(pMnode, pOld, &stbObj, pField0); break; case TSDB_ALTER_TABLE_UPDATE_OPTIONS: needRsp = false; @@ -1378,12 +1395,12 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p mndTransSetDbName(pTrans, pDb->name); if (needRsp) { - void* pCont = NULL; + void *pCont = NULL; int32_t contLen = 0; - if (mndBuildSMAlterStbRsp(pDb, pAlter, &stbObj, &pCont, &contLen)) goto _OVER; + if (mndBuildSMAlterStbRsp(pDb, pAlter, &stbObj, &pCont, &contLen) != 0) goto _OVER; mndTransSetRpcRsp(pTrans, pCont, contLen); } - + if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; @@ -1459,11 +1476,6 @@ _OVER: return code; } -static int32_t mndProcessVAlterStbRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); if (pRedoRaw == NULL) return -1; @@ -1599,11 +1611,6 @@ _OVER: return code; } -static int32_t mndProcessVDropStbRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5ee5b06a578f7c31ab18f66f2de1cdef2aa85a04..a0aa2067d3fa76f2ef7dec332c3dda35c0d29349 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -35,7 +35,6 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); -static int32_t mndProcessTaskDeployInternalRsp(SRpcMsg *pRsp); /*static int32_t mndProcessDropStreamReq(SRpcMsg *pReq);*/ /*static int32_t mndProcessDropStreamInRsp(SRpcMsg *pRsp);*/ static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq); @@ -44,17 +43,19 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); int32_t mndInitStream(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_STREAM, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndStreamActionEncode, - .decodeFp = (SdbDecodeFp)mndStreamActionDecode, - .insertFp = (SdbInsertFp)mndStreamActionInsert, - .updateFp = (SdbUpdateFp)mndStreamActionUpdate, - .deleteFp = (SdbDeleteFp)mndStreamActionDelete}; + SSdbTable table = { + .sdbType = SDB_STREAM, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndStreamActionEncode, + .decodeFp = (SdbDecodeFp)mndStreamActionDecode, + .insertFp = (SdbInsertFp)mndStreamActionInsert, + .updateFp = (SdbUpdateFp)mndStreamActionUpdate, + .deleteFp = (SdbDeleteFp)mndStreamActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); - mndSetMsgHandle(pMnode, TDMT_VND_TASK_DEPLOY_RSP, mndProcessTaskDeployInternalRsp); - mndSetMsgHandle(pMnode, TDMT_SND_TASK_DEPLOY_RSP, mndProcessTaskDeployInternalRsp); + mndSetMsgHandle(pMnode, TDMT_VND_TASK_DEPLOY_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_SND_TASK_DEPLOY_RSP, mndTransProcessRsp); /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);*/ /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/ @@ -195,11 +196,6 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) { sdbRelease(pSdb, pStream); } -static int32_t mndProcessTaskDeployInternalRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static SDbObj *mndAcquireDbByStream(SMnode *pMnode, char *streamName) { SName name = {0}; tNameFromString(&name, streamName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 41065a3fdd54d07cb75a65fba8e1da7bdd97d3a3..cbaf8bc1d15b13f59440b06797048e10ddd55b07 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -43,7 +43,6 @@ static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubs static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg); static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg); -static int32_t mndProcessSubscribeInternalRsp(SRpcMsg *pMsg); static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter); @@ -65,20 +64,22 @@ static int32_t mndSetSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeO } int32_t mndInitSubscribe(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_SUBSCRIBE, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndSubActionEncode, - .decodeFp = (SdbDecodeFp)mndSubActionDecode, - .insertFp = (SdbInsertFp)mndSubActionInsert, - .updateFp = (SdbUpdateFp)mndSubActionUpdate, - .deleteFp = (SdbDeleteFp)mndSubActionDelete}; - - mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndProcessSubscribeInternalRsp); - mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_DELETE_RSP, mndProcessSubscribeInternalRsp); + SSdbTable table = { + .sdbType = SDB_SUBSCRIBE, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndSubActionEncode, + .decodeFp = (SdbDecodeFp)mndSubActionDecode, + .insertFp = (SdbInsertFp)mndSubActionInsert, + .updateFp = (SdbUpdateFp)mndSubActionUpdate, + .deleteFp = (SdbDeleteFp)mndSubActionDelete, + }; + + mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_DELETE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_DROP_CGROUP, mndProcessDropCgroupReq); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_DROP_CGROUP_RSP, mndProcessSubscribeInternalRsp); + mndSetMsgHandle(pMnode, TDMT_MND_MQ_DROP_CGROUP_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_SUBSCRIPTIONS, mndRetrieveSubscribe); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextSubscribe); @@ -390,7 +391,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR mInfo("rebalance calculation completed, rebalanced vg:"); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); - mInfo("vgId:%d moved from consumer %" PRId64 " to consumer %" PRId64, pOutputRebVg->pVgEp->vgId, + mInfo("vgId:%d, moved from consumer %" PRId64 " to consumer %" PRId64, pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); } @@ -789,11 +790,6 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) { sdbRelease(pSdb, pSub); } -static int32_t mndProcessSubscribeInternalRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndSetDropSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { SSdbRaw *pRedoRaw = mndSubActionEncode(pSub); if (pRedoRaw == NULL) return -1; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 21b5e37e1ea5b1d5f534bae0efccd07d6f9115ae..a810a6a487238c434c6e7afafb430cf0803dec63 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -37,7 +37,6 @@ static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic); static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj *pNewTopic); static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq); static int32_t mndProcessDropTopicReq(SRpcMsg *pReq); -static int32_t mndProcessDropTopicInRsp(SRpcMsg *pRsp); static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); @@ -45,17 +44,19 @@ static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); static int32_t mndSetDropTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); int32_t mndInitTopic(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_TOPIC, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndTopicActionEncode, - .decodeFp = (SdbDecodeFp)mndTopicActionDecode, - .insertFp = (SdbInsertFp)mndTopicActionInsert, - .updateFp = (SdbUpdateFp)mndTopicActionUpdate, - .deleteFp = (SdbDeleteFp)mndTopicActionDelete}; + SSdbTable table = { + .sdbType = SDB_TOPIC, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndTopicActionEncode, + .decodeFp = (SdbDecodeFp)mndTopicActionDecode, + .insertFp = (SdbInsertFp)mndTopicActionInsert, + .updateFp = (SdbUpdateFp)mndTopicActionUpdate, + .deleteFp = (SdbDeleteFp)mndTopicActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TOPIC, mndProcessCreateTopicReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicReq); - mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndProcessDropTopicInRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextTopic); @@ -70,6 +71,57 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]) { return strchr(topic, '.') + 1; } +int32_t mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, col_id_t colId) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + bool found = false; + while (1) { + SMqTopicObj *pTopic = NULL; + pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); + if (pIter == NULL) break; + if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { + sdbRelease(pSdb, pTopic); + continue; + } + + SNode *pAst = NULL; + if (nodesStringToNode(pTopic->ast, &pAst) != 0) { + ASSERT(0); + return false; + } + + SHashObj *pColHash = NULL; + SNodeList *pNodeList; + nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList); + SNode *pNode = NULL; + FOREACH(pNode, pNodeList) { + SColumnNode *pCol = (SColumnNode *)pNode; + if (pCol->tableId != suid) goto NEXT; + if (pColHash == NULL) { + pColHash = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); + } + if (pCol->colId > 0) { + taosHashPut(pColHash, &pCol->colId, sizeof(int16_t), NULL, 0); + } + } + + if (taosHashGet(pColHash, &colId, sizeof(int16_t)) != NULL) { + found = true; + goto NEXT; + } + + NEXT: + sdbRelease(pSdb, pTopic); + nodesDestroyNode(pAst); + if (found) { + terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC; + return -1; + } + } + + return 0; +} + SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -557,11 +609,6 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } -static int32_t mndProcessDropTopicInRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index bbee59090d1600693478d38fec9ff47082bcc032..8c0ba65fb295758ed3b0bcbd517259b6ca7846ed 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -88,12 +88,14 @@ static int32_t mndTransGetActionsSize(SArray *pArray) { for (int32_t i = 0; i < actionNum; ++i) { STransAction *pAction = taosArrayGet(pArray, i); - if (pAction->actionType) { + if (pAction->actionType == TRANS_ACTION_RAW) { rawDataLen += (sdbGetRawTotalSize(pAction->pRaw) + sizeof(int32_t)); - } else { + } else if (pAction->actionType == TRANS_ACTION_MSG) { rawDataLen += (sizeof(STransAction) + pAction->contLen); + } else { + // empty } - rawDataLen += sizeof(pAction->actionType); + rawDataLen += sizeof(int8_t); } return rawDataLen; @@ -137,18 +139,20 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - if (pAction->actionType) { + if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) SDB_SET_INT32(pRaw, dataPos, len, _OVER) SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pRaw, len, _OVER) - } else { + } else if (pAction->actionType == TRANS_ACTION_MSG) { SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->msgSent, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->msgReceived, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) + } else { + // nothing } } @@ -159,18 +163,20 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - if (pAction->actionType) { + if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) SDB_SET_INT32(pRaw, dataPos, len, _OVER) SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pRaw, len, _OVER) - } else { + } else if (pAction->actionType == TRANS_ACTION_MSG) { SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->msgSent, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->msgReceived, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) + } else { + // nothing } } @@ -181,18 +187,20 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER) - if (pAction->actionType) { + if (pAction->actionType == TRANS_ACTION_RAW) { int32_t len = sdbGetRawTotalSize(pAction->pRaw); SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER) SDB_SET_INT32(pRaw, dataPos, len, _OVER) SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pRaw, len, _OVER) - } else { + } else if (pAction->actionType == TRANS_ACTION_MSG) { SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet), _OVER) SDB_SET_INT16(pRaw, dataPos, pAction->msgType, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->msgSent, _OVER) SDB_SET_INT8(pRaw, dataPos, pAction->msgReceived, _OVER) SDB_SET_INT32(pRaw, dataPos, pAction->contLen, _OVER) SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen, _OVER) + } else { + // nothing } } @@ -252,6 +260,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { int16_t policy = 0; int16_t conflict = 0; int16_t exec = 0; + int8_t actionType = 0; SDB_GET_INT16(pRaw, dataPos, &stage, _OVER) SDB_GET_INT16(pRaw, dataPos, &policy, _OVER) SDB_GET_INT16(pRaw, dataPos, &conflict, _OVER) @@ -279,9 +288,10 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &action.actionType, _OVER) + SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) + action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &action.stage, _OVER) - if (action.actionType) { + if (action.actionType == TRANS_ACTION_RAW) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) action.pRaw = taosMemoryMalloc(dataLen); @@ -290,7 +300,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; action.pRaw = NULL; - } else { + } else if (action.actionType == TRANS_ACTION_MSG) { SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) SDB_GET_INT8(pRaw, dataPos, &action.msgSent, _OVER) @@ -301,6 +311,8 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; action.pCont = NULL; + } else { + if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; } } @@ -308,9 +320,10 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &action.actionType, _OVER) + SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) + action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &action.stage, _OVER) - if (action.actionType) { + if (action.actionType == TRANS_ACTION_RAW) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) action.pRaw = taosMemoryMalloc(dataLen); @@ -319,7 +332,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; action.pRaw = NULL; - } else { + } else if (action.actionType == TRANS_ACTION_MSG) { SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) SDB_GET_INT8(pRaw, dataPos, &action.msgSent, _OVER) @@ -330,6 +343,8 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER; action.pCont = NULL; + } else { + if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; } } @@ -337,7 +352,8 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER) SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER) - SDB_GET_INT8(pRaw, dataPos, &action.actionType, _OVER) + SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER) + action.actionType = actionType; SDB_GET_INT8(pRaw, dataPos, &action.stage, _OVER) if (action.actionType) { SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER) @@ -348,7 +364,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); if (taosArrayPush(pTrans->commitActions, &action) == NULL) goto _OVER; action.pRaw = NULL; - } else { + } else if (action.actionType == TRANS_ACTION_MSG) { SDB_GET_BINARY(pRaw, dataPos, (void *)&action.epSet, sizeof(SEpSet), _OVER); SDB_GET_INT16(pRaw, dataPos, &action.msgType, _OVER) SDB_GET_INT8(pRaw, dataPos, &action.msgSent, _OVER) @@ -359,6 +375,8 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, action.pCont, action.contLen, _OVER); if (taosArrayPush(pTrans->commitActions, &action) == NULL) goto _OVER; action.pCont = NULL; + } else { + if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER; } } @@ -552,10 +570,12 @@ static void mndTransDropActions(SArray *pArray) { int32_t size = taosArrayGetSize(pArray); for (int32_t i = 0; i < size; ++i) { STransAction *pAction = taosArrayGet(pArray, i); - if (pAction->actionType) { + if (pAction->actionType == TRANS_ACTION_RAW) { taosMemoryFreeClear(pAction->pRaw); - } else { + } else if (pAction->actionType == TRANS_ACTION_MSG) { taosMemoryFreeClear(pAction->pCont); + } else { + // nothing } } @@ -583,27 +603,34 @@ static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction) { } int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw) { - STransAction action = {.stage = TRN_STAGE_REDO_ACTION, .actionType = true, .pRaw = pRaw}; + STransAction action = {.stage = TRN_STAGE_REDO_ACTION, .actionType = TRANS_ACTION_RAW, .pRaw = pRaw}; + return mndTransAppendAction(pTrans->redoActions, &action); +} + +int32_t mndTransAppendNullLog(STrans *pTrans) { + STransAction action = {.stage = TRN_STAGE_REDO_ACTION, .actionType = TRANS_ACTION_NULL}; return mndTransAppendAction(pTrans->redoActions, &action); } int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw) { - STransAction action = {.stage = TRN_STAGE_UNDO_ACTION, .actionType = true, .pRaw = pRaw}; + STransAction action = {.stage = TRN_STAGE_UNDO_ACTION, .actionType = TRANS_ACTION_RAW, .pRaw = pRaw}; return mndTransAppendAction(pTrans->undoActions, &action); } int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) { - STransAction action = {.stage = TRN_STAGE_COMMIT_ACTION, .actionType = true, .pRaw = pRaw}; + STransAction action = {.stage = TRN_STAGE_COMMIT_ACTION, .actionType = TRANS_ACTION_RAW, .pRaw = pRaw}; return mndTransAppendAction(pTrans->commitActions, &action); } int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction) { pAction->stage = TRN_STAGE_REDO_ACTION; + pAction->actionType = TRANS_ACTION_MSG; return mndTransAppendAction(pTrans->redoActions, pAction); } int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction) { pAction->stage = TRN_STAGE_UNDO_ACTION; + pAction->actionType = TRANS_ACTION_MSG; return mndTransAppendAction(pTrans->undoActions, pAction); } @@ -782,7 +809,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { } } -void mndTransProcessRsp(SRpcMsg *pRsp) { +int32_t mndTransProcessRsp(SRpcMsg *pRsp) { SMnode *pMnode = pRsp->info.node; int64_t signature = (int64_t)(pRsp->info.ahandle); int32_t transId = (int32_t)(signature >> 32); @@ -827,6 +854,7 @@ void mndTransProcessRsp(SRpcMsg *pRsp) { _OVER: mndReleaseTrans(pMnode, pTrans); + return 0; } static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { @@ -842,8 +870,15 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) pAction->rawWritten = 0; pAction->msgSent = 0; pAction->msgReceived = 0; + if (pAction->errCode == TSDB_CODE_RPC_REDIRECT) { + pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps; + mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage), + action, pAction->epSet.inUse); + } else { + mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action); + } pAction->errCode = 0; - mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action); + } } @@ -899,10 +934,15 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio } static int32_t mndTransExecSingleAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { - if (pAction->actionType) { + if (pAction->actionType == TRANS_ACTION_RAW) { return mndTransWriteSingleLog(pMnode, pTrans, pAction); - } else { + } else if (pAction->actionType == TRANS_ACTION_MSG) { return mndTransSendSingleMsg(pMnode, pTrans, pAction); + } else { + pAction->rawWritten = 0; + pAction->errCode = 0; + mDebug("trans:%d, %s:%d null action executed", pTrans->id, mndTransStr(pAction->stage), pAction->id); + return 0; } } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 9262aa167b20372dcabc4d8b78cf311911cf2162..4f09eda733f44cc65e6d4cc7b3e65211a7d55cec 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -29,11 +29,6 @@ static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); -static int32_t mndProcessCreateVnodeRsp(SRpcMsg *pRsp); -static int32_t mndProcessAlterVnodeRsp(SRpcMsg *pRsp); -static int32_t mndProcessDropVnodeRsp(SRpcMsg *pRsp); -static int32_t mndProcessCompactVnodeRsp(SRpcMsg *pRsp); - static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); @@ -50,11 +45,12 @@ int32_t mndInitVgroup(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndVgroupActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp); - mndSetMsgHandle(pMnode, TDMT_VND_ALTER_REPLICA_RSP, mndProcessAlterVnodeRsp); - mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIG_RSP, mndProcessAlterVnodeRsp); - mndSetMsgHandle(pMnode, TDMT_DND_DROP_VNODE_RSP, mndProcessDropVnodeRsp); - mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndProcessCompactVnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_REPLICA_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIG_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_ALTER_CONFIRM_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_VNODE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndCancelGetNextVgroup); @@ -512,12 +508,12 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); } - int32_t maxPos = 1; + SVnodeGid *pVgid = &pVgroup->vnodeGid[pVgroup->replica]; for (int32_t d = 0; d < taosArrayGetSize(pArray); ++d) { SDnodeObj *pDnode = taosArrayGet(pArray, d); bool used = false; - for (int32_t vn = 0; vn < maxPos; ++vn) { + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { if (pDnode->id == pVgroup->vnodeGid[vn].dnodeId) { used = true; break; @@ -530,59 +526,58 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { return -1; } - SVnodeGid *pVgid = &pVgroup->vnodeGid[maxPos]; pVgid->dnodeId = pDnode->id; pVgid->role = TAOS_SYNC_STATE_ERROR; - pDnode->numOfVnodes++; + mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is added", pVgroup->dbName, pVgroup->vgId, pVgroup->replica, + pVgid->dnodeId); - mInfo("db:%s, vgId:%d, vnode_index:%d dnode:%d is added", pVgroup->dbName, pVgroup->vgId, maxPos, pVgid->dnodeId); - maxPos++; - if (maxPos == 3) return 0; + pVgroup->replica++; + pDnode->numOfVnodes++; + return 0; } terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + mError("db:%s, failed to add vnode to vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); return -1; } -int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2) { +int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid) { taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { SDnodeObj *pDnode = taosArrayGet(pArray, i); mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes); } - int32_t removedNum = 0; + int32_t code = -1; for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) { SDnodeObj *pDnode = taosArrayGet(pArray, d); - for (int32_t vn = 0; vn < TSDB_MAX_REPLICA; ++vn) { + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; if (pVgid->dnodeId == pDnode->id) { - if (removedNum == 0) *del1 = *pVgid; - if (removedNum == 1) *del2 = *pVgid; - - mInfo("db:%s, vgId:%d, vn:%d dnode:%d is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); - memset(pVgid, 0, sizeof(SVnodeGid)); - removedNum++; + mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); pDnode->numOfVnodes--; - - if (removedNum == 2) goto _OVER; + pVgroup->replica--; + *pDelVgid = *pVgid; + *pVgid = pVgroup->vnodeGid[pVgroup->replica]; + memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid)); + code = 0; + goto _OVER; } } } _OVER: - if (removedNum != 2) return -1; + if (code != 0) { + terrno = TSDB_CODE_APP_ERROR; + mError("db:%s, failed to remove vnode from vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr()); + return -1; + } - for (int32_t vn = 1; vn < TSDB_MAX_REPLICA; ++vn) { + for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { SVnodeGid *pVgid = &pVgroup->vnodeGid[vn]; - if (pVgid->dnodeId != 0) { - memcpy(&pVgroup->vnodeGid[0], pVgid, sizeof(SVnodeGid)); - memset(pVgid, 0, sizeof(SVnodeGid)); - } + mInfo("db:%s, vgId:%d, vn:%d dnode:%d is reserved", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId); } - - mInfo("db:%s, vgId:%d, dnode:%d is keeped", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); return 0; } @@ -605,23 +600,6 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) { return epset; } -static int32_t mndProcessCreateVnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessAlterVnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessDropVnodeRsp(SRpcMsg *pRsp) { - mndTransProcessRsp(pRsp); - return 0; -} - -static int32_t mndProcessCompactVnodeRsp(SRpcMsg *pRsp) { return 0; } - static bool mndGetVgroupMaxReplicaFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; int64_t uid = *(int64_t *)p1; @@ -705,7 +683,7 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p } char buf1[20] = {0}; - const char *role = online ? syncStr(pVgroup->vnodeGid[i].role) : "OFFLINE"; + const char *role = online ? syncStr(pVgroup->vnodeGid[i].role) : "offline"; STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/dnode/mnode/impl/test/acct/CMakeLists.txt b/source/dnode/mnode/impl/test/acct/CMakeLists.txt index d72292e34bd605ec91b16788fadd9f1ff1c68cc4..40f8b0726e28446170a71bbbccde979376448fbb 100644 --- a/source/dnode/mnode/impl/test/acct/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/acct/CMakeLists.txt @@ -5,9 +5,7 @@ target_link_libraries( PUBLIC sut ) -if(NOT TD_WINDOWS) - add_test( - NAME acctTest - COMMAND acctTest - ) -endif(NOT TD_WINDOWS) +add_test( + NAME acctTest + COMMAND acctTest +) diff --git a/source/dnode/mnode/impl/test/func/CMakeLists.txt b/source/dnode/mnode/impl/test/func/CMakeLists.txt index 2a8eb0a39d89275ae204e6405de2b774b4412619..17adfde801c02292a1e26a797541cd3a500e19c4 100644 --- a/source/dnode/mnode/impl/test/func/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/func/CMakeLists.txt @@ -5,9 +5,9 @@ target_link_libraries( PUBLIC sut ) -if(NOT TD_WINDOWS) +# if(NOT TD_WINDOWS) add_test( NAME funcTest COMMAND funcTest ) -endif(NOT TD_WINDOWS) +# endif(NOT TD_WINDOWS) diff --git a/source/dnode/mnode/impl/test/profile/CMakeLists.txt b/source/dnode/mnode/impl/test/profile/CMakeLists.txt index b6586192b2b4c6e428c2f00fddb11527a1747707..049a4f3a9f45d6608fdb403e99b95b937e93c483 100644 --- a/source/dnode/mnode/impl/test/profile/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/profile/CMakeLists.txt @@ -5,9 +5,9 @@ target_link_libraries( PUBLIC sut ) -if(NOT TD_WINDOWS) +# if(NOT TD_WINDOWS) add_test( NAME profileTest COMMAND profileTest ) -endif(NOT TD_WINDOWS) +# endif(NOT TD_WINDOWS) diff --git a/source/dnode/mnode/impl/test/show/CMakeLists.txt b/source/dnode/mnode/impl/test/show/CMakeLists.txt index 9b4e21501ed478e527adfa69a5a2297e173876e1..33ba0c5697d9188aaa1ec0328afc6f143e49413c 100644 --- a/source/dnode/mnode/impl/test/show/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/show/CMakeLists.txt @@ -5,9 +5,9 @@ target_link_libraries( PUBLIC sut ) -if(NOT TD_WINDOWS) +# if(NOT TD_WINDOWS) add_test( NAME showTest COMMAND showTest ) -endif(NOT TD_WINDOWS) +# endif(NOT TD_WINDOWS) diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 438982ac6ae2ca13f2244acd978bdc58c723d6de..45b88318c474dfd3f053bb8803e92be32defcecf 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -63,6 +63,14 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } +int32_t qndPreprocessQueryMsg(SQnode *pQnode, SRpcMsg * pMsg) { + if (TDMT_VND_QUERY != pMsg->msgType) { + return 0; + } + + return qWorkerPreprocessQueryMsg(pQnode->pQuery, pMsg); +} + int32_t qndProcessQueryMsg(SQnode *pQnode, int64_t ts, SRpcMsg *pMsg) { int32_t code = -1; SReadHandle handle = {.pMsgCb = &pQnode->msgCb}; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 890b8d08ab97b6f7db1bd0329466cb6f67db8910..05b4a270d6d6b7bdbf524c8c344b909a64a8c852 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -32,7 +32,7 @@ target_sources( "src/sma/smaEnv.c" "src/sma/smaOpen.c" "src/sma/smaRollup.c" - "src/sma/smaTimeRange.c" + "src/sma/smaTimeRange2.c" # tsdb "src/tsdb/tsdbCommit.c" diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 308f89736d2e1efd2263a8a6c130cc059574d243..8c7a78a5afb7b7620a1dc2f64ae8cc70b042bff6 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -56,6 +56,7 @@ int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp); int32_t vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); @@ -85,7 +86,7 @@ typedef struct SMetaFltParam { tb_uid_t suid; int16_t cid; int16_t type; - char *val; + char * val; bool reverse; int (*filterFunc)(void *a, void *b, int16_t type); @@ -120,7 +121,8 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo bool isTsdbCacheLastRow(tsdbReaderT *pReader); int32_t tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list); int32_t tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list); -void *tsdbGetIdx(SMeta *pMeta); +void * tsdbGetIdx(SMeta *pMeta); +void * tsdbGetIvtIdx(SMeta *pMeta); int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle); bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle); @@ -147,6 +149,9 @@ bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid, int32_t *pNumOfRows, int16_t *pNumOfCols); +// sma +int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); + // need to reposition // structs @@ -193,7 +198,7 @@ struct SMetaEntry { int64_t version; int8_t type; tb_uid_t uid; - char *name; + char * name; union { struct { SSchemaWrapper schemaRow; @@ -221,17 +226,17 @@ struct SMetaEntry { struct SMetaReader { int32_t flags; - SMeta *pMeta; + SMeta * pMeta; SDecoder coder; SMetaEntry me; - void *pBuf; + void * pBuf; int32_t szBuf; }; struct SMTbCursor { - TBC *pDbc; - void *pKey; - void *pVal; + TBC * pDbc; + void * pKey; + void * pVal; int32_t kLen; int32_t vLen; SMetaReader mr; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 52e576ee5249de37a9b30c137bff98f123999081..63a8e7ec6831071fc97b5c17f0cb375bd883cf24 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -28,12 +28,12 @@ typedef struct SMetaDB SMetaDB; // metaDebug ================== // clang-format off -#define metaFatal(...) do { if (metaDebugFlag & DEBUG_FATAL) { taosPrintLog("META FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) -#define metaError(...) do { if (metaDebugFlag & DEBUG_ERROR) { taosPrintLog("META ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) -#define metaWarn(...) do { if (metaDebugFlag & DEBUG_WARN) { taosPrintLog("META WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) -#define metaInfo(...) do { if (metaDebugFlag & DEBUG_INFO) { taosPrintLog("META ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) -#define metaDebug(...) do { if (metaDebugFlag & DEBUG_DEBUG) { taosPrintLog("META ", DEBUG_DEBUG, metaDebugFlag, __VA_ARGS__); }} while(0) -#define metaTrace(...) do { if (metaDebugFlag & DEBUG_TRACE) { taosPrintLog("META ", DEBUG_TRACE, metaDebugFlag, __VA_ARGS__); }} while(0) +#define metaFatal(...) do { if (metaDebugFlag & DEBUG_FATAL) { taosPrintLog("MTA FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) +#define metaError(...) do { if (metaDebugFlag & DEBUG_ERROR) { taosPrintLog("MTA ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) +#define metaWarn(...) do { if (metaDebugFlag & DEBUG_WARN) { taosPrintLog("MTA WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) +#define metaInfo(...) do { if (metaDebugFlag & DEBUG_INFO) { taosPrintLog("MTA ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) +#define metaDebug(...) do { if (metaDebugFlag & DEBUG_DEBUG) { taosPrintLog("MTA ", DEBUG_DEBUG, metaDebugFlag, __VA_ARGS__); }} while(0) +#define metaTrace(...) do { if (metaDebugFlag & DEBUG_TRACE) { taosPrintLog("MTA ", DEBUG_TRACE, metaDebugFlag, __VA_ARGS__); }} while(0) // clang-format on // metaOpen ================== @@ -69,12 +69,11 @@ struct SMeta { TTB* pUidIdx; TTB* pNameIdx; TTB* pCtbIdx; -#ifdef USE_INVERTED_INDEX + // ivt idx and idx void* pTagIvtIdx; -#else - TTB* pTagIdx; -#endif - TTB* pTtlIdx; + TTB* pTagIdx; + TTB* pTtlIdx; + TTB* pSmaIdx; SMetaIdx* pIdx; }; @@ -117,7 +116,7 @@ typedef struct { } SSmaIdxKey; // metaTable ================== -int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, +int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey); #ifndef META_REFACT diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 03dd2ea66cb387edeed3a4949936db5affb9592b..4ca62f1de9fbe9183d74e9df1dfeca8fbde2e0fb 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -223,6 +223,8 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t ve // TODO: This is the basic params, and should wrap the params to a queryHandle. int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); +int32_t tdGetTSmaDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a62b4c4409ae0a4561c5150e4d6bd669698e666c..171d52164c76688bd6cd3225c16b2486cc16befa 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -34,18 +34,97 @@ extern "C" { typedef struct TSDBROW TSDBROW; typedef struct TSDBKEY TSDBKEY; +typedef struct TABLEID TABLEID; typedef struct SDelOp SDelOp; static int tsdbKeyCmprFn(const void *p1, const void *p2); // tsdbMemTable2.c ============================================================================================== -typedef struct SMemTable SMemTable; +typedef struct SMemTable SMemTable; +typedef struct SMemData SMemData; +typedef struct SMemDataIter SMemDataIter; int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable); void tsdbMemTableDestroy2(SMemTable *pMemTable); +int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk); +int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); + +/* SMemDataIter */ +void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter); +bool tsdbMemDataIterNext(SMemDataIter *pIter); +void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow); + +// tsdbCommit2.c ============================================================================================== +int32_t tsdbBegin2(STsdb *pTsdb); +int32_t tsdbCommit2(STsdb *pTsdb); + +// tsdbFile.c ============================================================================================== +typedef int32_t TSDB_FILE_T; +typedef struct SDFInfo SDFInfo; +typedef struct SDFile SDFile; +typedef struct SDFileSet SDFileSet; + +// SDFile +void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype); +void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile); +int tsdbOpenDFile(SDFile *pDFile, int flags); +void tsdbCloseDFile(SDFile *pDFile); +int64_t tsdbSeekDFile(SDFile *pDFile, int64_t offset, int whence); +int64_t tsdbWriteDFile(SDFile *pDFile, void *buf, int64_t nbyte); +void tsdbUpdateDFileMagic(SDFile *pDFile, void *pCksm); +int tsdbAppendDFile(SDFile *pDFile, void *buf, int64_t nbyte, int64_t *offset); +int tsdbRemoveDFile(SDFile *pDFile); +int64_t tsdbReadDFile(SDFile *pDFile, void *buf, int64_t nbyte); +int tsdbCopyDFile(SDFile *pSrc, SDFile *pDest); +int tsdbEncodeSDFile(void **buf, SDFile *pDFile); +void *tsdbDecodeSDFile(STsdb *pRepo, void *buf, SDFile *pDFile); +int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T fType); +int tsdbUpdateDFileHeader(SDFile *pDFile); +int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo); +int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *version); + +// SDFileSet +void tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint32_t ver); +void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet); +int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet); +void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet); +int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet); +void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet); +int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to); +int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet *pSet, bool updateHeader); +int tsdbUpdateDFileSetHeader(SDFileSet *pSet); +int tsdbScanAndTryFixDFileSet(STsdb *pRepo, SDFileSet *pSet); +void tsdbCloseDFileSet(SDFileSet *pSet); +int tsdbOpenDFileSet(SDFileSet *pSet, int flags); +void tsdbRemoveDFileSet(SDFileSet *pSet); +int tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest); +void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey); + +// tsdbFS.c ============================================================================================== +typedef struct STsdbFS STsdbFS; +typedef struct SFSIter SFSIter; +typedef struct STsdbFSMeta STsdbFSMeta; + +STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg); +void *tsdbFreeFS(STsdbFS *pfs); +int tsdbOpenFS(STsdb *pRepo); +void tsdbCloseFS(STsdb *pRepo); +void tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd); +int tsdbEndFSTxn(STsdb *pRepo); +int tsdbEndFSTxnWithError(STsdbFS *pfs); +void tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta); +// void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile); +int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet); + +void tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction); +void tsdbFSIterSeek(SFSIter *pIter, int fid); +SDFileSet *tsdbFSIterNext(SFSIter *pIter); +int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta); +int tsdbRLockFS(STsdbFS *pFs); +int tsdbWLockFS(STsdbFS *pFs); +int tsdbUnLockFS(STsdbFS *pFs); // tsdbMemTable ================ -typedef struct STsdbRow STsdbRow; typedef struct STbData STbData; typedef struct STsdbMemTable STsdbMemTable; typedef struct SMergeInfo SMergeInfo; @@ -56,15 +135,6 @@ void tsdbMemTableDestroy(STsdbMemTable *pMemTable); int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); -// tsdbCommit ================ - -// tsdbFS ================ -typedef struct STsdbFS STsdbFS; - -// tsdbSma ================ -typedef struct SSmaEnv SSmaEnv; -typedef struct SSmaEnvs SSmaEnvs; - // structs typedef struct { int minFid; @@ -99,7 +169,7 @@ struct STable { #define TABLE_TID(t) (t)->tid #define TABLE_UID(t) (t)->uid -int tsdbPrepareCommit(STsdb *pTsdb); +// int tsdbPrepareCommit(STsdb *pTsdb); typedef enum { TSDB_FILE_HEAD = 0, // .head TSDB_FILE_DATA, // .data @@ -110,7 +180,7 @@ typedef enum { TSDB_FILE_META, // meta } E_TSDB_FILE_T; -typedef struct { +struct SDFInfo { uint32_t magic; uint32_t fver; uint32_t len; @@ -119,22 +189,22 @@ typedef struct { uint32_t offset; uint64_t size; uint64_t tombSize; -} SDFInfo; +}; -typedef struct { +struct SDFile { SDFInfo info; STfsFile f; TdFilePtr pFile; uint8_t state; -} SDFile; +}; -typedef struct { +struct SDFileSet { int fid; int8_t state; // -128~127 uint8_t ver; // 0~255, DFileSet version uint16_t reserve; SDFile files[TSDB_FILE_MAX]; -} SDFileSet; +}; struct STbData { tb_uid_t uid; @@ -159,11 +229,11 @@ struct STsdbMemTable { SHashObj *pHashIdx; }; -typedef struct { +struct STsdbFSMeta { uint32_t version; // Commit version from 0 to increase int64_t totalPoints; // total points int64_t totalStorage; // Uncompressed total storage -} STsdbFSMeta; +}; // ================== typedef struct { @@ -175,9 +245,7 @@ typedef struct { struct STsdbFS { TdThreadRwlock lock; - SFSStatus *cstatus; // current status - SHashObj *metaCache; // meta cache - SHashObj *metaCacheComp; // meta cache for compact + SFSStatus *cstatus; // current status bool intxn; SFSStatus *nstatus; // new status }; @@ -243,31 +311,21 @@ static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) { // tsdbReadImpl typedef struct SReadH SReadH; +struct TSDBKEY { + int64_t version; + TSKEY ts; +}; + typedef struct { + uint64_t suid; + uint64_t uid; uint32_t len; uint32_t offset; uint32_t hasLast : 2; uint32_t numOfBlocks : 30; - uint64_t uid; - TSKEY maxKey; + TSDBKEY maxKey; } SBlockIdx; -#ifdef TD_REFACTOR_3 -typedef struct { - int64_t last : 1; - int64_t offset : 63; - int32_t algorithm : 8; - int32_t numOfRows : 24; - int32_t len; - int32_t keyLen; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols - int16_t numOfSubBlocks; - int16_t numOfCols; // not including timestamp column - TSKEY keyFirst; - TSKEY keyLast; -} SBlock; - -#else - typedef enum { TSDB_SBLK_VER_0 = 0, TSDB_SBLK_VER_MAX, @@ -290,16 +348,9 @@ typedef struct { int64_t offset; uint64_t aggrStat : 1; uint64_t aggrOffset : 63; - TSKEY keyFirst; - TSKEY keyLast; -} SBlockV0; - -#define SBlock SBlockV0 // latest SBlock definition - -static FORCE_INLINE bool tsdbIsSupBlock(SBlock *pBlock) { return pBlock->numOfSubBlocks == 1; } -static FORCE_INLINE bool tsdbIsSubBlock(SBlock *pBlock) { return pBlock->numOfSubBlocks == 0; } - -#endif + TSDBKEY minKey; + TSDBKEY maxKey; +} SBlock; typedef struct { int32_t delimiter; // For recovery usage @@ -308,33 +359,13 @@ typedef struct { SBlock blocks[]; } SBlockInfo; -#ifdef TD_REFACTOR_3 -typedef struct { - int16_t colId; - uint16_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows - uint16_t reserve : 15; - int32_t len; - uint32_t type : 8; - uint32_t offset : 24; - int64_t sum; - int64_t max; - int64_t min; - int16_t maxIndex; - int16_t minIndex; - int16_t numOfNull; - uint8_t offsetH; - char padding[1]; -} SBlockCol; -#else typedef struct { int16_t colId; uint16_t type : 6; uint16_t blen : 10; // 0 no bitmap if all rows are NORM, > 0 bitmap length uint32_t len; // data length + bitmap length uint32_t offset; -} SBlockColV0; - -#define SBlockCol SBlockColV0 // latest SBlockCol definition +} SBlockCol; typedef struct { int16_t colId; @@ -344,31 +375,7 @@ typedef struct { int64_t sum; int64_t max; int64_t min; -} SAggrBlkColV0; - -#define SAggrBlkCol SAggrBlkColV0 // latest SAggrBlkCol definition - -#endif - -// Code here just for back-ward compatibility -static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) { -#ifdef TD_REFACTOR_3 - pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1); - pBlockCol->offsetH = (uint8_t)(offset >> 24); -#else - pBlockCol->offset = offset; -#endif -} - -static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) { -#ifdef TD_REFACTOR_3 - uint32_t offset1 = pBlockCol->offset; - uint32_t offset2 = pBlockCol->offsetH; - return (offset1 | (offset2 << 24)); -#else - return pBlockCol->offset; -#endif -} +} SAggrBlkCol; typedef struct { int32_t delimiter; // For recovery usage @@ -409,8 +416,7 @@ struct SReadH { #define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf) #define TSDB_READ_EXBUF(rh) ((rh)->pExBuf) -#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) \ - (sizeof(SBlockData) + sizeof(SBlockColV##blkVer) * (ncols) + sizeof(TSCKSUM)) +#define TSDB_BLOCK_STATIS_SIZE(ncols, blkVer) (sizeof(SBlockData) + sizeof(SBlockCol) * (ncols) + sizeof(TSCKSUM)) static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) { switch (blkVer) { @@ -420,7 +426,7 @@ static FORCE_INLINE size_t tsdbBlockStatisSize(int nCols, uint32_t blkVer) { } } -#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkColV##blkVer) * (ncols) + sizeof(TSCKSUM)) +#define TSDB_BLOCK_AGGR_SIZE(ncols, blkVer) (sizeof(SAggrBlkCol) * (ncols) + sizeof(TSCKSUM)) static FORCE_INLINE size_t tsdbBlockAggrSize(int nCols, uint32_t blkVer) { switch (blkVer) { @@ -518,11 +524,11 @@ static FORCE_INLINE void *taosTZfree(void *ptr) { void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn); -static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { +static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t minutes, int8_t precision) { if (key < 0) { - return (int)((key + 1) / tsTickPerMin[precision] / days - 1); + return (int)((key + 1) / tsTickPerMin[precision] / minutes - 1); } else { - return (int)((key / tsTickPerMin[precision] / days)); + return (int)((key / tsTickPerMin[precision] / minutes)); } } @@ -546,7 +552,6 @@ static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) { #define TSDB_FILE_STATE_OK 0 #define TSDB_FILE_STATE_BAD 1 -#define TSDB_FILE_INFO(tf) (&((tf)->info)) #define TSDB_FILE_F(tf) (&((tf)->f)) #define TSDB_FILE_PFILE(tf) ((tf)->pFile) #define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname) @@ -564,7 +569,6 @@ static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) { #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) #define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD) -typedef int32_t TSDB_FILE_T; typedef enum { TSDB_FS_VER_0 = 0, TSDB_FS_VER_MAX, @@ -585,129 +589,9 @@ static FORCE_INLINE uint32_t tsdbGetDFSVersion(TSDB_FILE_T fType) { // latest v } } -void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype); -void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile); -int tsdbEncodeSDFile(void **buf, SDFile *pDFile); -void *tsdbDecodeSDFile(STsdb *pRepo, void *buf, SDFile *pDFile); -int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T fType); -int tsdbUpdateDFileHeader(SDFile *pDFile); -int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo); -int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *version); - -static FORCE_INLINE void tsdbSetDFileInfo(SDFile *pDFile, SDFInfo *pInfo) { pDFile->info = *pInfo; } - -static FORCE_INLINE int tsdbOpenDFile(SDFile *pDFile, int flags) { - ASSERT(!TSDB_FILE_OPENED(pDFile)); - - pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags); - if (pDFile->pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return 0; -} - -static FORCE_INLINE void tsdbCloseDFile(SDFile *pDFile) { - if (TSDB_FILE_OPENED(pDFile)) { - taosCloseFile(&pDFile->pFile); - TSDB_FILE_SET_CLOSED(pDFile); - } -} - -static FORCE_INLINE int64_t tsdbSeekDFile(SDFile *pDFile, int64_t offset, int whence) { - // ASSERT(TSDB_FILE_OPENED(pDFile)); - - int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence); - if (loffset < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return loffset; -} - -static FORCE_INLINE int64_t tsdbWriteDFile(SDFile *pDFile, void *buf, int64_t nbyte) { - ASSERT(TSDB_FILE_OPENED(pDFile)); - - int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte); - if (nwrite < nbyte) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return nwrite; -} - -static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile *pDFile, void *pCksm) { - pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t *)(pCksm), sizeof(TSCKSUM)); -} - -static FORCE_INLINE int tsdbAppendDFile(SDFile *pDFile, void *buf, int64_t nbyte, int64_t *offset) { - ASSERT(TSDB_FILE_OPENED(pDFile)); - - int64_t toffset; - - if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) { - return -1; - } - - ASSERT(pDFile->info.size == toffset); - - if (offset) { - *offset = toffset; - } - - if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) { - return -1; - } - - pDFile->info.size += nbyte; - - return (int)nbyte; -} - -static FORCE_INLINE int tsdbRemoveDFile(SDFile *pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); } - -static FORCE_INLINE int64_t tsdbReadDFile(SDFile *pDFile, void *buf, int64_t nbyte) { - ASSERT(TSDB_FILE_OPENED(pDFile)); - - int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte); - if (nread < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return nread; -} - -static FORCE_INLINE int tsdbCopyDFile(SDFile *pSrc, SDFile *pDest) { - if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc)); - return 0; -} - // =============== SDFileSet -typedef struct { - int fid; - int8_t state; - uint8_t ver; - uint16_t reserve; -#if 0 - SDFInfo info; -#endif - STfsFile f; - TdFilePtr pFile; - -} SSFile; // files split by days with fid - -#define TSDB_LATEST_FSET_VER 0 - +#define TSDB_LATEST_FSET_VER 0 #define TSDB_FSET_FID(s) ((s)->fid) #define TSDB_FSET_STATE(s) ((s)->state) #define TSDB_FSET_VER(s) ((s)->ver) @@ -727,55 +611,6 @@ typedef struct { } \ } while (0); -void tsdbInitDFileSet(STsdb *pRepo, SDFileSet *pSet, SDiskID did, int fid, uint32_t ver); -void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet); -int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet); -void *tsdbDecodeDFileSet(STsdb *pRepo, void *buf, SDFileSet *pSet); -int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet); -void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet); -int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to); -int tsdbCreateDFileSet(STsdb *pRepo, SDFileSet *pSet, bool updateHeader); -int tsdbUpdateDFileSetHeader(SDFileSet *pSet); -int tsdbScanAndTryFixDFileSet(STsdb *pRepo, SDFileSet *pSet); - -static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet *pSet) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype)); - } -} - -static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet *pSet, int flags) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) { - tsdbCloseDFileSet(pSet); - return -1; - } - } - return 0; -} - -static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet *pSet) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype)); - } -} - -static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) { - tsdbRemoveDFileSet(pDest); - return -1; - } - } - - return 0; -} - -static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey) { - *minKey = fid * days * tsTickPerMin[precision]; - *maxKey = *minKey + days * tsTickPerMin[precision] - 1; -} - static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet *pSet) { for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) { @@ -803,69 +638,26 @@ typedef struct { #define FS_VERSION(pfs) ((pfs)->cstatus->meta.version) #define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version) -typedef struct { +struct SFSIter { int direction; uint64_t version; // current FS version STsdbFS *pfs; int index; // used to position next fset when version the same int fid; // used to seek when version is changed SDFileSet *pSet; -} SFSIter; +}; #define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC #define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC -STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg); -void *tsdbFreeFS(STsdbFS *pfs); -int tsdbOpenFS(STsdb *pRepo); -void tsdbCloseFS(STsdb *pRepo); -void tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd); -int tsdbEndFSTxn(STsdb *pRepo); -int tsdbEndFSTxnWithError(STsdbFS *pfs); -void tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta); -// void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile); -int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet); - -void tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction); -void tsdbFSIterSeek(SFSIter *pIter, int fid); -SDFileSet *tsdbFSIterNext(SFSIter *pIter); -int tsdbLoadMetaCache(STsdb *pRepo, bool recoverMeta); - -static FORCE_INLINE int tsdbRLockFS(STsdbFS *pFs) { - int code = taosThreadRwlockRdlock(&(pFs->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int tsdbWLockFS(STsdbFS *pFs) { - int code = taosThreadRwlockWrlock(&(pFs->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int tsdbUnLockFS(STsdbFS *pFs) { - int code = taosThreadRwlockUnlock(&(pFs->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - struct TSDBROW { int64_t version; STSRow2 tsRow; }; -struct TSDBKEY { - int64_t version; - TSKEY ts; +struct TABLEID { + tb_uid_t suid; + tb_uid_t uid; }; struct SDelOp { @@ -875,6 +667,25 @@ struct SDelOp { SDelOp *pNext; }; +typedef struct { + tb_uid_t suid; + tb_uid_t uid; + int64_t version; + TSKEY sKey; + TSKEY eKey; +} SDelInfo; + +struct SMemTable { + STsdb *pTsdb; + int32_t nRef; + TSDBKEY minKey; + TSDBKEY maxKey; + int64_t nRows; + int64_t nDelOp; + SArray *aSkmInfo; + SArray *aMemData; +}; + static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; TSDBKEY *pKey2 = (TSDBKEY *)p2; @@ -894,6 +705,34 @@ static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) { return 0; } +typedef struct SMemSkipListNode SMemSkipListNode; +typedef struct SMemSkipList { + uint32_t seed; + int32_t size; + int8_t maxLevel; + int8_t level; + SMemSkipListNode *pHead; + SMemSkipListNode *pTail; +} SMemSkipList; + +struct SMemData { + tb_uid_t suid; + tb_uid_t uid; + TSDBKEY minKey; + TSDBKEY maxKey; + SDelOp *delOpHead; + SDelOp *delOpTail; + SMemSkipList sl; +}; + +struct SMemDataIter { + SMemData *pMemData; + int8_t backward; + TSDBROW *pRow; + SMemSkipListNode *pNode; // current node + TSDBROW row; +}; + #endif #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index a5907cf9912aa9af0caff46e0aea85d690076cbb..5f4f7e70daf089d22fa4e80978e787d10dc08c09 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -84,7 +84,7 @@ int32_t vnodeAsyncCommit(SVnode* pVnode); int32_t vnodeSyncOpen(SVnode* pVnode, char* path); void vnodeSyncStart(SVnode* pVnode); void vnodeSyncClose(SVnode* pVnode); -void vnodeSyncAlter(SVnode* pVnode, SRpcMsg* pMsg); +int32_t vnodeSyncAlter(SVnode* pVnode, SRpcMsg* pMsg); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index d3b5f29aac9e1ed96d8fe3569065ecaa8a1a1706..15469786e0a00f81fb2a3de5505275fed9e37f1e 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -87,7 +87,7 @@ int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* p int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids); -int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp *pMetaRsp); +int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); int metaGetTableEntryByName(SMetaReader* pReader, const char* name); @@ -104,6 +104,7 @@ int32_t metaSnapshotReaderOpen(SMeta* pMeta, SMetaSnapshotReader** ppRea int32_t metaSnapshotReaderClose(SMetaSnapshotReader* pReader); int32_t metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nData); void* metaGetIdx(SMeta* pMeta); +void* metaGetIvtIdx(SMeta* pMeta); int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg); int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); @@ -112,7 +113,7 @@ int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid); int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg); int tsdbClose(STsdb** pTsdb); int tsdbBegin(STsdb* pTsdb); -int tsdbCommit(STsdb* pTsdb); +int32_t tsdbCommit(STsdb* pTsdb); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index f23e7f88056d6a397a5979bda11dd4f080ba0212..95510c25e57e58d2e739dae0e3b1ec630bc2e596 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -53,86 +53,83 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { // open env ret = tdbOpen(pMeta->path, pVnode->config.szPage, pVnode->config.szCache, &pMeta->pEnv); if (ret < 0) { - metaError("vgId:%d failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d, failed to open meta env since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTbDb ret = tdbTbOpen("table.db", sizeof(STbDbKey), -1, tbDbKeyCmpr, pMeta->pEnv, &pMeta->pTbDb); if (ret < 0) { - metaError("vgId:%d failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d, failed to open meta table db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pSkmDb ret = tdbTbOpen("schema.db", sizeof(SSkmDbKey), -1, skmDbKeyCmpr, pMeta->pEnv, &pMeta->pSkmDb); if (ret < 0) { - metaError("vgId:%d failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d, failed to open meta schema db since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pUidIdx ret = tdbTbOpen("uid.idx", sizeof(tb_uid_t), sizeof(int64_t), uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pUidIdx); if (ret < 0) { - metaError("vgId:%d failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d, failed to open meta uid idx since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pNameIdx ret = tdbTbOpen("name.idx", -1, sizeof(tb_uid_t), NULL, pMeta->pEnv, &pMeta->pNameIdx); if (ret < 0) { - metaError("vgId:%d failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d, failed to open meta name index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pCtbIdx ret = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); if (ret < 0) { - metaError("vgId:%d failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d, failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pTagIdx -#ifdef USE_INVERTED_INDEX // TODO(yihaoDeng), refactor later char indexFullPath[128] = {0}; sprintf(indexFullPath, "%s/%s", pMeta->path, "invert"); taosMkDir(indexFullPath); ret = indexOpen(indexOptsCreate(), indexFullPath, (SIndex **)&pMeta->pTagIvtIdx); if (ret < 0) { - metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } -#else ret = tdbTbOpen("tag.idx", -1, 0, tagIdxKeyCmpr, pMeta->pEnv, &pMeta->pTagIdx); if (ret < 0) { - metaError("vgId:%d failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d, failed to open meta tag index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } -#endif // open pTtlIdx ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx); if (ret < 0) { - metaError("vgId:%d failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d, failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open pSmaIdx ret = tdbTbOpen("sma.idx", sizeof(SSmaIdxKey), 0, smaIdxKeyCmpr, pMeta->pEnv, &pMeta->pSmaIdx); if (ret < 0) { - metaError("vgId:%d failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d, failed to open meta sma index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open index if (metaOpenIdx(pMeta) < 0) { - metaError("vgId:%d failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno)); + metaError("vgId:%d, failed to open meta index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } - metaDebug("vgId:%d meta is opened", TD_VID(pVnode)); + metaDebug("vgId:%d, meta is opened", TD_VID(pVnode)); *ppMeta = pMeta; return 0; @@ -141,11 +138,8 @@ _err: if (pMeta->pIdx) metaCloseIdx(pMeta); if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx); if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx); -#ifdef USE_INVERTED_INDEX if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); -#else if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); -#endif if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index a57eb4e899a2fc18d8a8a1f8c7bcd0d80a528f07..21a55d646331dd2ecb0a8c69c22edf4e6bd0f8cf 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -425,7 +425,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { for (int i = 0; i < pSW->number; ++i) { smaId = *(tb_uid_t *)taosArrayGet(pSmaIds, i); if (metaGetTableEntryByUid(&mr, smaId) < 0) { - metaWarn("vgId:%d no entry for tbId: %" PRIi64 ", smaId: %" PRIi64, TD_VID(pMeta->pVnode), uid, smaId); + metaWarn("vgId:%d, no entry for tbId: %" PRIi64 ", smaId: %" PRIi64, TD_VID(pMeta->pVnode), uid, smaId); continue; } pTSma = pSW->tSma + smaIdx; @@ -464,7 +464,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { _err: metaReaderClear(&mr); taosArrayDestroy(pSmaIds); - tdFreeTSmaWrapper(pSW, deepCopy); + tFreeTSmaWrapper(pSW, deepCopy); return NULL; } @@ -473,7 +473,7 @@ STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { SMetaReader mr = {0}; metaReaderInit(&mr, pMeta, 0); if (metaGetTableEntryByUid(&mr, indexUid) < 0) { - metaWarn("vgId:%d failed to get table entry for smaId: %" PRIi64, TD_VID(pMeta->pVnode), indexUid); + metaWarn("vgId:%d, failed to get table entry for smaId: %" PRIi64, TD_VID(pMeta->pVnode), indexUid); metaReaderClear(&mr); return NULL; } @@ -576,15 +576,15 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) { const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) { ASSERT(pEntry->type == TSDB_CHILD_TABLE); STag *tag = (STag *)pEntry->ctbEntry.pTags; - if (type == TSDB_DATA_TYPE_JSON){ - if(tag->nTag == 0){ + if (type == TSDB_DATA_TYPE_JSON) { + if (tag->nTag == 0) { return NULL; } return tag; } bool find = tTagGet(tag, val); - if(!find){ + if (!find) { return NULL; } return val; @@ -605,8 +605,6 @@ typedef struct { int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { SIdxCursor *pCursor = NULL; - char *tagData = param->val; - int32_t ret = 0, valid = 0; pCursor = (SIdxCursor *)taosMemoryCalloc(1, sizeof(SIdxCursor)); pCursor->pMeta = pMeta; @@ -623,12 +621,16 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { int32_t nKey = 0; int32_t nTagData = 0; - if(IS_VAR_DATA_TYPE(param->type)){ - nTagData = strlen(param->val); - }else{ + void * tagData = NULL; + + if (IS_VAR_DATA_TYPE(param->type)) { + tagData = varDataVal(param->val); + nTagData = varDataLen(param->val); + } else { + tagData = param->val; nTagData = tDataTypes[param->type].bytes; } - ret = metaCreateTagIdxKey(pCursor->suid, pCursor->cid, param->val, nTagData, pCursor->type, + ret = metaCreateTagIdxKey(pCursor->suid, pCursor->cid, tagData, nTagData, pCursor->type, param->reverse ? INT64_MAX : INT64_MIN, &pKey, &nKey); if (ret != 0) { goto END; @@ -637,6 +639,7 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { if (tdbTbcMoveTo(pCursor->pCur, pKey, nKey, &cmp) < 0) { goto END; } + void * entryKey = NULL, *entryVal = NULL; int32_t nEntryKey, nEntryVal; while (1) { @@ -649,7 +652,12 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); if (cmp == 0) { // match - tb_uid_t tuid = *(tb_uid_t *)(p->data + tDataTypes[pCursor->type].bytes); + tb_uid_t tuid = 0; + if (IS_VAR_DATA_TYPE(pKey->type)) { + tuid = *(tb_uid_t *)(p->data + varDataTLen(p->data)); + } else { + tuid = *(tb_uid_t *)(p->data + tDataTypes[pCursor->type].bytes); + } taosArrayPush(pUids, &tuid); } else if (cmp == 1) { // not match but should continue to iter @@ -670,4 +678,4 @@ END: taosMemoryFree(pCursor); return ret; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 75595d83a64941e0caf6f2f399345c09a226286e..689cd511c40cf89e465602dac341558e3e90092d 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -28,9 +28,9 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { int vLen = 0; const void *pKey = NULL; const void *pVal = NULL; - void *pBuf = NULL; + void * pBuf = NULL; int32_t szBuf = 0; - void *p = NULL; + void * p = NULL; SMetaReader mr = {0}; // validate req @@ -57,12 +57,12 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) { if (metaHandleSmaEntry(pMeta, &me) < 0) goto _err; - metaDebug("vgId:%d tsma is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pCfg->indexName, pCfg->indexUid); + metaDebug("vgId:%d, tsma is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pCfg->indexName, pCfg->indexUid); return 0; _err: - metaError("vgId:%d failed to create tsma: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pCfg->indexName, + metaError("vgId:%d, failed to create tsma: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pCfg->indexName, pCfg->indexUid, tstrerror(terrno)); return -1; } @@ -83,8 +83,8 @@ int32_t metaDropTSma(SMeta *pMeta, int64_t indexUid) { static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) { STbDbKey tbDbKey; - void *pKey = NULL; - void *pVal = NULL; + void * pKey = NULL; + void * pVal = NULL; int kLen = 0; int vLen = 0; SEncoder coder = {0}; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index cb00f9f88711f56c36b6f79b0d06966c0d53aab0..c6a6dd81473d75a319fb19ebc6bfa906eebbf6b0 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -15,6 +15,7 @@ #include "meta.h" +static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME); static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME); @@ -25,7 +26,7 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type); -static int metaUpdateMetaRsp(tb_uid_t uid, char* tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) { +static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) { pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema)); if (NULL == pMetaRsp->pSchemas) { terrno = TSDB_CODE_VND_OUT_OF_MEMORY; @@ -43,15 +44,77 @@ static int metaUpdateMetaRsp(tb_uid_t uid, char* tbName, SSchemaWrapper *pSchema return 0; } +static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema) { +#ifdef USE_INVERTED_INDEX + if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) { + return -1; + } + void * data = pCtbEntry->ctbEntry.pTags; + const char *tagName = pSchema->name; + + tb_uid_t suid = pCtbEntry->ctbEntry.suid; + tb_uid_t tuid = pCtbEntry->uid; + const void *pTagData = pCtbEntry->ctbEntry.pTags; + int32_t nTagData = 0; + + SArray *pTagVals = NULL; + if (tTagToValArray((const STag *)data, &pTagVals) != 0) { + return -1; + } + char key[512] = {0}; + + SIndexMultiTerm *terms = indexMultiTermCreate(); + int16_t nCols = taosArrayGetSize(pTagVals); + for (int i = 0; i < nCols; i++) { + STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); + char type = pTagVal->type; + sprintf(key, "%s_%s", tagName, pTagVal->pKey); + int32_t nKey = strlen(key); + + SIndexTerm *term = NULL; + if (type == TSDB_DATA_TYPE_NULL) { + // handle null value + } else if (type == TSDB_DATA_TYPE_NCHAR) { + if (pTagVal->nData > 0) { + char * val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); + int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); + memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); + type = TSDB_DATA_TYPE_VARCHAR; + term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, val, len); + } else if (pTagVal->nData == 0) { + char * val = NULL; + int32_t len = 0; + // handle NULL key + } + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + double val = *(double *)(&pTagVal->i64); + int len = 0; + term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len); + } else if (type == TSDB_DATA_TYPE_BOOL) { + int val = *(int *)(&pTagVal->i64); + int len = 0; + term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len); + } + if (term != NULL) { + indexMultiTermAdd(terms, term); + } + memset(key, 0, sizeof(key)); + } + tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid); + indexMultiTermDestroy(terms); +#endif + return 0; +} + int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry me = {0}; int kLen = 0; int vLen = 0; const void *pKey = NULL; const void *pVal = NULL; - void *pBuf = NULL; + void * pBuf = NULL; int32_t szBuf = 0; - void *p = NULL; + void * p = NULL; SMetaReader mr = {0}; // validate req @@ -79,12 +142,12 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { if (metaHandleEntry(pMeta, &me) < 0) goto _err; - metaDebug("vgId:%d super table is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + metaDebug("vgId:%d, super table is created, name:%s uid: %" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); return 0; _err: - metaError("vgId:%d failed to create super table: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, + metaError("vgId:%d, failed to create super table: %s uid: %" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->suid, tstrerror(terrno)); return -1; } @@ -105,7 +168,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) { } // drop all child tables - TBC *pCtbIdxc = NULL; + TBC * pCtbIdxc = NULL; SArray *pArray = taosArrayInit(8, sizeof(tb_uid_t)); tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); @@ -153,15 +216,15 @@ _drop_super_table: _exit: tdbFree(pKey); tdbFree(pData); - metaDebug("vgId:%d super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + metaDebug("vgId:%d, super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid); return 0; } int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry oStbEntry = {0}; SMetaEntry nStbEntry = {0}; - TBC *pUidIdxc = NULL; - TBC *pTbDbc = NULL; + TBC * pUidIdxc = NULL; + TBC * pTbDbc = NULL; const void *pData; int nData; int64_t oversion; @@ -269,18 +332,18 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq) { if (metaHandleEntry(pMeta, &me) < 0) goto _err; - metaDebug("vgId:%d table %s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, + metaDebug("vgId:%d, table %s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, pReq->type); return 0; _err: - metaError("vgId:%d failed to create table:%s type:%s since %s", TD_VID(pMeta->pVnode), pReq->name, + metaError("vgId:%d, failed to create table:%s type:%s since %s", TD_VID(pMeta->pVnode), pReq->name, pReq->type == TSDB_CHILD_TABLE ? "child table" : "normal table", tstrerror(terrno)); return -1; } int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) { - void *pData = NULL; + void * pData = NULL; int nData = 0; int rc = 0; tb_uid_t uid; @@ -306,7 +369,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi } static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { - void *pData = NULL; + void * pData = NULL; int nData = 0; int rc = 0; int64_t version; @@ -344,12 +407,12 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) { void * pVal = NULL; int nVal = 0; - const void *pData = NULL; + const void * pData = NULL; int nData = 0; int ret = 0; tb_uid_t uid; int64_t oversion; - SSchema *pColumn = NULL; + SSchema * pColumn = NULL; SMetaEntry entry = {0}; SSchemaWrapper *pSchema; int c; @@ -499,7 +562,7 @@ _err: static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { SMetaEntry ctbEntry = {0}; SMetaEntry stbEntry = {0}; - void *pVal = NULL; + void * pVal = NULL; int nVal = 0; int ret; int c; @@ -530,7 +593,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA oversion = *(int64_t *)pData; // search table.db - TBC *pTbDbc = NULL; + TBC * pTbDbc = NULL; SDecoder dc1 = {0}; SDecoder dc2 = {0}; @@ -554,7 +617,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaDecodeEntry(&dc2, &stbEntry); SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; - SSchema *pColumn = NULL; + SSchema * pColumn = NULL; int32_t iCol = 0; for (;;) { pColumn = NULL; @@ -584,8 +647,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); } else { const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags; - STag *pNewTag = NULL; - SArray *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); + STag * pNewTag = NULL; + SArray * pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); if (!pTagArray) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -669,8 +732,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { STbDbKey tbDbKey; - void *pKey = NULL; - void *pVal = NULL; + void * pKey = NULL; + void * pVal = NULL; int kLen = 0; int vLen = 0; SEncoder coder = {0}; @@ -753,16 +816,11 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, STagIdxKey **ppTagIdxKey, int32_t *nTagIdxKey) { - // int32_t nTagData = 0; - - // if (pTagData) { - // if (IS_VAR_DATA_TYPE(type)) { - // nTagData = varDataTLen(pTagData); - // } else { - // nTagData = tDataTypes[type].bytes; - // } - // } - *nTagIdxKey = sizeof(STagIdxKey) + nTagData + sizeof(tb_uid_t); + if (IS_VAR_DATA_TYPE(type)) { + *nTagIdxKey = sizeof(STagIdxKey) + nTagData + VARSTR_HEADER_SIZE + sizeof(tb_uid_t); + } else { + *nTagIdxKey = sizeof(STagIdxKey) + nTagData + sizeof(tb_uid_t); + } *ppTagIdxKey = (STagIdxKey *)taosMemoryMalloc(*nTagIdxKey); if (*ppTagIdxKey == NULL) { @@ -774,8 +832,16 @@ int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_ (*ppTagIdxKey)->cid = cid; (*ppTagIdxKey)->isNull = (pTagData == NULL) ? 1 : 0; (*ppTagIdxKey)->type = type; - if (nTagData) memcpy((*ppTagIdxKey)->data, pTagData, nTagData); - *(tb_uid_t *)((*ppTagIdxKey)->data + nTagData) = uid; + + // refactor + if (IS_VAR_DATA_TYPE(type)) { + memcpy((*ppTagIdxKey)->data, (uint16_t *)&nTagData, VARSTR_HEADER_SIZE); + memcpy((*ppTagIdxKey)->data + VARSTR_HEADER_SIZE, pTagData, nTagData); + *(tb_uid_t *)((*ppTagIdxKey)->data + VARSTR_HEADER_SIZE + nTagData) = uid; + } else { + memcpy((*ppTagIdxKey)->data, pTagData, nTagData); + *(tb_uid_t *)((*ppTagIdxKey)->data + nTagData) = uid; + } return 0; } @@ -785,14 +851,14 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) { } static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { - void *pData = NULL; + void * pData = NULL; int nData = 0; STbDbKey tbDbKey = {0}; SMetaEntry stbEntry = {0}; - STagIdxKey *pTagIdxKey = NULL; + STagIdxKey * pTagIdxKey = NULL; int32_t nTagIdxKey; const SSchema *pTagColumn; // = &stbEntry.stbEntry.schema.pSchema[0]; - const void *pTagData = NULL; // + const void * pTagData = NULL; // int32_t nTagData = 0; SDecoder dc = {0}; @@ -820,28 +886,16 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { } else { // pTagData = pCtbEntry->ctbEntry.pTags; // nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; + pTagData = pCtbEntry->ctbEntry.pTags; + nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; + return metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn); } - - // update tag index -#ifdef USE_INVERTED_INDEX - tb_uid_t suid = pCtbEntry->ctbEntry.suid; - tb_uid_t tuid = pCtbEntry->uid; - - SIndexMultiTerm *tmGroup = indexMultiTermCreate(); - - SIndexTerm *tm = indexTermCreate(suid, ADD_VALUE, pTagColumn->type, pTagColumn->name, sizeof(pTagColumn->name), - pTagData, pTagData == NULL ? 0 : strlen(pTagData)); - indexMultiTermAdd(tmGroup, tm); - int ret = indexPut((SIndex *)pMeta->pTagIvtIdx, tmGroup, tuid); - indexMultiTermDestroy(tmGroup); -#else if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { return -1; } tdbTbInsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); metaDestroyTagIdxKey(pTagIdxKey); -#endif tDecoderClear(&dc); tdbFree(pData); return 0; @@ -849,7 +903,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { SEncoder coder = {0}; - void *pVal = NULL; + void * pVal = NULL; int vLen = 0; int rcode = 0; SSkmDbKey skmDbKey = {0}; @@ -926,10 +980,5 @@ _err: return -1; } // refactor later -void *metaGetIdx(SMeta *pMeta) { -#ifdef USE_INVERTED_INDEX - return pMeta->pTagIvtIdx; -#else - return pMeta->pTagIdx; -#endif -} +void *metaGetIdx(SMeta *pMeta) { return pMeta->pTagIdx; } +void *metaGetIvtIdx(SMeta *pMeta) { return pMeta->pTagIvtIdx; } diff --git a/source/dnode/vnode/src/sma/sma.c b/source/dnode/vnode/src/sma/sma.c index 7a2b6a2757ad4153313586310b01148c4efb49b1..04f65275d7d20ab41564a8e8c6e67c908b3bc649 100644 --- a/source/dnode/vnode/src/sma/sma.c +++ b/source/dnode/vnode/src/sma/sma.c @@ -20,7 +20,7 @@ int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) { - smaWarn("vgId:%d insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + smaWarn("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); } // TODO: destroy SSDataBlocks(msg) return code; @@ -30,7 +30,7 @@ int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tdProcessTSmaCreateImpl(pSma, version, msg)) < 0) { - smaWarn("vgId:%d create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno)); + smaWarn("vgId:%d, create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno)); } // TODO: destroy SSDataBlocks(msg) return code; @@ -39,7 +39,7 @@ int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg) { int32_t tdUpdateExpireWindow(SSma* pSma, const SSubmitReq* pMsg, int64_t version) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tdUpdateExpiredWindowImpl(pSma, pMsg, version)) < 0) { - smaWarn("vgId:%d update expired sma window failed since %s", SMA_VID(pSma), tstrerror(terrno)); + smaWarn("vgId:%d, update expired sma window failed since %s", SMA_VID(pSma), tstrerror(terrno)); } return code; } @@ -47,7 +47,15 @@ int32_t tdUpdateExpireWindow(SSma* pSma, const SSubmitReq* pMsg, int64_t version int32_t tdGetTSmaData(SSma* pSma, char* pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tdGetTSmaDataImpl(pSma, pData, indexUid, querySKey, nMaxResult)) < 0) { - smaWarn("vgId:%d get tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + smaWarn("vgId:%d, get tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + return code; +} + +int32_t smaGetTSmaDays(SVnodeCfg* pCfg, void* pCont, uint32_t contLen, int32_t *days) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tdGetTSmaDaysImpl(pCfg, pCont, contLen, days)) < 0) { + smaWarn("vgId:%d get tSma days failed since %s", pCfg->vgId, tstrerror(terrno)); } return code; } diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 8285b74e509f53a8ed3a9d2e5745d2f56135087e..179f573e8d72c3ae6938edb31e61ef6b9ec8a675 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -222,7 +222,7 @@ int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat) { if (!pStat) return 0; int ref = T_REF_INC(pStat); - smaDebug("vgId:%d ref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref); + smaDebug("vgId:%d, ref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref); return 0; } @@ -230,7 +230,7 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { if (!pStat) return 0; int ref = T_REF_DEC(pStat); - smaDebug("vgId:%d unref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref); + smaDebug("vgId:%d, unref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref); return 0; } @@ -278,7 +278,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { if (pSmaStatItem) { - tdDestroyTSma(pSmaStatItem->pTSma); + tDestroyTSma(pSmaStatItem->pTSma); taosMemoryFreeClear(pSmaStatItem->pTSma); taosHashCleanup(pSmaStatItem->expiredWindows); taosMemoryFreeClear(pSmaStatItem); @@ -321,7 +321,7 @@ int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { int32_t tdLockSma(SSma *pSma) { int code = taosThreadMutexLock(&pSma->mutex); if (code != 0) { - smaError("vgId:%d failed to lock td since %s", SMA_VID(pSma), strerror(errno)); + smaError("vgId:%d, failed to lock td since %s", SMA_VID(pSma), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); return -1; } @@ -334,7 +334,7 @@ int32_t tdUnLockSma(SSma *pSma) { pSma->locked = false; int code = taosThreadMutexUnlock(&pSma->mutex); if (code != 0) { - smaError("vgId:%d failed to unlock td since %s", SMA_VID(pSma), strerror(errno)); + smaError("vgId:%d, failed to unlock td since %s", SMA_VID(pSma), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); return -1; } @@ -376,7 +376,7 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { if (did.level < 0 || did.id < 0) { tdUnLockSma(pSma); - smaError("vgId:%d init sma env failed since invalid did(%d,%d)", SMA_VID(pSma), did.level, did.id); + smaError("vgId:%d, init sma env failed since invalid did(%d,%d)", SMA_VID(pSma), did.level, did.id); return TSDB_CODE_FAILED; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 42c215ee7c276ac6ed7d299390bef7e2083741ce..e738d3a4084a583852dcdf11c73d8cdafb006632 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -58,30 +58,30 @@ static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SA if (!suid || !tbUids) { terrno = TSDB_CODE_INVALID_PTR; - smaError("vgId:%d failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); return TSDB_CODE_FAILED; } pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { - smaError("vgId:%d failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); + smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; return TSDB_CODE_FAILED; } if (pRSmaInfo->taskInfo[0] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[0], tbUids, true) != 0)) { - smaError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); return TSDB_CODE_FAILED; } else { - smaDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), + smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), pRSmaInfo->taskInfo[0], *suid, *(int64_t *)taosArrayGet(tbUids, 0)); } if (pRSmaInfo->taskInfo[1] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[1], tbUids, true) != 0)) { - smaError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); return TSDB_CODE_FAILED; } else { - smaDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), + smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), pRSmaInfo->taskInfo[1], *suid, *(int64_t *)taosArrayGet(tbUids, 0)); } @@ -170,14 +170,14 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { SMeta *pMeta = pVnode->pMeta; SMsgCb *pMsgCb = &pVnode->msgCb; if (!pReq->rollup) { - smaTrace("vgId:%d return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); return TSDB_CODE_SUCCESS; } SRSmaParam *param = &pReq->pRSmaParam; if ((param->qmsg1Len == 0) && (param->qmsg2Len == 0)) { - smaWarn("vgId:%d no qmsg1/qmsg2 for rollup stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + smaWarn("vgId:%d, no qmsg1/qmsg2 for rollup stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); return TSDB_CODE_SUCCESS; } @@ -192,7 +192,7 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t)); if (pRSmaInfo) { - smaWarn("vgId:%d rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); + smaWarn("vgId:%d, rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); return TSDB_CODE_SUCCESS; } @@ -238,7 +238,7 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; } else { - smaDebug("vgId:%d register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid); + smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid); } return TSDB_CODE_SUCCESS; @@ -372,11 +372,11 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3 SArray *pResult = NULL; if (!taskInfo) { - smaDebug("vgId:%d no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid); + smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid); return TSDB_CODE_SUCCESS; } - smaDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, taskInfo, suid); + smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, taskInfo, suid); qSetStreamInput(taskInfo, pMsg, inputType, true); while (1) { @@ -400,7 +400,11 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3 } if (taosArrayGetSize(pResult) > 0) { - blockDebugShowData(pResult); +#if 1 + char flag[10] = {0}; + snprintf(flag, 10, "level %" PRIi8, level); + blockDebugShowData(pResult, flag); +#endif STsdb *sinkTsdb = (level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2); SSubmitReq *pReq = NULL; if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), suid) != 0) { @@ -414,7 +418,7 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3 } taosMemoryFreeClear(pReq); } else { - smaDebug("vgId:%d no rsma % " PRIi8 " data generated since %s", SMA_VID(pSma), level, tstrerror(terrno)); + smaDebug("vgId:%d, no rsma % " PRIi8 " data generated since %s", SMA_VID(pSma), level, tstrerror(terrno)); } taosArrayDestroy(pResult); @@ -435,16 +439,16 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { - smaDebug("vgId:%d no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); + smaDebug("vgId:%d, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); return TSDB_CODE_SUCCESS; } if (!pRSmaInfo->taskInfo[0]) { - smaDebug("vgId:%d no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid); + smaDebug("vgId:%d, no rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), suid); return TSDB_CODE_SUCCESS; } if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - // TODO: use the proper schema instead of 0, and cache STSchema in cache + // TODO: cache STSchema STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, -1); if (!pTSchema) { terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index f88afcaddf82eed173f00693ea9d905ea195bd91..b72be06455d8181dca8a27ea1c58cfa72ddef39f 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -326,13 +326,13 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { if (!pDataBlocks) { terrno = TSDB_CODE_INVALID_PTR; - smaWarn("vgId:%d insert tSma data failed since pDataBlocks is NULL", SMA_VID(pSma)); + smaWarn("vgId:%d, insert tSma data failed since pDataBlocks is NULL", SMA_VID(pSma)); return terrno; } if (taosArrayGetSize(pDataBlocks) <= 0) { terrno = TSDB_CODE_INVALID_PARA; - smaWarn("vgId:%d insert tSma data failed since pDataBlocks is empty", SMA_VID(pSma)); + smaWarn("vgId:%d, insert tSma data failed since pDataBlocks is empty", SMA_VID(pSma)); return TSDB_CODE_FAILED; } @@ -487,11 +487,11 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { smaCloseDBF(&tSmaH.dFile); } tdSetTSmaDataFile(&tSmaH, indexUid, fid); - smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 + smaDebug("vgId:%d, write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 " queryKey:%" PRIi64, SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey); if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { - smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma), + smaWarn("vgId:%d, open DB file %s failed since %s", SMA_VID(pSma), tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); tdDestroyTSmaWriteH(&tSmaH); tdUnRefSmaStat(pSma, pStat); @@ -501,7 +501,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { } if (tdInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) { - smaWarn("vgId:%d insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 + smaWarn("vgId:%d, insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 " since %s", SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno)); tdSmaEndCommit(pEnv); @@ -510,14 +510,14 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { return TSDB_CODE_FAILED; } - smaDebug("vgId:%d insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64, + smaDebug("vgId:%d, insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64, SMA_VID(pSma), indexUid, skey, groupId); // TODO:tsdbEndTSmaCommit(); // Step 3: reset the SSmaStat tdResetExpiredWindow(pSma, pStat, indexUid, skey); } else { - smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, + smaWarn("vgId:%d, invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, SMA_VID(pSma), skey, tlen, indexUid); } } @@ -532,7 +532,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tdDropTSmaDataImpl(pSma, indexUid)) < 0) { - smaWarn("vgId:%d drop tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + smaWarn("vgId:%d, drop tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); } return code; } @@ -553,11 +553,11 @@ static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyL // TODO: insert tsma data blocks into B+Tree(TTB) if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { - smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", + smaWarn("vgId:%d, insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); return TSDB_CODE_FAILED; } - smaDebug("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed", + smaDebug("vgId:%d, insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed", SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); #ifdef _TEST_SMA_PRINT_DEBUG_LOG_ @@ -565,7 +565,7 @@ static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyL void *data = tdGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); ASSERT(data != NULL); for (uint32_t v = 0; v < valueSize; v += 8) { - smaWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); + smaWarn("vgId:%d, insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); } #endif return TSDB_CODE_SUCCESS; @@ -594,11 +594,11 @@ static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUi if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { // error handling tdUnRefSmaStat(pSma, pStat); - smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey, + smaWarn("vgId:%d, remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey, indexUid); return TSDB_CODE_FAILED; } - smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), + smaDebug("vgId:%d, remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), skey, indexUid); // TODO: use a standalone interface to received state upate notification from stream computing module. /** @@ -612,7 +612,7 @@ static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUi } else { // error handling tdUnRefSmaStat(pSma, pStat); - smaWarn("vgId:%d expired window %" PRIi64 " not exists for sma index %" PRIi64, SMA_VID(pSma), skey, indexUid); + smaWarn("vgId:%d, expired window %" PRIi64 " not exists for sma index %" PRIi64, SMA_VID(pSma), skey, indexUid); return TSDB_CODE_FAILED; } @@ -632,19 +632,19 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) { // clear local cache if (pEnv) { - smaDebug("vgId:%d drop tSma local cache for %" PRIi64, SMA_VID(pSma), indexUid); + smaDebug("vgId:%d, drop tSma local cache for %" PRIi64, SMA_VID(pSma), indexUid); SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) { if (tdSmaStatIsDropped(pItem)) { - smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); + smaDebug("vgId:%d, tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode } tdWLockSmaEnv(pEnv); if (tdSmaStatIsDropped(pItem)) { tdUnLockSmaEnv(pEnv); - smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); + smaDebug("vgId:%d, tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode } tdSmaStatSetDropped(pItem); @@ -654,19 +654,19 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) { int32_t refVal = INT32_MAX; while (true) { if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) { - smaDebug("vgId:%d drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); + smaDebug("vgId:%d, drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); break; } - smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); + smaDebug("vgId:%d, wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); taosSsleep(1); if (++nSleep > SMA_DROP_EXPIRED_TIME) { - smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal); + smaDebug("vgId:%d, drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal); break; }; } tdFreeSmaStatItem(pItem); - smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", SMA_VID(pSma), indexUid); + smaDebug("vgId:%d, getTSmaDataImpl failed since no index %" PRIi64 " in local cache", SMA_VID(pSma), indexUid); } } // clear sma data files @@ -690,7 +690,7 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query if (!pEnv) { terrno = TSDB_CODE_INVALID_PTR; - smaWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", SMA_VID(pSma)); + smaWarn("vgId:%d, getTSmaDataImpl failed since pTSmaEnv is NULL", SMA_VID(pSma)); return TSDB_CODE_FAILED; } @@ -703,7 +703,7 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query // it's NULL. tdUnRefSmaStat(pSma, pStat); terrno = TSDB_CODE_TDB_INVALID_ACTION; - smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, SMA_VID(pSma), indexUid); + smaDebug("vgId:%d, getTSmaDataImpl failed since no index %" PRIi64, SMA_VID(pSma), indexUid); return TSDB_CODE_FAILED; } @@ -722,17 +722,17 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query if (!tdSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query tdUnRefSmaStat(pSma, pStat); terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; - smaWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid, + smaWarn("vgId:%d, getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid, tstrerror(terrno), smaStat); return TSDB_CODE_FAILED; } if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { // TODO: mark this window as expired. - smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, + smaDebug("vgId:%d, skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, indexUid); } else { - smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, + smaDebug("vgId:%d, skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, indexUid); } @@ -750,7 +750,7 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query smaDebug("### vgId:%d read from DBF %s days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi8 " queryKey:%" PRIi64, SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey); if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) { - smaWarn("vgId:%d open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno)); + smaWarn("vgId:%d, open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno)); return TSDB_CODE_FAILED; } @@ -759,13 +759,13 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query int64_t queryGroupId = 0; tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); - smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path, + smaDebug("vgId:%d, get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); void *result = NULL; int32_t valueSize = 0; if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { - smaWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", + smaWarn("vgId:%d, get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); smaCloseDBF(&tReadH.dFile); return TSDB_CODE_FAILED; @@ -774,7 +774,7 @@ int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY query #ifdef _TEST_SMA_PRINT_DEBUG_LOG_ for (uint32_t v = 0; v < valueSize; v += 8) { - smaWarn("vgId:%d get sma data v[%d]=%" PRIi64, SMA_VID(pSma), v, *(int64_t *)POINTER_SHIFT(result, v)); + smaWarn("vgId:%d, get sma data v[%d]=%" PRIi64, SMA_VID(pSma), v, *(int64_t *)POINTER_SHIFT(result, v)); } #endif taosMemoryFreeClear(result); // TODO: fill the result to output @@ -828,7 +828,7 @@ int32_t tdDropTSma(SSma *pSma, char *pMsg) { // TODO: send msg to stream computing to drop tSma // if ((send msg to stream computing) < 0) { - // tdDestroyTSma(&vCreateSmaReq); + // tDestroyTSma(&vCreateSmaReq); // return -1; // } // @@ -888,7 +888,7 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; taosHashCleanup(pItem->expiredWindows); taosMemoryFree(pItem); - smaWarn("vgId:%d set expire window, get tsma meta failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), + smaWarn("vgId:%d, set expire window, get tsma meta failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); return TSDB_CODE_FAILED; } @@ -915,12 +915,12 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde taosHashCleanup(pItem->expiredWindows); taosMemoryFreeClear(pItem->pTSma); taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); - smaWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid, + smaWarn("vgId:%d, smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid, winSKey); return TSDB_CODE_FAILED; } - smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid, + smaDebug("vgId:%d, smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid, winSKey); return TSDB_CODE_SUCCESS; } @@ -935,18 +935,18 @@ static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t inde int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version) { // no time-range-sma, just return success if (atomic_load_16(&SMA_TSMA_NUM(pSma)) <= 0) { - smaTrace("vgId:%d not update expire window since no tSma", SMA_VID(pSma)); + smaTrace("vgId:%d, not update expire window since no tSma", SMA_VID(pSma)); return TSDB_CODE_SUCCESS; } if (!SMA_META(pSma)) { terrno = TSDB_CODE_INVALID_PTR; - smaError("vgId:%d update expire window failed since no meta ptr", SMA_VID(pSma)); + smaError("vgId:%d, update expire window failed since no meta ptr", SMA_VID(pSma)); return TSDB_CODE_FAILED; } if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) < 0) { - smaError("vgId:%d init sma env failed since %s", SMA_VID(pSma), terrstr(terrno)); + smaError("vgId:%d, init sma env failed since %s", SMA_VID(pSma), terrstr(terrno)); terrno = TSDB_CODE_TDB_INIT_FAILED; return TSDB_CODE_FAILED; } @@ -982,25 +982,25 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t ve SSubmitBlkIter blkIter = {0}; if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) { - pSW = tdFreeTSmaWrapper(pSW, false); + pSW = tFreeTSmaWrapper(pSW, false); break; } while (true) { STSRow *row = tGetSubmitBlkNext(&blkIter); if (!row) { - pSW = tdFreeTSmaWrapper(pSW, false); + pSW = tFreeTSmaWrapper(pSW, false); break; } if (!pSW || (pTSma && (pTSma->tableUid != msgIter.suid))) { if (pSW) { - pSW = tdFreeTSmaWrapper(pSW, false); + pSW = tFreeTSmaWrapper(pSW, false); } if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), msgIter.suid, false))) { break; } if ((pSW->number) <= 0 || !pSW->tSma) { - pSW = tdFreeTSmaWrapper(pSW, false); + pSW = tFreeTSmaWrapper(pSW, false); break; } @@ -1020,12 +1020,12 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t ve if (lastWinSKey != winSKey) { lastWinSKey = winSKey; if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) { - pSW = tdFreeTSmaWrapper(pSW, false); + pSW = tFreeTSmaWrapper(pSW, false); tdUnRefSmaStat(pSma, pStat); return TSDB_CODE_FAILED; } } else { - smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", + smaDebug("vgId:%d, smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", SMA_VID(pSma), pTSma->indexUid, winSKey); } } diff --git a/source/dnode/vnode/src/sma/smaTimeRange2.c b/source/dnode/vnode/src/sma/smaTimeRange2.c new file mode 100644 index 0000000000000000000000000000000000000000..5ef171c7991c47494efa265274852c48e0bac6b7 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaTimeRange2.c @@ -0,0 +1,1084 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" +#include "tsdb.h" + +typedef STsdbCfg STSmaKeepCfg; + +#undef _TEST_SMA_PRINT_DEBUG_LOG_ +#define SMA_STORAGE_MINUTES_MAX 86400 +#define SMA_STORAGE_MINUTES_DAY 1440 +#define SMA_STORAGE_MINUTES_MIN 1440 +#define SMA_STORAGE_TSDB_MINUTES 86400 +#define SMA_STORAGE_TSDB_TIMES 10 +#define SMA_STORAGE_SPLIT_FACTOR 14400 // least records in tsma file TODO: the feasible value? +#define SMA_KEY_LEN 16 // TSKEY+groupId 8+8 +#define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds + +#define SMA_STATE_ITEM_HASH_SLOT 32 + +typedef struct { + SSma *pSma; + SDBFile dFile; + const SArray *pDataBlocks; // sma data + int64_t interval; // interval with the precision of DB +} STSmaWriteH; + +typedef struct { + int32_t iter; + int32_t fid; +} SmaFsIter; + +typedef struct { + STsdb *pTsdb; + SSma *pSma; + SDBFile dFile; + int64_t interval; // interval with the precision of DB + int32_t blockSize; // size of SMA block item + int32_t days; + int8_t storageLevel; + SmaFsIter smaFsIter; +} STSmaReadH; + +typedef enum { + SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma + SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma +} ESmaStorageLevel; + +// static func + +static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); +static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval); +static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, + int8_t intervalUnit); +static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit); +static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH); +static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel); +static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid); +static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); +static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); +static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, + TXN *txn); +// expired window + +static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version); +static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey); +static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); + +/** + * @brief Judge the tsma file split days + * + * @param pCfg + * @param pCont + * @param contLen + * @param days unit is minute + * @return int32_t + */ +int32_t tdGetTSmaDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { + SDecoder coder = {0}; + tDecoderInit(&coder, pCont, contLen); + + STSma tsma = {0}; + if (tDecodeSVCreateTSmaReq(&coder, &tsma) < 0) { + terrno = TSDB_CODE_MSG_DECODE_ERROR; + goto _err; + } + STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg; + int64_t mInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE); + int64_t records = pTsdbCfg->days / mInterval; + + if (records >= SMA_STORAGE_SPLIT_FACTOR) { + *days = pTsdbCfg->days; + } else { + int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2; + + if (daysPerFile > SMA_STORAGE_MINUTES_MAX) { + *days = SMA_STORAGE_MINUTES_MAX; + } else { + *days = (int32_t)daysPerFile; + } + + if(*days < pTsdbCfg->days) { + *days = pTsdbCfg->days; + } + } + tDecoderClear(&coder); + return 0; +_err: + tDecoderClear(&coder); + return -1; +} + +// read data + +// implementation + +/** + * @brief + * + * @param pSmaH + * @param pSma + * @param interval + * @param intervalUnit + * @return int32_t + */ +static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit) { + STSmaKeepCfg *pCfg = SMA_TSDB_CFG(pSma); + pSmaH->pSma = pSma; + pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); + pSmaH->storageLevel = tdGetSmaStorageLevel(pCfg, interval); + pSmaH->days = tdGetTSmaDays(pSma, pSmaH->interval, pSmaH->storageLevel); + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Init of tSma FS + * + * @param pReadH + * @param indexUid + * @param skey + * @return int32_t + */ +static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) { + SSma *pSma = pSmaH->pSma; + + int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, SMA_TSDB_CFG(pSma)->precision)); + char tSmaFile[TSDB_FILENAME_LEN] = {0}; + snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); + pSmaH->dFile.path = strdup(tSmaFile); + pSmaH->smaFsIter.iter = 0; + pSmaH->smaFsIter.fid = fid; + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Set and open tSma file if it has key locates in queryWin. + * + * @param pReadH + * @param param + * @param queryWin + * @return true + * @return false + */ +static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { + // SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf; + // int32_t nSmaFs = taosArrayGetSize(smaFs); + + smaCloseDBF(&pReadH->dFile); + +#if 0 + while (pReadH->smaFsIter.iter < nSmaFs) { + void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter); + if (pSmaFile) { // match(indexName, queryWindow) + // TODO: select the file by index_name ... + pReadH->dFile = pSmaFile; + ++pReadH->smaFsIter.iter; + break; + } + ++pReadH->smaFsIter.iter; + } + + if (pReadH->pDFile) { + tdDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]"); + return true; + } +#endif + + return false; +} + +/** + * @brief Approximate value for week/month/year. + * + * @param interval + * @param intervalUnit + * @param precision + * @param adjusted Interval already adjusted according to DB precision + * @return int64_t + */ +static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) { + if (adjusted) { + return interval; + } + + switch (intervalUnit) { + case TIME_UNIT_YEAR: // approximate value + interval *= 365 * 86400 * 1e3; + break; + case TIME_UNIT_MONTH: // approximate value + interval *= 30 * 86400 * 1e3; + break; + case TIME_UNIT_WEEK: // approximate value + interval *= 7 * 86400 * 1e3; + break; + case TIME_UNIT_DAY: // the interval for tSma calculation must <= day + interval *= 86400 * 1e3; + break; + case TIME_UNIT_HOUR: + interval *= 3600 * 1e3; + break; + case TIME_UNIT_MINUTE: + interval *= 60 * 1e3; + break; + case TIME_UNIT_SECOND: + interval *= 1e3; + break; + default: + break; + } + + switch (precision) { + case TSDB_TIME_PRECISION_MILLI: + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval / 1e3; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second + return interval / 1e6; + } else { // ms + return interval; + } + break; + case TSDB_TIME_PRECISION_MICRO: + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns + return interval / 1e3; + } else { // ms + return interval * 1e3; + } + break; + case TSDB_TIME_PRECISION_NANO: + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval * 1e3; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns + return interval; + } else { // ms + return interval * 1e6; + } + break; + default: // ms + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval / 1e3; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns + return interval / 1e6; + } else { // ms + return interval; + } + break; + } + return interval; +} + +static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, + int8_t intervalUnit) { + pSmaH->pSma = pSma; + pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); + pSmaH->pDataBlocks = pDataBlocks; + pSmaH->dFile.fid = SMA_IVLD_FID; + return TSDB_CODE_SUCCESS; +} + +static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH) { + if (pSmaH) { + smaCloseDBF(&pSmaH->dFile); + } +} + +static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) { + SSma *pSma = pSmaH->pSma; + ASSERT(!pSmaH->dFile.path && !pSmaH->dFile.pDB); + + pSmaH->dFile.fid = fid; + char tSmaFile[TSDB_FILENAME_LEN] = {0}; + snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); + pSmaH->dFile.path = strdup(tSmaFile); + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief + * + * @param pSma + * @param interval Interval calculated by DB's precision + * @param storageLevel + * @return int32_t + */ +static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel) { + STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); + int32_t daysPerFile = pCfg->days; // unit is minute + + if (storageLevel == SMA_STORAGE_LEVEL_TSDB) { + int32_t minutes = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]); + if (minutes > SMA_STORAGE_TSDB_MINUTES) { + daysPerFile = SMA_STORAGE_TSDB_MINUTES; + } + } + + return daysPerFile; +} + +/** + * @brief Judge the tSma storage level + * + * @param pCfg + * @param interval + * @return int32_t + */ +static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval) { + int64_t mInterval = convertTimeFromPrecisionToUnit(interval, pCfg->precision, TIME_UNIT_MINUTE); + if (pCfg->days / mInterval >= SMA_STORAGE_SPLIT_FACTOR) { + return SMA_STORAGE_LEVEL_DFILESET; + } + return SMA_STORAGE_LEVEL_TSDB; +} + +/** + * @brief Insert/Update Time-range-wise SMA data. + * - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g. + * v3f1900.tsma.${sma_index_name}. The days is the same with that for TS data files. + * - If interval >= SMA_STORAGE_SPLIT_HOURS, save the SMA data to e.g. vnode3/tsma/v3f632.tsma.${sma_index_name}. The + * days is 30 times of the interval, and the minimum days is SMA_STORAGE_TSDB_DAYS(30d). + * - The destination file of one data block for some interval is determined by its start TS key. + * + * @param pSma + * @param msg + * @return int32_t + */ +int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { + STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); + const SArray *pDataBlocks = (const SArray *)msg; + int64_t testSkey = TSKEY_INITIAL_VAL; + + // TODO: destroy SSDataBlocks(msg) + + // For super table aggregation, the sma data is stored in vgroup calculated from the hash value of stable name. Thus + // the sma data would arrive ahead of the update-expired-window msg. + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + if (!pDataBlocks) { + terrno = TSDB_CODE_INVALID_PTR; + smaWarn("vgId:%d insert tSma data failed since pDataBlocks is NULL", SMA_VID(pSma)); + return terrno; + } + + if (taosArrayGetSize(pDataBlocks) <= 0) { + terrno = TSDB_CODE_INVALID_PARA; + smaWarn("vgId:%d insert tSma data failed since pDataBlocks is empty", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SSmaStatItem *pItem = NULL; + + tdRefSmaStat(pSma, pStat); + + if (pStat && SMA_STAT_ITEMS(pStat)) { + pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); + } + + if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tdSmaStatIsDropped(pItem)) { + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + + STSma *pTSma = pItem->pTSma; + STSmaWriteH tSmaH = {0}; + + if (tdInitTSmaWriteH(&tSmaH, pSma, pDataBlocks, pTSma->interval, pTSma->intervalUnit) != 0) { + return TSDB_CODE_FAILED; + } + + char rPath[TSDB_FILENAME_LEN] = {0}; + char aPath[TSDB_FILENAME_LEN] = {0}; + snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); + tfsAbsoluteName(SMA_TFS(pSma), SMA_ENV_DID(pEnv), rPath, aPath); + if (!taosCheckExistFile(aPath)) { + if (tfsMkdirRecurAt(SMA_TFS(pSma), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + } + + // Step 1: Judge the storage level and days + int32_t storageLevel = tdGetSmaStorageLevel(pCfg, tSmaH.interval); + int32_t minutePerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel); + + char smaKey[SMA_KEY_LEN] = {0}; // key: skey + groupId + char dataBuf[512] = {0}; // val: aggr data // TODO: handle 512 buffer? + void *pDataBuf = NULL; + int32_t sz = taosArrayGetSize(pDataBlocks); + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock *pDataBlock = taosArrayGet(pDataBlocks, i); + int32_t colNum = pDataBlock->info.numOfCols; + int32_t rows = pDataBlock->info.rows; + int32_t rowSize = pDataBlock->info.rowSize; + int64_t groupId = pDataBlock->info.groupId; + for (int32_t j = 0; j < rows; ++j) { + printf("|"); + TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval + void *pSmaKey = &smaKey; + bool isStartKey = false; + + int32_t tlen = 0; // reset the len + pDataBuf = &dataBuf; // reset the buf + for (int32_t k = 0; k < colNum; ++k) { + SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + if (!isStartKey) { + isStartKey = true; + skey = *(TSKEY *)var; + testSkey = skey; + printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId); + tdEncodeTSmaKey(groupId, skey, &pSmaKey); + } else { + printf(" %" PRIi64 " |", *(int64_t *)var); + tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); + break; + } + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + printf(" %15d |", *(uint8_t *)var); + tlen += taosEncodeFixedU8(&pDataBuf, *(uint8_t *)var); + break; + case TSDB_DATA_TYPE_TINYINT: + printf(" %15d |", *(int8_t *)var); + tlen += taosEncodeFixedI8(&pDataBuf, *(int8_t *)var); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf(" %15d |", *(int16_t *)var); + tlen += taosEncodeFixedI16(&pDataBuf, *(int16_t *)var); + break; + case TSDB_DATA_TYPE_USMALLINT: + printf(" %15d |", *(uint16_t *)var); + tlen += taosEncodeFixedU16(&pDataBuf, *(uint16_t *)var); + break; + case TSDB_DATA_TYPE_INT: + printf(" %15d |", *(int32_t *)var); + tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var); + break; + case TSDB_DATA_TYPE_FLOAT: + printf(" %15f |", *(float *)var); + tlen += taosEncodeBinary(&pDataBuf, var, sizeof(float)); + break; + case TSDB_DATA_TYPE_UINT: + printf(" %15u |", *(uint32_t *)var); + tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var); + break; + case TSDB_DATA_TYPE_BIGINT: + printf(" %15ld |", *(int64_t *)var); + tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf(" %15lf |", *(double *)var); + tlen += taosEncodeBinary(&pDataBuf, var, sizeof(double)); + case TSDB_DATA_TYPE_UBIGINT: + printf(" %15lu |", *(uint64_t *)var); + tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var); + break; + case TSDB_DATA_TYPE_NCHAR: { + char tmpChar[100] = {0}; + strncpy(tmpChar, varDataVal(var), varDataLen(var)); + printf(" %s |", tmpChar); + tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); + break; + } + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + char tmpChar[100] = {0}; + strncpy(tmpChar, varDataVal(var), varDataLen(var)); + printf(" %s |", tmpChar); + tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); + break; + } + case TSDB_DATA_TYPE_VARBINARY: + // TODO: add binary/varbinary + TASSERT(0); + default: + printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + TASSERT(0); + break; + } + } + printf("\n"); + // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) { + if (tlen > 0) { + int32_t fid = (int32_t)(TSDB_KEY_FID(skey, minutePerFile, pCfg->precision)); + + // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index + // file + // - Set and open the DFile or the B+Tree file + // TODO: tsdbStartTSmaCommit(); + if (fid != tSmaH.dFile.fid) { + if (tSmaH.dFile.fid != SMA_IVLD_FID) { + tdSmaEndCommit(pEnv); + smaCloseDBF(&tSmaH.dFile); + } + tdSetTSmaDataFile(&tSmaH, indexUid, fid); + smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 + " queryKey:%" PRIi64, + SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey); + if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { + smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma), + tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); + tdDestroyTSmaWriteH(&tSmaH); + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + tdSmaBeginCommit(pEnv); + } + + if (tdInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) { + smaWarn("vgId:%d insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 + " since %s", + SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno)); + tdSmaEndCommit(pEnv); + tdDestroyTSmaWriteH(&tSmaH); + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + + smaDebug("vgId:%d insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64, + SMA_VID(pSma), indexUid, skey, groupId); + // TODO:tsdbEndTSmaCommit(); + + // Step 3: reset the SSmaStat + tdResetExpiredWindow(pSma, pStat, indexUid, skey); + } else { + smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, + SMA_VID(pSma), skey, tlen, indexUid); + } + } + } + tdSmaEndCommit(pEnv); // TODO: not commit for every insert + tdDestroyTSmaWriteH(&tSmaH); + tdUnRefSmaStat(pSma, pStat); + + return TSDB_CODE_SUCCESS; +} + +int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tdDropTSmaDataImpl(pSma, indexUid)) < 0) { + smaWarn("vgId:%d drop tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + return code; +} + +/** + * @brief Insert TSma data blocks to DB File build by B+Tree + * + * @param pSmaH + * @param smaKey tableUid-colId-skeyOfWindow(8-2-8) + * @param keyLen + * @param pData + * @param dataLen + * @return int32_t + */ +static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, + TXN *txn) { + SDBFile *pDBFile = &pSmaH->dFile; + + // TODO: insert tsma data blocks into B+Tree(TTB) + if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { + smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", + SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); + return TSDB_CODE_FAILED; + } + smaDebug("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed", + SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); + +#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ + uint32_t valueSize = 0; + void *data = tdGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); + ASSERT(data != NULL); + for (uint32_t v = 0; v < valueSize; v += 8) { + smaWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); + } +#endif + return TSDB_CODE_SUCCESS; +} + +/** + * @brief When sma data received from stream computing, make the relative expired window valid. + * + * @param pSma + * @param pStat + * @param indexUid + * @param skey + * @return int32_t + */ +static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey) { + SSmaStatItem *pItem = NULL; + + tdRefSmaStat(pSma, pStat); + + if (pStat && SMA_STAT_ITEMS(pStat)) { + pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); + } + if ((pItem) && ((pItem = *(SSmaStatItem **)pItem))) { + // pItem resides in hash buffer all the time unless drop sma index + // TODO: multithread protect + if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { + // error handling + tdUnRefSmaStat(pSma, pStat); + smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey, + indexUid); + return TSDB_CODE_FAILED; + } + smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), + skey, indexUid); + // TODO: use a standalone interface to received state upate notification from stream computing module. + /** + * @brief state + * - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK. + * - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g. + * when batch data caculation not finised) + * - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB. + */ + pItem->state = TSDB_SMA_STAT_OK; + } else { + // error handling + tdUnRefSmaStat(pSma, pStat); + smaWarn("vgId:%d expired window %" PRIi64 " not exists for sma index %" PRIi64, SMA_VID(pSma), skey, indexUid); + return TSDB_CODE_FAILED; + } + + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Drop tSma data and local cache + * - insert/query reference + * @param pSma + * @param msg + * @return int32_t + */ +static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) { + SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); + + // clear local cache + if (pEnv) { + smaDebug("vgId:%d drop tSma local cache for %" PRIi64, SMA_VID(pSma), indexUid); + + SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); + if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) { + if (tdSmaStatIsDropped(pItem)) { + smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); + return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode + } + + tdWLockSmaEnv(pEnv); + if (tdSmaStatIsDropped(pItem)) { + tdUnLockSmaEnv(pEnv); + smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); + return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode + } + tdSmaStatSetDropped(pItem); + tdUnLockSmaEnv(pEnv); + + int32_t nSleep = 0; + int32_t refVal = INT32_MAX; + while (true) { + if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) { + smaDebug("vgId:%d drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); + break; + } + smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); + taosSsleep(1); + if (++nSleep > SMA_DROP_EXPIRED_TIME) { + smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal); + break; + }; + } + + tdFreeSmaStatItem(pItem); + smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", SMA_VID(pSma), indexUid); + } + } + // clear sma data files + // TODO: + return TSDB_CODE_SUCCESS; +} + +/** + * @brief + * + * @param pSma Return the data between queryWin and fill the pData. + * @param pData + * @param indexUid + * @param pQuerySKey + * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM. + * @return int32_t + */ +int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { + SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); + SSmaStat *pStat = NULL; + + if (!pEnv) { + terrno = TSDB_CODE_INVALID_PTR; + smaWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + pStat = SMA_ENV_STAT(pEnv); + + tdRefSmaStat(pSma, pStat); + SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); + if (!pItem || !(pItem = *(SSmaStatItem **)pItem)) { + // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if + // it's NULL. + tdUnRefSmaStat(pSma, pStat); + terrno = TSDB_CODE_TDB_INVALID_ACTION; + smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, SMA_VID(pSma), indexUid); + return TSDB_CODE_FAILED; + } + +#if 0 + int32_t nQueryWin = taosArrayGetSize(pQuerySKey); + for (int32_t n = 0; n < nQueryWin; ++n) { + TSKEY skey = taosArrayGet(pQuerySKey, n); + if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY))) { + // TODO: mark this window as expired. + } + } +#endif + +#if 1 + int8_t smaStat = 0; + if (!tdSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query + tdUnRefSmaStat(pSma, pStat); + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + smaWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid, + tstrerror(terrno), smaStat); + return TSDB_CODE_FAILED; + } + + if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { + // TODO: mark this window as expired. + smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, + indexUid); + } else { + smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, + indexUid); + } + + STSma *pTSma = pItem->pTSma; +#endif + +#if 1 + STSmaReadH tReadH = {0}; + tdInitTSmaReadH(&tReadH, pSma, pTSma->interval, pTSma->intervalUnit); + smaCloseDBF(&tReadH.dFile); + + tdUnRefSmaStat(pSma, pStat); + + tdInitTSmaFile(&tReadH, indexUid, querySKey); + smaDebug("### vgId:%d read from DBF %s days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi8 " queryKey:%" PRIi64, + SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey); + if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) { + smaWarn("vgId:%d open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + + char smaKey[SMA_KEY_LEN] = {0}; + void *pSmaKey = &smaKey; + int64_t queryGroupId = 0; + tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); + + smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path, + *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); + + void *result = NULL; + int32_t valueSize = 0; + if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { + smaWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", + SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); + smaCloseDBF(&tReadH.dFile); + return TSDB_CODE_FAILED; + } +#endif + +#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ + for (uint32_t v = 0; v < valueSize; v += 8) { + smaWarn("vgId:%d get sma data v[%d]=%" PRIi64, SMA_VID(pSma), v, *(int64_t *)POINTER_SHIFT(result, v)); + } +#endif + taosMemoryFreeClear(result); // TODO: fill the result to output + +#if 0 + int32_t nResult = 0; + int64_t lastKey = 0; + + while (true) { + if (nResult >= nMaxResult) { + break; + } + + // set and open the file according to the STSma param + if (tdSetAndOpenTSmaFile(&tReadH, queryWin)) { + char bTree[100] = "\0"; + while (strncmp(bTree, "has more nodes", 100) == 0) { + if (nResult >= nMaxResult) { + break; + } + // tdGetDataFromBTree(bTree, queryWin, lastKey) + // fill the pData + ++nResult; + } + } + } +#endif + // read data from file and fill the result + smaCloseDBF(&tReadH.dFile); + return TSDB_CODE_SUCCESS; +} + +int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { + SSmaCfg *pCfg = (SSmaCfg *)pMsg; + + if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) { + return -1; + } + + tdTSmaAdd(pSma, 1); + return 0; +} + +int32_t tdDropTSma(SSma *pSma, char *pMsg) { +#if 0 + SVDropTSmaReq vDropSmaReq = {0}; + if (!tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + // TODO: send msg to stream computing to drop tSma + // if ((send msg to stream computing) < 0) { + // tDestroyTSma(&vCreateSmaReq); + // return -1; + // } + // + + if (metaDropTSma(SMA_META(pSma), vDropSmaReq.indexUid) < 0) { + // TODO: handle error + return -1; + } + + if (tdDropTSmaData(pSma, vDropSmaReq.indexUid) < 0) { + // TODO: handle error + return -1; + } + + tdTSmaSub(pSma, 1); +#endif + + // TODO: return directly or go on follow steps? + return TSDB_CODE_SUCCESS; +} + +static SSmaStatItem *tdNewSmaStatItem(int8_t state) { + SSmaStatItem *pItem = NULL; + + pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem)); + if (!pItem) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pItem->state = state; + pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), + true, HASH_ENTRY_LOCK); + if (!pItem->expiredWindows) { + taosMemoryFreeClear(pItem); + return NULL; + } + + return pItem; +} + +static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, + int64_t version) { + SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); + if (!pItem) { + // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later + pItem = tdNewSmaStatItem(TSDB_SMA_STAT_OK); // TODO use the real state + if (!pItem) { + // Response to stream computing: OOM + // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. + return TSDB_CODE_FAILED; + } + + // cache smaMeta + STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); + if (!pTSma) { + terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; + taosHashCleanup(pItem->expiredWindows); + taosMemoryFree(pItem); + smaWarn("vgId:%d set expire window, get tsma meta failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), + indexUid, tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + pItem->pTSma = pTSma; + + if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { + // If error occurs during put smaStatItem, free the resources of pItem + taosHashCleanup(pItem->expiredWindows); + taosMemoryFree(pItem); + return TSDB_CODE_FAILED; + } + } else if (!(pItem = *(SSmaStatItem **)pItem)) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &version, sizeof(version)) != 0) { + // If error occurs during taosHashPut expired windows, remove the smaIndex from pSma->pSmaStat, thus TSDB would + // tell query module to query raw TS data. + // N.B. + // 1) It is assumed to be extemely little probability event of fail to taosHashPut. + // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired + // windows failed to put into hash table. + taosHashCleanup(pItem->expiredWindows); + taosMemoryFreeClear(pItem->pTSma); + taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); + smaWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid, + winSKey); + return TSDB_CODE_FAILED; + } + + smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid, + winSKey); + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Update expired window according to msg from stream computing module. + * + * @param pSma + * @param msg SSubmitReq + * @return int32_t + */ +int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version) { + // no time-range-sma, just return success + if (atomic_load_16(&SMA_TSMA_NUM(pSma)) <= 0) { + smaTrace("vgId:%d not update expire window since no tSma", SMA_VID(pSma)); + return TSDB_CODE_SUCCESS; + } + + if (!SMA_META(pSma)) { + terrno = TSDB_CODE_INVALID_PTR; + smaError("vgId:%d update expire window failed since no meta ptr", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) < 0) { + smaError("vgId:%d init sma env failed since %s", SMA_VID(pSma), terrstr(terrno)); + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + // Firstly, assume that tSma can only be created on super table/normal table. + // getActiveTimeWindow + + SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); + + TASSERT(pEnv && pStat && pItemsHash); + + // basic procedure + // TODO: optimization + tdRefSmaStat(pSma, pStat); + + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock = NULL; + SInterval interval = {0}; + TSKEY lastWinSKey = INT64_MIN; + + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { + return TSDB_CODE_FAILED; + } + + while (true) { + tGetSubmitMsgNext(&msgIter, &pBlock); + if (!pBlock) break; + + STSmaWrapper *pSW = NULL; + STSma *pTSma = NULL; + + SSubmitBlkIter blkIter = {0}; + if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) { + pSW = tFreeTSmaWrapper(pSW, false); + break; + } + + while (true) { + STSRow *row = tGetSubmitBlkNext(&blkIter); + if (!row) { + pSW = tFreeTSmaWrapper(pSW, false); + break; + } + if (!pSW || (pTSma && (pTSma->tableUid != msgIter.suid))) { + if (pSW) { + pSW = tFreeTSmaWrapper(pSW, false); + } + if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), msgIter.suid, false))) { + break; + } + if ((pSW->number) <= 0 || !pSW->tSma) { + pSW = tFreeTSmaWrapper(pSW, false); + break; + } + + pTSma = pSW->tSma; + + interval.interval = pTSma->interval; + interval.intervalUnit = pTSma->intervalUnit; + interval.offset = pTSma->offset; + interval.precision = SMA_TSDB_CFG(pSma)->precision; + interval.sliding = pTSma->sliding; + interval.slidingUnit = pTSma->slidingUnit; + } + + // TODO: process multiple tsma for one table uid + TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); + + if (lastWinSKey != winSKey) { + lastWinSKey = winSKey; + if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) { + pSW = tFreeTSmaWrapper(pSW, false); + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + } else { + smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", + SMA_VID(pSma), pTSma->indexUid, winSKey); + } + } + } + + tdUnRefSmaStat(pSma, pStat); + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 26e9dfe2e2ceba9c854c51676d304428b1b86082..d94c3e387a09dd891825a6d6ed11b96a248d9605 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -20,6 +20,28 @@ void tqTmrRspFunc(void* param, void* tmrId) { atomic_store_8(&pHandle->pushHandle.tmrStopped, 1); } +static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataBlkRsp* pRsp) { + SStreamDataSubmit* pSubmit = *ppSubmit; + while (pSubmit != NULL) { + ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1); + if (tqDataExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) { + /*ASSERT(0);*/ + } + // update processed + atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); + streamQSetSuccess(&pHandle->pushHandle.inputQ); + streamDataSubmitRefDec(pSubmit); + if (pRsp->blockNum > 0) { + *ppSubmit = pSubmit; + return 0; + } else { + pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ); + } + } + *ppSubmit = pSubmit; + return -1; +} + int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { SMqDataBlkRsp rsp = {0}; // 1. guard and set status executing @@ -42,38 +64,14 @@ int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { } // 3. exec, after each success, update processed ver // first run - while (pSubmit != NULL) { - ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1); - if (tqDataExec(pTq, &pHandle->execHandle, pSubmit->data, &rsp, 0) < 0) { - /*ASSERT(0);*/ - } - // update processed - atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); - streamQSetSuccess(&pHandle->pushHandle.inputQ); - streamDataSubmitRefDec(pSubmit); - if (rsp.blockNum > 0) { - goto SEND_RSP; - } else { - pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ); - } + if (tqLoopExecFromQueue(pTq, pHandle, &pSubmit, &rsp) == 0) { + goto SEND_RSP; } // set exec status closing atomic_store_8(&pHandle->pushHandle.execStatus, TASK_STATUS__CLOSING); // second run - while (pSubmit != NULL) { - ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1); - if (tqDataExec(pTq, &pHandle->execHandle, pSubmit->data, &rsp, 0) < 0) { - /*ASSERT(0);*/ - } - // update processed - atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); - streamQSetSuccess(&pHandle->pushHandle.inputQ); - streamDataSubmitRefDec(pSubmit); - if (rsp.blockNum > 0) { - goto SEND_RSP; - } else { - pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ); - } + if (tqLoopExecFromQueue(pTq, pHandle, &pSubmit, &rsp) == 0) { + goto SEND_RSP; } // set exec status idle atomic_store_8(&pHandle->pushHandle.execStatus, TASK_STATUS__IDLE); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 0a85cb46383f7183efeea2dd44a869933a27587c..59a795b77a79f09232e6736a54aa9e5741e411b0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -86,71 +86,34 @@ static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError); static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo); static void tsdbLoadAndMergeFromCache(STsdb *pTsdb, SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, TSKEY maxKey, int maxRows, int8_t update); -int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf); +static int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf); +static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn); -int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { - SDiskID did; - SDFileSet nSet = {0}; - STsdbFS *pfs = REPO_FS(pRepo); - int level; - - ASSERT(pSet->fid >= pRtn->minFid); +int tsdbBegin(STsdb *pTsdb) { + if (!pTsdb) return 0; - level = tsdbGetFidLevel(pSet->fid, pRtn); + STsdbMemTable *pMem; - if (tfsAllocDisk(pRepo->pVnode->pTfs, level, &did) < 0) { - terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + if (tsdbMemTableCreate(pTsdb, &pTsdb->mem) < 0) { return -1; } - if (did.level > TSDB_FSET_LEVEL(pSet)) { - // Need to move the FSET to higher level - tsdbInitDFileSet(pRepo, &nSet, did, pSet->fid, FS_TXN_VERSION(pfs)); - - if (tsdbCopyDFileSet(pSet, &nSet) < 0) { - tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno)); - return -1; - } - - if (tsdbUpdateDFileSet(pfs, &nSet) < 0) { - return -1; - } - - tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id); - } else { - // On a correct level - if (tsdbUpdateDFileSet(pfs, pSet) < 0) { - return -1; - } - } - return 0; } -int tsdbPrepareCommit(STsdb *pTsdb) { - if (pTsdb->mem == NULL) return 0; - - ASSERT(pTsdb->imem == NULL); - - pTsdb->imem = pTsdb->mem; - pTsdb->mem = NULL; - return 0; -} - -int tsdbCommit(STsdb *pRepo) { +int32_t tsdbCommit(STsdb *pTsdb) { + int32_t code = 0; SCommitH commith = {0}; SDFileSet *pSet = NULL; int fid; - // if (pRepo->imem == NULL) return 0; - pRepo->imem = pRepo->mem; - pRepo->mem = NULL; + ASSERT(pTsdb->imem == NULL && pTsdb->mem); + pTsdb->imem = pTsdb->mem; + pTsdb->mem = NULL; - tsdbStartCommit(pRepo); - // Resource initialization - if (tsdbInitCommitH(&commith, pRepo) < 0) { + // start commit + tsdbStartCommit(pTsdb); + if (tsdbInitCommitH(&commith, pTsdb) < 0) { return -1; } @@ -158,14 +121,14 @@ int tsdbCommit(STsdb *pRepo) { tsdbSeekCommitIter(&commith, commith.rtn.minKey); while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { if (pSet->fid < commith.rtn.minFid) { - tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + tsdbInfo("vgId:%d, FSET %d on level %d disk id %d expires, remove it", REPO_ID(pTsdb), pSet->fid, TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); } else { break; } } - // Loop to commit to each file + // commit fid = tsdbNextCommitFid(&(commith)); while (true) { // Loop over both on disk and memory @@ -174,7 +137,7 @@ int tsdbCommit(STsdb *pRepo) { if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) { // Only has existing FSET but no memory data to commit in this // existing FSET, only check if file in correct retention - if (tsdbApplyRtnOnFSet(pRepo, pSet, &(commith.rtn)) < 0) { + if (tsdbApplyRtnOnFSet(pTsdb, pSet, &(commith.rtn)) < 0) { tsdbDestroyCommitH(&commith); return -1; } @@ -205,12 +168,64 @@ int tsdbCommit(STsdb *pRepo) { } } + // end commit tsdbDestroyCommitH(&commith); - tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS); + tsdbEndCommit(pTsdb, TSDB_CODE_SUCCESS); + + return code; +} + +static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { + SDiskID did; + SDFileSet nSet = {0}; + STsdbFS *pfs = REPO_FS(pRepo); + int level; + + ASSERT(pSet->fid >= pRtn->minFid); + + level = tsdbGetFidLevel(pSet->fid, pRtn); + + if (tfsAllocDisk(pRepo->pVnode->pTfs, level, &did) < 0) { + terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + return -1; + } + + if (did.level > TSDB_FSET_LEVEL(pSet)) { + // Need to move the FSET to higher level + tsdbInitDFileSet(pRepo, &nSet, did, pSet->fid, FS_TXN_VERSION(pfs)); + + if (tsdbCopyDFileSet(pSet, &nSet) < 0) { + tsdbError("vgId:%d, failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno)); + return -1; + } + + if (tsdbUpdateDFileSet(pfs, &nSet) < 0) { + return -1; + } + + tsdbInfo("vgId:%d, FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id); + } else { + // On a correct level + if (tsdbUpdateDFileSet(pfs, pSet) < 0) { + return -1; + } + } return 0; } +// int tsdbPrepareCommit(STsdb *pTsdb) { +// if (pTsdb->mem == NULL) return 0; + +// ASSERT(pTsdb->imem == NULL); + +// pTsdb->imem = pTsdb->mem; +// pTsdb->mem = NULL; +// return 0; +// } + void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) { STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo); TSKEY minKey, midKey, maxKey, now; @@ -224,14 +239,14 @@ void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) { pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->days, pCfg->precision)); pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->days, pCfg->precision)); pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->days, pCfg->precision)); - tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey, + tsdbDebug("vgId:%d, now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey, pRtn->minFid, pRtn->midFid, pRtn->maxFid); } static void tsdbStartCommit(STsdb *pRepo) { STsdbMemTable *pMem = pRepo->imem; - tsdbInfo("vgId:%d start to commit", REPO_ID(pRepo)); + tsdbInfo("vgId:%d, start to commit", REPO_ID(pRepo)); tsdbStartFSTxn(pRepo, 0, 0); } @@ -240,7 +255,7 @@ static void tsdbEndCommit(STsdb *pTsdb, int eno) { tsdbEndFSTxn(pTsdb); tsdbMemTableDestroy(pTsdb->imem); pTsdb->imem = NULL; - tsdbInfo("vgId:%d commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); + tsdbInfo("vgId:%d, commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); } static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo) { @@ -413,7 +428,7 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { if (tsdbWriteBlockIdx(TSDB_COMMIT_HEAD_FILE(pCommith), pCommith->aBlkIdx, (void **)(&(TSDB_COMMIT_BUF(pCommith)))) < 0) { - tsdbError("vgId:%d failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + tsdbError("vgId:%d, failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); tsdbCloseCommitFile(pCommith, true); // revert the file change tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); @@ -421,7 +436,7 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { } if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) { - tsdbError("vgId:%d failed to update FSET %d header since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + tsdbError("vgId:%d, failed to update FSET %d header since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); tsdbCloseCommitFile(pCommith, true); // revert the file change tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); @@ -531,8 +546,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid return -1; } - tsdbDebug("vgId:%d FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet), - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + tsdbDebug("vgId:%d, FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), + TSDB_FSET_FID(pSet), TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); } else { pCommith->isRFileSet = false; } @@ -543,7 +558,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid tsdbInitDFileSet(pRepo, pWSet, did, fid, FS_TXN_VERSION(REPO_FS(pRepo))); if (tsdbCreateDFileSet(pRepo, pWSet, true) < 0) { - tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), + tsdbError("vgId:%d, failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno)); if (pCommith->isRFileSet) { tsdbCloseAndUnsetFSet(&(pCommith->readh)); @@ -554,7 +569,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid pCommith->isDFileSame = false; pCommith->isLFileSame = false; - tsdbDebug("vgId:%d FSET %d at level %d disk id %d is created to commit", REPO_ID(pRepo), TSDB_FSET_FID(pWSet), + tsdbDebug("vgId:%d, FSET %d at level %d disk id %d is created to commit", REPO_ID(pRepo), TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet)); } else { did.level = TSDB_FSET_LEVEL(pSet); @@ -567,7 +582,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); tsdbInitDFile(pRepo, pWHeadf, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); if (tsdbCreateDFile(pRepo, pWHeadf, true, TSDB_FILE_HEAD) < 0) { - tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), + tsdbError("vgId:%d, failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), tstrerror(terrno)); if (pCommith->isRFileSet) { @@ -582,7 +597,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid tsdbInitDFileEx(pWDataf, pRDataf); // if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { if (tsdbOpenDFile(pWDataf, TD_FILE_WRITE) < 0) { - tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), + tsdbError("vgId:%d, failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), tstrerror(terrno)); tsdbCloseDFileSet(pWSet); @@ -603,7 +618,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid // if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { if (tsdbOpenDFile(pWLastf, TD_FILE_WRITE) < 0) { - tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tsdbError("vgId:%d, failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), tstrerror(terrno)); tsdbCloseDFileSet(pWSet); @@ -618,7 +633,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid pCommith->isLFileSame = false; if (tsdbCreateDFile(pRepo, pWLastf, true, TSDB_FILE_LAST) < 0) { - tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tsdbError("vgId:%d, failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), tstrerror(terrno)); tsdbCloseDFileSet(pWSet); @@ -635,11 +650,11 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pWSmadF = TSDB_COMMIT_SMAD_FILE(pCommith); if (!taosCheckExistFile(TSDB_FILE_FULL_NAME(pRSmadF))) { - tsdbDebug("vgId:%d create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmadF)); + tsdbDebug("vgId:%d, create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmadF)); tsdbInitDFile(pRepo, pWSmadF, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_SMAD); if (tsdbCreateDFile(pRepo, pWSmadF, true, TSDB_FILE_SMAD) < 0) { - tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmadF), + tsdbError("vgId:%d, failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmadF), tstrerror(terrno)); tsdbCloseDFileSet(pWSet); @@ -652,7 +667,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid } else { tsdbInitDFileEx(pWSmadF, pRSmadF); if (tsdbOpenDFile(pWSmadF, O_RDWR) < 0) { - tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmadF), + tsdbError("vgId:%d, failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmadF), tstrerror(terrno)); tsdbCloseDFileSet(pWSet); @@ -671,7 +686,7 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid if ((pCommith->isLFileSame) && taosCheckExistFile(TSDB_FILE_FULL_NAME(pRSmalF))) { tsdbInitDFileEx(pWSmalF, pRSmalF); if (tsdbOpenDFile(pWSmalF, O_RDWR) < 0) { - tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF), + tsdbError("vgId:%d, failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF), tstrerror(terrno)); tsdbCloseDFileSet(pWSet); @@ -682,11 +697,11 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid } } } else { - tsdbDebug("vgId:%d create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmalF)); + tsdbDebug("vgId:%d, create data file %s as not exist", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pRSmalF)); tsdbInitDFile(pRepo, pWSmalF, did, fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_SMAL); if (tsdbCreateDFile(pRepo, pWSmalF, true, TSDB_FILE_SMAL) < 0) { - tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF), + tsdbError("vgId:%d, failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWSmalF), tstrerror(terrno)); tsdbCloseDFileSet(pWSet); @@ -704,8 +719,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid // extern int32_t tsTsdbMetaCompactRatio; -int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, - SBlockIdx *pIdx) { +static int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, + SBlockIdx *pIdx) { size_t nSupBlocks; size_t nSubBlocks; uint32_t tlen; @@ -757,7 +772,7 @@ int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray pIdx->uid = TABLE_UID(pTable); pIdx->hasLast = pBlock->last ? 1 : 0; - pIdx->maxKey = pBlock->keyLast; + pIdx->maxKey = pBlock->maxKey; pIdx->numOfBlocks = (uint32_t)nSupBlocks; pIdx->len = tlen; pIdx->offset = (uint32_t)offset; @@ -765,7 +780,7 @@ int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray return 0; } -int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { +static int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { SBlockIdx *pBlkIdx; size_t nidx = taosArrayGetSize(pIdxA); int tlen = 0, size; @@ -878,7 +893,7 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) { return -1; } } else { - if (tsdbCommitMemData(pCommith, pIter, pBlock->keyFirst - 1, true) < 0) { + if (tsdbCommitMemData(pCommith, pIter, pBlock->minKey.ts - 1, true) < 0) { return -1; } } @@ -887,7 +902,7 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) { } if (tsdbWriteBlockInfo(pCommith) < 0) { - tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), + tsdbError("vgId:%d, failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); return -1; } @@ -929,7 +944,7 @@ static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) { } if (tsdbMoveBlock(pCommith, bidx) < 0) { - tsdbError("vgId:%d failed to move block into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), + tsdbError("vgId:%d, failed to move block into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); taosMemoryFreeClear(pTSchema); return -1; @@ -939,7 +954,7 @@ static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) { } if (tsdbWriteBlockInfo(pCommith) < 0) { - tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), + tsdbError("vgId:%d, failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); taosMemoryFreeClear(pTSchema); return -1; @@ -973,9 +988,9 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { TSKEY key = *(TSKEY *)arg1; SBlock *pBlock = (SBlock *)arg2; - if (key < pBlock->keyFirst) { + if (key < pBlock->minKey.ts) { return -1; - } else if (key > pBlock->keyLast) { + } else if (key > pBlock->maxKey.ts) { return 1; } else { return 0; @@ -999,8 +1014,8 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { * @param ppExBuf * @return int */ -int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols, - SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) { +static int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols, + SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) { STsdbCfg *pCfg = REPO_CFG(pRepo); SBlockData *pBlockData = NULL; SAggrBlkData *pAggrBlkData = NULL; @@ -1158,7 +1173,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM))); if (ncol != 0) { - tsdbSetBlockColOffset(pBlockCol, toffset); + pBlockCol->offset = toffset; pBlockCol->len = flen; // data + bitmaps pBlockCol->blen = tBitmapsLen; ++tcol; @@ -1203,16 +1218,16 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF pBlock->numOfSubBlocks = isSuper ? 1 : 0; pBlock->numOfCols = nColsNotAllNull; pBlock->numOfBSma = nColsOfBlockSma; - pBlock->keyFirst = dataColsKeyFirst(pDataCols); - pBlock->keyLast = dataColsKeyLast(pDataCols); + pBlock->minKey.ts = dataColsKeyFirst(pDataCols); + pBlock->maxKey.ts = dataColsKeyLast(pDataCols); pBlock->aggrStat = aggrStatus; pBlock->blkVer = SBlockVerLatest; pBlock->aggrOffset = (uint64_t)offsetAggr; - tsdbDebug("vgId:%d uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64 + tsdbDebug("vgId:%d, uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, REPO_ID(pRepo), TABLE_UID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, - pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); + pBlock->numOfCols, pBlock->minKey.ts, pBlock->maxKey.ts); return 0; } @@ -1295,7 +1310,7 @@ static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { if (bidx == nBlocks - 1) { keyLimit = pCommith->maxKey; } else { - keyLimit = pBlock[1].keyFirst - 1; + keyLimit = pBlock[1].minKey.ts - 1; } SSkipListIterator titer = *(pIter->pIter); @@ -1337,8 +1352,8 @@ static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { } subBlocks[pBlock->numOfSubBlocks] = block; supBlock = *pBlock; - supBlock.keyFirst = mInfo.keyFirst; - supBlock.keyLast = mInfo.keyLast; + supBlock.minKey.ts = mInfo.keyFirst; + supBlock.maxKey.ts = mInfo.keyLast; supBlock.numOfSubBlocks++; supBlock.numOfRows = pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed; supBlock.offset = taosArrayGetSize(pCommith->aSubBlk) * sizeof(SBlock); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 844cfc094b5650899373152ef7168f6ed4909129..321667c55bcdcd2829df186b4af16b6792135c65 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -15,14 +15,422 @@ #include "tsdb.h" -int tsdbBegin(STsdb *pTsdb) { - if (!pTsdb) return 0; - - STsdbMemTable *pMem; +typedef struct { + SMemTable *pMemTable; + int32_t minutes; + int8_t precision; + TSKEY nCommitKey; + int32_t fid; + TSKEY minKey; + TSKEY maxKey; + SReadH readh; + SDFileSet wSet; + SArray *aBlkIdx; + SArray *aSupBlk; + SArray *aSubBlk; + SArray *aDelInfo; +} SCommitH; - if (tsdbMemTableCreate(pTsdb, &pTsdb->mem) < 0) { +static int32_t tsdbCommitStart(SCommitH *pCHandle, STsdb *pTsdb); +static int32_t tsdbCommitEnd(SCommitH *pCHandle); +static int32_t tsdbCommitImpl(SCommitH *pCHandle); + +int32_t tsdbBegin2(STsdb *pTsdb) { + int32_t code = 0; + + ASSERT(pTsdb->mem == NULL); + code = tsdbMemTableCreate2(pTsdb, (SMemTable **)&pTsdb->mem); + if (code) { + tsdbError("vgId:%d failed to begin TSDB since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + goto _exit; + } + +_exit: + return code; +} + +int32_t tsdbCommit2(STsdb *pTsdb) { + int32_t code = 0; + SCommitH ch = {0}; + + // start to commit + code = tsdbCommitStart(&ch, pTsdb); + if (code) { + goto _exit; + } + + // commit + code = tsdbCommitImpl(&ch); + if (code) { + goto _err; + } + + // end commit + code = tsdbCommitEnd(&ch); + if (code) { + goto _exit; + } + +_exit: + return code; + +_err: + tsdbError("vgId:%d failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbCommitStart(SCommitH *pCHandle, STsdb *pTsdb) { + int32_t code = 0; + SMemTable *pMemTable = (SMemTable *)pTsdb->mem; + + tsdbInfo("vgId:%d start to commit", TD_VID(pTsdb->pVnode)); + + // switch to commit + ASSERT(pTsdb->imem == NULL && pTsdb->mem); + pTsdb->imem = pTsdb->mem; + pTsdb->mem = NULL; + + // open handle + pCHandle->pMemTable = pMemTable; + pCHandle->minutes = pTsdb->keepCfg.days; + pCHandle->precision = pTsdb->keepCfg.precision; + pCHandle->nCommitKey = pMemTable->minKey.ts; + + code = tsdbInitReadH(&pCHandle->readh, pTsdb); + if (code) { + goto _err; + } + pCHandle->aBlkIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pCHandle->aBlkIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pCHandle->aSupBlk = taosArrayInit(0, sizeof(SBlock)); + if (pCHandle->aSupBlk == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pCHandle->aSubBlk = taosArrayInit(0, sizeof(SBlock)); + if (pCHandle->aSubBlk == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pCHandle->aDelInfo = taosArrayInit(0, sizeof(SDelInfo)); + if (pCHandle->aDelInfo == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + // start FS transaction + tsdbStartFSTxn(pTsdb, 0, 0); + + return code; + +_err: + return code; +} + +static int32_t tsdbCommitEnd(SCommitH *pCHandle) { + int32_t code = 0; + STsdb *pTsdb = pCHandle->pMemTable->pTsdb; + SMemTable *pMemTable = (SMemTable *)pTsdb->imem; + + // end transaction + code = tsdbEndFSTxn(pTsdb); + if (code) { + goto _err; + } + + // close handle + taosArrayClear(pCHandle->aDelInfo); + taosArrayClear(pCHandle->aSubBlk); + taosArrayClear(pCHandle->aSupBlk); + taosArrayClear(pCHandle->aBlkIdx); + tsdbDestroyReadH(&pCHandle->readh); + + // destroy memtable (todo: unref it) + pTsdb->imem = NULL; + tsdbMemTableDestroy2(pMemTable); + + tsdbInfo("vgId:%d commit over", TD_VID(pTsdb->pVnode)); + return code; + +_err: + return code; +} + +static int32_t tsdbCommitTableStart(SCommitH *pCHandle) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbCommitTableEnd(SCommitH *pCHandle) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbCommitTable(SCommitH *pCHandle, SMemData *pMemData, SBlockIdx *pBlockIdx) { + int32_t code = 0; + SMemDataIter iter = {0}; + + // commit table start + code = tsdbCommitTableStart(pCHandle); + if (code) { + goto _err; + } + + // commit table impl + if (pMemData && pBlockIdx) { + // TODO + } else if (pMemData) { + // TODO + } else { + // TODO + } + + // commit table end + code = tsdbCommitTableEnd(pCHandle); + if (code) { + goto _err; + } + + return code; + +_err: + return code; +} + +static int32_t tsdbTableIdCmprFn(const void *p1, const void *p2) { + TABLEID *pId1 = (TABLEID *)p1; + TABLEID *pId2 = (TABLEID *)p2; + + if (pId1->suid < pId2->suid) { + return -1; + } else if (pId1->suid > pId2->suid) { + return 1; + } + + if (pId1->uid < pId2->uid) { + return -1; + } else if (pId1->uid > pId2->uid) { + return 1; + } + + return 0; +} + +static int32_t tsdbWriteBlockIdx(SDFile *pFile, SArray *pArray, uint8_t **ppBuf) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbCommitFileStart(SCommitH *pCHandle) { + int32_t code = 0; + STsdb *pTsdb = pCHandle->pMemTable->pTsdb; + SDFileSet *pSet = NULL; + + taosArrayClear(pCHandle->aBlkIdx); + + return code; +} + +static int32_t tsdbCommitFileEnd(SCommitH *pCHandle) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbCommitFile(SCommitH *pCHandle) { + int32_t code = 0; + SMemData *pMemData; + SBlockIdx *pBlockIdx; + int32_t iMemData; + int32_t nMemData; + int32_t iBlockIdx; + int32_t nBlockIdx; + + // commit file start + code = tsdbCommitFileStart(pCHandle); + if (code) { + goto _err; + } + + // commit file impl + iMemData = 0; + nMemData = taosArrayGetSize(pCHandle->pMemTable->aMemData); + iBlockIdx = 0; + nBlockIdx = 0; // todo + + for (;;) { + if (iMemData >= nMemData && iBlockIdx >= nBlockIdx) break; + + pMemData = NULL; + pBlockIdx = NULL; + if (iMemData < nMemData) { + pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData); + } + if (iBlockIdx < nBlockIdx) { + // pBlockIdx = ; + } + + if (pMemData && pBlockIdx) { + int32_t c = tsdbTableIdCmprFn(pMemData, pBlockIdx); + if (c < 0) { + iMemData++; + pBlockIdx = NULL; + } else if (c == 0) { + iMemData++; + iBlockIdx++; + } else { + iBlockIdx++; + pMemData = NULL; + } + } else { + if (pMemData) { + iMemData++; + } else { + iBlockIdx++; + } + } + + code = tsdbCommitTable(pCHandle, pMemData, pBlockIdx); + if (code) { + goto _err; + } + } + + // commit file end + code = tsdbCommitFileEnd(pCHandle); + if (code) { + goto _err; + } + + return code; + +_err: + return code; +} + +static int32_t tsdbCommitData(SCommitH *pCHandle) { + int32_t code = 0; + int32_t fid; + + if (pCHandle->pMemTable->nRows == 0) goto _exit; + + // loop to commit to each file + for (;;) { + if (pCHandle->nCommitKey == TSKEY_MAX) break; + + pCHandle->fid = TSDB_KEY_FID(pCHandle->nCommitKey, pCHandle->minutes, pCHandle->precision); + tsdbGetFidKeyRange(pCHandle->minutes, pCHandle->precision, pCHandle->fid, &pCHandle->minKey, &pCHandle->maxKey); + code = tsdbCommitFile(pCHandle); + if (code) { + goto _err; + } + } + +_exit: + return code; + +_err: + return code; +} + +static int32_t delInfoCmprFn(const void *p1, const void *p2) { + SDelInfo *pDelInfo1 = (SDelInfo *)p1; + SDelInfo *pDelInfo2 = (SDelInfo *)p2; + + if (pDelInfo1->suid < pDelInfo2->suid) { + return -1; + } else if (pDelInfo1->suid > pDelInfo2->suid) { + return 1; + } + + if (pDelInfo1->uid < pDelInfo2->uid) { return -1; + } else if (pDelInfo1->uid > pDelInfo2->uid) { + return 1; + } + + if (pDelInfo1->version < pDelInfo2->version) { + return -1; + } else if (pDelInfo1->version > pDelInfo2->version) { + return 1; } return 0; } +static int32_t tsdbCommitDelete(SCommitH *pCHandle) { + int32_t code = 0; + SDelInfo delInfo; + SMemData *pMemData; + + if (pCHandle->pMemTable->nDelOp == 0) goto _exit; + + // load del array (todo) + + // loop to append SDelInfo + for (int32_t iMemData = 0; iMemData < taosArrayGetSize(pCHandle->pMemTable->aMemData); iMemData++) { + pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData); + + for (SDelOp *pDelOp = pMemData->delOpHead; pDelOp; pDelOp = pDelOp->pNext) { + delInfo = (SDelInfo){.suid = pMemData->suid, + .uid = pMemData->uid, + .version = pDelOp->version, + .sKey = pDelOp->sKey, + .eKey = pDelOp->eKey}; + if (taosArrayPush(pCHandle->aDelInfo, &delInfo) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + } + + taosArraySort(pCHandle->aDelInfo, delInfoCmprFn); + + // write to new file + +_exit: + return code; + +_err: + return code; +} + +static int32_t tsdbCommitCache(SCommitH *pCHandle) { + int32_t code = 0; + // TODO + return code; +} + +static int32_t tsdbCommitImpl(SCommitH *pCHandle) { + int32_t code = 0; + + // commit data + code = tsdbCommitData(pCHandle); + if (code) { + goto _err; + } + + // commit delete + code = tsdbCommitDelete(pCHandle); + if (code) { + goto _err; + } + + // commit cache if need (todo) + if (0) { + code = tsdbCommitCache(pCHandle); + if (code) { + goto _err; + } + } + + return code; + +_err: + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 6dfd73158ea15b3f36b23158b0de54a7a904725c..f1941a3bad04c0c49ed33d75faba6abccb965dd9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -37,11 +37,11 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired); // static int tsdbProcessExpiredFS(STsdb *pRepo); // static int tsdbCreateMeta(STsdb *pRepo); -static void tsdbGetRootDir(int repoid, const char* dir, char dirName[]) { +static void tsdbGetRootDir(int repoid, const char *dir, char dirName[]) { snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, dir); } -static void tsdbGetDataDir(int repoid, const char* dir, char dirName[]) { +static void tsdbGetDataDir(int repoid, const char *dir, char dirName[]) { snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, dir); } @@ -216,16 +216,7 @@ STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg) { return NULL; } - pfs->metaCache = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (pfs->metaCache == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbFreeFS(pfs); - return NULL; - } - pfs->intxn = false; - pfs->metaCacheComp = NULL; - pfs->nstatus = tsdbNewFSStatus(maxFSet); if (pfs->nstatus == NULL) { tsdbFreeFS(pfs); @@ -238,8 +229,6 @@ STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg) { void *tsdbFreeFS(STsdbFS *pfs) { if (pfs) { pfs->nstatus = tsdbFreeFSStatus(pfs->nstatus); - taosHashCleanup(pfs->metaCache); - pfs->metaCache = NULL; pfs->cstatus = tsdbFreeFSStatus(pfs->cstatus); taosThreadRwlockDestroy(&(pfs->lock)); taosMemoryFree(pfs); @@ -260,7 +249,7 @@ int tsdbOpenFS(STsdb *pRepo) { tsdbGetRtnSnap(pRepo, &pRepo->rtn); if (taosCheckExistFile(current)) { if (tsdbOpenFSFromCurrent(pRepo) < 0) { - tsdbError("vgId:%d failed to open FS since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbError("vgId:%d, failed to open FS since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } @@ -271,19 +260,19 @@ int tsdbOpenFS(STsdb *pRepo) { } else { // should skip expired fileset inside of the function if (tsdbRestoreCurrent(pRepo) < 0) { - tsdbError("vgId:%d failed to restore current file since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbError("vgId:%d, failed to restore current file since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } } if (tsdbScanAndTryFixFS(pRepo) < 0) { - tsdbError("vgId:%d failed to scan and fix FS since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbError("vgId:%d, failed to scan and fix FS since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } // // Load meta cache if has meta file // if ((!(pRepo->state & TSDB_STATE_BAD_META)) && tsdbLoadMetaCache(pRepo, true) < 0) { - // tsdbError("vgId:%d failed to open FS while loading meta cache since %s", REPO_ID(pRepo), tstrerror(terrno)); + // tsdbError("vgId:%d, failed to open FS while loading meta cache since %s", REPO_ID(pRepo), tstrerror(terrno)); // return -1; // } @@ -607,7 +596,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { // current file exists, try to recover pFile = taosOpenFile(current, TD_FILE_READ); if (pFile == NULL) { - tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno)); + tsdbError("vgId:%d, failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -618,20 +607,20 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { int nread = (int)taosReadFile(pFile, buffer, TSDB_FILE_HEAD_SIZE); if (nread < 0) { - tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current, + tsdbError("vgId:%d, failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } if (nread < TSDB_FILE_HEAD_SIZE) { - tsdbError("vgId:%d failed to read header of file %s, read bytes:%d", REPO_ID(pRepo), current, nread); + tsdbError("vgId:%d, failed to read header of file %s, read bytes:%d", REPO_ID(pRepo), current, nread); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; goto _err; } if (!taosCheckChecksumWhole((uint8_t *)buffer, TSDB_FILE_HEAD_SIZE)) { - tsdbError("vgId:%d header of file %s failed checksum check", REPO_ID(pRepo), current); + tsdbError("vgId:%d, header of file %s failed checksum check", REPO_ID(pRepo), current); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; goto _err; } @@ -652,19 +641,19 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { nread = (int)taosReadFile(pFile, buffer, fsheader.len); if (nread < 0) { - tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno)); + tsdbError("vgId:%d, failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } if (nread < fsheader.len) { - tsdbError("vgId:%d failed to read %d bytes from file %s", REPO_ID(pRepo), fsheader.len, current); + tsdbError("vgId:%d, failed to read %d bytes from file %s", REPO_ID(pRepo), fsheader.len, current); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; goto _err; } if (!taosCheckChecksumWhole((uint8_t *)buffer, fsheader.len)) { - tsdbError("vgId:%d file %s is corrupted since wrong checksum", REPO_ID(pRepo), current); + tsdbError("vgId:%d, file %s is corrupted since wrong checksum", REPO_ID(pRepo), current); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; goto _err; } @@ -694,7 +683,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { SFSStatus *pStatus = pfs->cstatus; // if (tsdbScanAndTryFixMFile(pRepo) < 0) { - // tsdbError("vgId:%d failed to fix MFile since %s", REPO_ID(pRepo), tstrerror(terrno)); + // tsdbError("vgId:%d, failed to fix MFile since %s", REPO_ID(pRepo), tstrerror(terrno)); // return -1; // } @@ -704,7 +693,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(pStatus->df, i); if (tsdbScanAndTryFixDFileSet(pRepo, pSet) < 0) { - tsdbError("vgId:%d failed to fix MFile since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbError("vgId:%d, failed to fix MFile since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } } @@ -724,7 +713,7 @@ static int tsdbScanRootDir(STsdb *pRepo) { tsdbGetRootDir(REPO_ID(pRepo), pRepo->dir, rootDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), rootDir); if (tdir == NULL) { - tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); + tsdbError("vgId:%d, failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); return -1; } @@ -741,7 +730,7 @@ static int tsdbScanRootDir(STsdb *pRepo) { // } (void)tfsRemoveFile(pf); - tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), pf->aname); + tsdbDebug("vgId:%d, invalid file %s is removed", REPO_ID(pRepo), pf->aname); } tfsClosedir(tdir); @@ -758,7 +747,7 @@ static int tsdbScanDataDir(STsdb *pRepo) { tsdbGetDataDir(REPO_ID(pRepo), pRepo->dir, dataDir); STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), dataDir); if (tdir == NULL) { - tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); + tsdbError("vgId:%d, failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); return -1; } @@ -767,7 +756,7 @@ static int tsdbScanDataDir(STsdb *pRepo) { if (!tsdbIsTFileInFS(pfs, pf)) { (void)tfsRemoveFile(pf); - tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), pf->aname); + tsdbDebug("vgId:%d, invalid file %s is removed", REPO_ID(pRepo), pf->aname); } } @@ -811,7 +800,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { fArray = taosArrayInit(1024, sizeof(STfsFile)); if (fArray == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir, + tsdbError("vgId:%d, failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); regfree(®ex); return -1; @@ -819,7 +808,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { tdir = tfsOpendir(REPO_TFS(pRepo), dataDir); if (tdir == NULL) { - tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir, + tsdbError("vgId:%d, failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); taosArrayDestroy(fArray); regfree(®ex); @@ -840,12 +829,12 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { } } else if (code == REG_NOMATCH) { // Not match - tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), pf->aname); + tsdbInfo("vgId:%d, invalid file %s exists, remove it", REPO_ID(pRepo), pf->aname); (void)tfsRemoveFile(pf); continue; } else { // Has other error - tsdbError("vgId:%d failed to restore DFileSet Array while run regexec since %s", REPO_ID(pRepo), strerror(code)); + tsdbError("vgId:%d, failed to restore DFileSet Array while run regexec since %s", REPO_ID(pRepo), strerror(code)); terrno = TAOS_SYSTEM_ERROR(code); tfsClosedir(tdir); taosArrayDestroy(fArray); @@ -876,7 +865,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); if (index >= taosArrayGetSize(fArray)) { - tsdbError("vgId:%d incomplete DFileSet, fid:%d", REPO_ID(pRepo), fset.fid); + tsdbError("vgId:%d, incomplete DFileSet, fid:%d", REPO_ID(pRepo), fset.fid); taosArrayDestroy(fArray); return -1; } @@ -902,14 +891,14 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { fset.fid = tfid; } else { if (tfid != fset.fid) { - tsdbError("vgId:%d incomplete dFileSet, fid:%d", REPO_ID(pRepo), fset.fid); + tsdbError("vgId:%d, incomplete dFileSet, fid:%d", REPO_ID(pRepo), fset.fid); taosArrayDestroy(fArray); return -1; } } if (ttype != ftype) { - tsdbError("vgId:%d incomplete dFileSet, fid:%d", REPO_ID(pRepo), fset.fid); + tsdbError("vgId:%d, incomplete dFileSet, fid:%d", REPO_ID(pRepo), fset.fid); taosArrayDestroy(fArray); return -1; } @@ -918,14 +907,14 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { // if (tsdbOpenDFile(pDFile, O_RDONLY) < 0) { if (tsdbOpenDFile(pDFile, TD_FILE_READ) < 0) { - tsdbError("vgId:%d failed to open DFile %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), + tsdbError("vgId:%d, failed to open DFile %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno)); taosArrayDestroy(fArray); return -1; } if (tsdbLoadDFileHeader(pDFile, &(pDFile->info)) < 0) { - tsdbError("vgId:%d failed to load DFile %s header since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), + tsdbError("vgId:%d, failed to load DFile %s header since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno)); taosArrayDestroy(fArray); return -1; @@ -943,7 +932,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { if (pDFile->info.size != file_size) { int64_t tfsize = pDFile->info.size; pDFile->info.size = file_size; - tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo), + tsdbInfo("vgId:%d, file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tfsize, pDFile->info.size); } } @@ -952,7 +941,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { index++; } - tsdbInfo("vgId:%d FSET %d is restored", REPO_ID(pRepo), fset.fid); + tsdbInfo("vgId:%d, FSET %d is restored", REPO_ID(pRepo), fset.fid); taosArrayPush(pfs->cstatus->df, &fset); } @@ -963,20 +952,13 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { } static int tsdbRestoreCurrent(STsdb *pRepo) { - // // Loop to recover mfile - // if (tsdbRestoreMeta(pRepo) < 0) { - // tsdbError("vgId:%d failed to restore current since %s", REPO_ID(pRepo), tstrerror(terrno)); - // return -1; - // } - - // Loop to recover dfile set if (tsdbRestoreDFileSet(pRepo) < 0) { - tsdbError("vgId:%d failed to restore DFileSet since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbError("vgId:%d, failed to restore DFileSet since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } if (tsdbSaveFSStatus(pRepo, pRepo->fs->cstatus) < 0) { - tsdbError("vgId:%d failed to restore current since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbError("vgId:%d, failed to restore current since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } @@ -1024,11 +1006,11 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired) { if (fset.fid < pRepo->rtn.minFid) { ++*nExpired; } - tsdbDebug("vgId:%d scan DFileSet %d header", REPO_ID(pRepo), fset.fid); + tsdbDebug("vgId:%d, scan DFileSet %d header", REPO_ID(pRepo), fset.fid); // if (tsdbOpenDFileSet(&fset, O_RDWR) < 0) { if (tsdbOpenDFileSet(&fset, TD_FILE_WRITE | TD_FILE_READ) < 0) { - tsdbError("vgId:%d failed to open DFileSet %d since %s, continue", REPO_ID(pRepo), fset.fid, tstrerror(terrno)); + tsdbError("vgId:%d, failed to open DFileSet %d since %s, continue", REPO_ID(pRepo), fset.fid, tstrerror(terrno)); continue; } @@ -1038,17 +1020,44 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired) { if ((tsdbLoadDFileHeader(pDFile, &info) < 0) || pDFile->info.size != info.size || pDFile->info.magic != info.magic) { if (tsdbUpdateDFileHeader(pDFile) < 0) { - tsdbError("vgId:%d failed to update DFile header of %s since %s, continue", REPO_ID(pRepo), + tsdbError("vgId:%d, failed to update DFile header of %s since %s, continue", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno)); } else { - tsdbInfo("vgId:%d DFile header of %s is updated", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); + tsdbInfo("vgId:%d, DFile header of %s is updated", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); TSDB_FILE_FSYNC(pDFile); } } else { - tsdbDebug("vgId:%d DFile header of %s is correct", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); + tsdbDebug("vgId:%d, DFile header of %s is correct", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); } } tsdbCloseDFileSet(&fset); } } + +int tsdbRLockFS(STsdbFS *pFs) { + int code = taosThreadRwlockRdlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +int tsdbWLockFS(STsdbFS *pFs) { + int code = taosThreadRwlockWrlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +int tsdbUnLockFS(STsdbFS *pFs) { + int code = taosThreadRwlockUnlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 04be2a48deb36ea343a55fa66a74a03ad79acce9..a88cb8a353aebe3eec8a565a6f1ebcb04dc5b5db 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -25,7 +25,7 @@ static const char *TSDB_FNAME_SUFFIX[] = { "meta", // TSDB_FILE_META }; -static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char* dname, char *fname); +static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char *dname, char *fname); // static int tsdbRollBackMFile(SMFile *pMFile); static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo); static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo); @@ -181,7 +181,7 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) { tsdbInitDFileEx(&df, pDFile); if (!taosCheckExistFile(TSDB_FILE_FULL_NAME(pDFile))) { - tsdbError("vgId:%d data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo), + tsdbError("vgId:%d, data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); // pRepo->state |= TSDB_STATE_BAD_DATA; TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD); @@ -211,17 +211,17 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) { } tsdbCloseDFile(&df); - tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), + tsdbInfo("vgId:%d, file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), file_size, pDFile->info.size); } else if (pDFile->info.size > file_size) { - tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it", + tsdbError("vgId:%d, data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), file_size, pDFile->info.size); // pRepo->state |= TSDB_STATE_BAD_DATA; TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return 0; } else { - tsdbDebug("vgId:%d file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); + tsdbDebug("vgId:%d, file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); } return 0; @@ -447,4 +447,137 @@ static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, c snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s-ver%" PRIu32, vid, TSDB_FNAME_SUFFIX[ftype], ver); } } +} + +int tsdbOpenDFile(SDFile *pDFile, int flags) { + ASSERT(!TSDB_FILE_OPENED(pDFile)); + + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags); + if (pDFile->pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return 0; +} + +void tsdbCloseDFile(SDFile *pDFile) { + if (TSDB_FILE_OPENED(pDFile)) { + taosCloseFile(&pDFile->pFile); + TSDB_FILE_SET_CLOSED(pDFile); + } +} + +int64_t tsdbSeekDFile(SDFile *pDFile, int64_t offset, int whence) { + // ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence); + if (loffset < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return loffset; +} + +int64_t tsdbWriteDFile(SDFile *pDFile, void *buf, int64_t nbyte) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte); + if (nwrite < nbyte) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nwrite; +} + +void tsdbUpdateDFileMagic(SDFile *pDFile, void *pCksm) { + pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t *)(pCksm), sizeof(TSCKSUM)); +} + +int tsdbAppendDFile(SDFile *pDFile, void *buf, int64_t nbyte, int64_t *offset) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t toffset; + + if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) { + return -1; + } + + ASSERT(pDFile->info.size == toffset); + + if (offset) { + *offset = toffset; + } + + if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) { + return -1; + } + + pDFile->info.size += nbyte; + + return (int)nbyte; +} + +int tsdbRemoveDFile(SDFile *pDFile) { return tfsRemoveFile(TSDB_FILE_F(pDFile)); } + +int64_t tsdbReadDFile(SDFile *pDFile, void *buf, int64_t nbyte) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte); + if (nread < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nread; +} + +int tsdbCopyDFile(SDFile *pSrc, SDFile *pDest) { + if (tfsCopyFile(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + pDest->info = pSrc->info; + return 0; +} + +void tsdbCloseDFileSet(SDFileSet *pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype)); + } +} + +int tsdbOpenDFileSet(SDFileSet *pSet, int flags) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) { + tsdbCloseDFileSet(pSet); + return -1; + } + } + return 0; +} + +void tsdbRemoveDFileSet(SDFileSet *pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype)); + } +} + +int tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) { + tsdbRemoveDFileSet(pDest); + return -1; + } + } + + return 0; +} + +void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey) { + *minKey = fid * days * tsTickPerMin[precision]; + *maxKey = *minKey + days * tsTickPerMin[precision] - 1; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 350e7235413cbca9f0dd0bdb1df0a938b389a5f8..fa392baa163179444ac6bb338c51b5637e60152f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -297,8 +297,8 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo tSkipListPutBatchByIter(pTbData->pData, &blkIter, (iter_next_fn_t)tGetSubmitBlkNext); #ifdef TD_DEBUG_PRINT_ROW - printf("!!! %s:%d table %" PRIi64 " has %d rows in skiplist\n\n", __func__, __LINE__, pTbData->uid, - SL_SIZE(pTbData->pData)); + printf("!!! %s:%d vgId:%d dir:%s table:%" PRIi64 " has %d rows in skiplist\n\n", __func__, __LINE__, + TD_VID(pTsdb->pVnode), pTsdb->dir, pTbData->uid, SL_SIZE(pTbData->pData)); #endif // Set statistics diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c index 94c88a14ffb31a6f80e4bd3ed9d6ab3207d2eb65..cf8557bba3a1917400659bd559cb4dceb561770a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -15,57 +15,34 @@ #include "tsdb.h" -typedef struct SMemData SMemData; -typedef struct SMemSkipList SMemSkipList; -typedef struct SMemSkipListNode SMemSkipListNode; - struct SMemSkipListNode { int8_t level; SMemSkipListNode *forwards[0]; }; -struct SMemSkipList { - uint32_t seed; - int32_t size; - int8_t maxLevel; - int8_t level; - SMemSkipListNode *pHead; - SMemSkipListNode *pTail; -}; - -struct SMemData { - tb_uid_t suid; - tb_uid_t uid; - TSDBKEY minKey; - TSDBKEY maxKey; - SDelOp *delOpHead; - SDelOp *delOpTail; - SMemSkipList sl; -}; +typedef struct { + tb_uid_t uid; + STSchema *pTSchema; +} SSkmInfo; -struct SMemTable { - STsdb *pTsdb; - int32_t nRef; - TSDBKEY minKey; - TSDBKEY maxKey; - int64_t nRows; - SArray *pArray; // SArray -}; +#define SL_MAX_LEVEL 5 #define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2) -#define SL_NODE_HALF_SIZE(l) (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)) #define SL_NODE_FORWARD(n, l) ((n)->forwards[l]) #define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)]) #define SL_NODE_DATA(n) (&SL_NODE_BACKWARD(n, (n)->level)) -#define SL_HEAD_FORWARD(sl, l) SL_NODE_FORWARD((sl)->pHead, l) -#define SL_TAIL_BACKWARD(sl, l) SL_NODE_FORWARD((sl)->pTail, l) +#define SL_MOVE_BACKWARD 0x1 +#define SL_MOVE_FROM_POS 0x2 static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData); static int memDataPCmprFn(const void *p1, const void *p2); static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow); static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl); +static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int64_t version, + SVSubmitBlk *pSubmitBlk); +static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags); // SMemTable ============================================== int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { @@ -82,8 +59,9 @@ int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTable) { pMemTable->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX}; pMemTable->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN}; pMemTable->nRows = 0; - pMemTable->pArray = taosArrayInit(512, sizeof(SMemData *)); - if (pMemTable->pArray == NULL) { + pMemTable->nDelOp = 0; + pMemTable->aMemData = taosArrayInit(512, sizeof(SMemData *)); + if (pMemTable->aMemData == NULL) { taosMemoryFree(pMemTable); code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -98,7 +76,7 @@ _err: } void tsdbMemTableDestroy2(SMemTable *pMemTable) { - taosArrayDestroyEx(pMemTable->pArray, NULL /*TODO*/); + taosArrayDestroyEx(pMemTable->aMemData, NULL /*TODO*/); taosMemoryFree(pMemTable); } @@ -109,6 +87,7 @@ int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmit TSDBROW row = {.version = version}; ASSERT(pMemTable); + ASSERT(pSubmitBlk->nData > 0); { // check if table exists (todo) @@ -116,44 +95,16 @@ int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmit code = tsdbGetOrCreateMemData(pMemTable, pSubmitBlk->suid, pSubmitBlk->uid, &pMemData); if (code) { - tsdbError("vgId:%d failed to create/get table data since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, failed to create/get table data since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); goto _err; } // do insert - int32_t nt; - uint8_t *pt; - int32_t n = 0; - uint8_t *p = pSubmitBlk->pData; - SVBufPool *pPool = pTsdb->pVnode->inUse; - int8_t level; - SMemSkipListNode *pNode; - while (n < pSubmitBlk->nData) { - nt = tGetTSRow(p + n, &row.tsRow); - n += nt; - - ASSERT(n <= pSubmitBlk->nData); - - // build the node - level = tsdbMemSkipListRandLevel(&pMemData->sl); - pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + nt + sizeof(version)); - if (pNode == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - pNode->level = level; - tPutTSDBRow((uint8_t *)SL_NODE_DATA(pNode), &row); - - // put the node (todo) - - // set info - if (tsdbKeyCmprFn(&row, &pMemData->minKey) < 0) pMemData->minKey = *(TSDBKEY *)&row; - if (tsdbKeyCmprFn(&row, &pMemData->maxKey) > 0) pMemData->maxKey = *(TSDBKEY *)&row; + code = tsdbInsertTableDataImpl(pMemTable, pMemData, version, pSubmitBlk); + if (code) { + goto _err; } - if (tsdbKeyCmprFn(&pMemTable->minKey, &pMemData->minKey) < 0) pMemTable->minKey = pMemData->minKey; - if (tsdbKeyCmprFn(&pMemTable->maxKey, &pMemData->maxKey) > 0) pMemTable->maxKey = pMemData->maxKey; - return code; _err: @@ -199,18 +150,106 @@ int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_ui // update the state of pMemTable, pMemData, last and lastrow (todo) } - tsdbDebug("vgId:%d delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64 + pMemTable->nDelOp++; + + tsdbDebug("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64 " since %s", TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code)); return code; _err: - tsdbError("vgId:%d failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64 + tsdbError("vgId:%d, failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64 " since %s", TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code)); return code; } +void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter) { + SMemSkipListNode *pos[SL_MAX_LEVEL]; + + pIter->pMemData = pMemData; + pIter->backward = backward; + pIter->pRow = NULL; + if (pKey == NULL) { + // create from head or tail + if (backward) { + pIter->pNode = SL_NODE_BACKWARD(pMemData->sl.pTail, 0); + } else { + pIter->pNode = SL_NODE_FORWARD(pMemData->sl.pHead, 0); + } + } else { + // create from a key + if (backward) { + memDataMovePosTo(pMemData, pos, pKey, SL_MOVE_BACKWARD); + pIter->pNode = SL_NODE_BACKWARD(pos[0], 0); + } else { + memDataMovePosTo(pMemData, pos, pKey, 0); + pIter->pNode = SL_NODE_FORWARD(pos[0], 0); + } + } +} + +bool tsdbMemDataIterNext(SMemDataIter *pIter) { + SMemSkipListNode *pHead = pIter->pMemData->sl.pHead; + SMemSkipListNode *pTail = pIter->pMemData->sl.pTail; + + pIter->pRow = NULL; + if (pIter->backward) { + ASSERT(pIter->pNode != pTail); + + if (pIter->pNode == pHead) { + return false; + } + + pIter->pNode = SL_NODE_BACKWARD(pIter->pNode, 0); + if (pIter->pNode == pHead) { + return false; + } + } else { + ASSERT(pIter->pNode != pHead); + + if (pIter->pNode == pTail) { + return false; + } + + pIter->pNode = SL_NODE_FORWARD(pIter->pNode, 0); + if (pIter->pNode == pTail) { + return false; + } + } + + return true; +} + +void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow) { + if (pIter->pRow) { + *ppRow = pIter->pRow; + } else { + SMemSkipListNode *pHead = pIter->pMemData->sl.pHead; + SMemSkipListNode *pTail = pIter->pMemData->sl.pTail; + + if (pIter->backward) { + ASSERT(pIter->pNode != pTail); + + if (pIter->pNode == pHead) { + *ppRow = NULL; + } else { + tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row); + *ppRow = &pIter->row; + } + } else { + ASSERT(pIter->pNode != pHead); + + if (pIter->pNode == pTail) { + *ppRow = NULL; + } else { + tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row); + *ppRow = &pIter->row; + } + } + } +} + static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) { int32_t code = 0; int32_t idx = 0; @@ -220,14 +259,14 @@ static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_ui int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel; // get - idx = taosArraySearchIdx(pMemTable->pArray, &pMemDataT, memDataPCmprFn, TD_GE); + idx = taosArraySearchIdx(pMemTable->aMemData, &pMemDataT, memDataPCmprFn, TD_GE); if (idx >= 0) { - pMemData = (SMemData *)taosArrayGet(pMemTable->pArray, idx); + pMemData = (SMemData *)taosArrayGet(pMemTable->aMemData, idx); if (memDataPCmprFn(&pMemDataT, &pMemData) == 0) goto _exit; } // create - pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData) + SL_NODE_HALF_SIZE(maxLevel) * 2); + pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData) + SL_NODE_SIZE(maxLevel) * 2); if (pMemData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -242,15 +281,19 @@ static int32_t tsdbGetOrCreateMemData(SMemTable *pMemTable, tb_uid_t suid, tb_ui pMemData->sl.maxLevel = maxLevel; pMemData->sl.level = 0; pMemData->sl.pHead = (SMemSkipListNode *)&pMemData[1]; - pMemData->sl.pTail = (SMemSkipListNode *)POINTER_SHIFT(pMemData->sl.pHead, SL_NODE_HALF_SIZE(maxLevel)); + pMemData->sl.pTail = (SMemSkipListNode *)POINTER_SHIFT(pMemData->sl.pHead, SL_NODE_SIZE(maxLevel)); + pMemData->sl.pHead->level = maxLevel; + pMemData->sl.pTail->level = maxLevel; for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel; iLevel++) { - SL_HEAD_FORWARD(&pMemData->sl, iLevel) = pMemData->sl.pTail; - SL_TAIL_BACKWARD(&pMemData->sl, iLevel) = pMemData->sl.pHead; + SL_NODE_FORWARD(pMemData->sl.pHead, iLevel) = pMemData->sl.pTail; + SL_NODE_BACKWARD(pMemData->sl.pHead, iLevel) = NULL; + SL_NODE_BACKWARD(pMemData->sl.pTail, iLevel) = pMemData->sl.pHead; + SL_NODE_FORWARD(pMemData->sl.pTail, iLevel) = NULL; } if (idx < 0) idx = 0; - if (taosArrayInsert(pMemTable->pArray, idx, &pMemData) == NULL) { + if (taosArrayInsert(pMemTable->aMemData, idx, &pMemData) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -313,264 +356,180 @@ static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { return level; } -#if 0 //==================================================================================== - -#define SL_MAX_LEVEL 5 - -struct SMemSkipListCurosr { - SMemSkipList *pSl; - SMemSkipListNode *pNodes[SL_MAX_LEVEL]; -}; - -typedef struct { - int64_t version; - uint32_t szRow; - const STSRow *pRow; -} STsdbRow; - -#define HASH_BUCKET(SUID, UID, NBUCKET) (TABS((SUID) + (UID)) % (NBUCKET)) - -#define SL_HEAD_NODE(sl) ((sl)->pHead) -#define SL_TAIL_NODE(sl) ((SMemSkipListNode *)&SL_NODE_FORWARD(SL_HEAD_NODE(sl), (sl)->maxLevel)) -#define SL_HEAD_NODE_FORWARD(n, l) SL_NODE_FORWARD(n, l) -#define SL_TAIL_NODE_BACKWARD(n, l) SL_NODE_FORWARD(n, l) - -static int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl); -static int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pRow); -static int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow); -static int32_t tsdbMemSkipListCursorCreate(int8_t maxLevel, SMemSkipListCurosr **ppSlc); -static void tsdbMemSkipListCursorDestroy(SMemSkipListCurosr *pSlc); -static void tsdbMemSkipListCursorInit(SMemSkipListCurosr *pSlc, SMemSkipList *pSl); -static void tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNode *pNode); -static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t version, TSKEY ts, int32_t flags); -static void tsdbMemSkipListCursorMoveToFirst(SMemSkipListCurosr *pSlc); -static void tsdbMemSkipListCursorMoveToLast(SMemSkipListCurosr *pSlc); -static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc); -static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc); -static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow); - -// SMemTable ======================== -int32_t tsdbInsertData2(SMemTable *pMemTb, int64_t version, const SVSubmitBlk *pSubmitBlk) { - SMemData *pMemData; - STsdb *pTsdb = pMemTb->pTsdb; - SVnode *pVnode = pTsdb->pVnode; - SVBufPool *pPool = pVnode->inUse; - tb_uid_t suid = pSubmitBlk->suid; - tb_uid_t uid = pSubmitBlk->uid; - int32_t iBucket; - - // search SMemData by hash - iBucket = HASH_BUCKET(suid, uid, pMemTb->nBucket); - for (pMemData = pMemTb->pBuckets[iBucket]; pMemData; pMemData = pMemData->pHashNext) { - if (pMemData->suid == suid && pMemData->uid == uid) break; - } +static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags) { + SMemSkipListNode *px; + SMemSkipListNode *pn; + TSDBKEY *pTKey; + int c; + int backward = flags & SL_MOVE_BACKWARD; + int fromPos = flags & SL_MOVE_FROM_POS; - // create pMemData if need - if (pMemData == NULL) { - int8_t maxLevel = pVnode->config.tsdbCfg.slLevel; - int32_t tsize = sizeof(*pMemData) + SL_NODE_HALF_SIZE(maxLevel) * 2; - SMemSkipListNode *pHead, *pTail; - - pMemData = vnodeBufPoolMalloc(pPool, tsize); - if (pMemData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + if (backward) { + px = pMemData->sl.pTail; - pMemData->pHashNext = NULL; - pMemData->suid = suid; - pMemData->uid = uid; - pMemData->minKey = TSKEY_MAX; - pMemData->maxKey = TSKEY_MIN; - pMemData->minVer = -1; - pMemData->maxVer = -1; - pMemData->nRows = 0; - pMemData->sl.seed = taosRand(); - pMemData->sl.maxLevel = maxLevel; - pMemData->sl.level = 0; - pMemData->sl.size = 0; - pHead = SL_HEAD_NODE(&pMemData->sl); - pTail = SL_TAIL_NODE(&pMemData->sl); - pHead->level = maxLevel; - pTail->level = maxLevel; - for (int iLevel = 0; iLevel < maxLevel; iLevel++) { - SL_HEAD_NODE_FORWARD(pHead, iLevel) = pTail; - SL_TAIL_NODE_BACKWARD(pTail, iLevel) = pHead; + for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= pMemData->sl.level; iLevel--) { + pos[iLevel] = px; } - // add to hash - if (pMemTb->nHash >= pMemTb->nBucket) { - // rehash (todo) + if (pMemData->sl.level) { + if (fromPos) px = pos[pMemData->sl.level - 1]; + + for (int8_t iLevel = pMemData->sl.level - 1; iLevel >= 0; iLevel--) { + pn = SL_NODE_BACKWARD(px, iLevel); + while (pn != pMemData->sl.pHead) { + pTKey = (TSDBKEY *)SL_NODE_DATA(pn); + + c = tsdbKeyCmprFn(pTKey, pKey); + if (c <= 0) { + break; + } else { + px = pn; + pn = SL_NODE_BACKWARD(px, iLevel); + } + } + + pos[iLevel] = px; + } } - iBucket = HASH_BUCKET(suid, uid, pMemTb->nBucket); - pMemData->pHashNext = pMemTb->pBuckets[iBucket]; - pMemTb->pBuckets[iBucket] = pMemData; - pMemTb->nHash++; - - // sort organize (todo) - } + } else { + px = pMemData->sl.pHead; - // do insert data to SMemData - SMemSkipListNode *forwards[SL_MAX_LEVEL]; - SMemSkipListNode *pNode; - int32_t iRow; - STsdbRow tRow = {.version = version}; - SEncoder ec = {0}; - SDecoder dc = {0}; - - tDecoderInit(&dc, pSubmitBlk->pData, pSubmitBlk->nData); - tsdbMemSkipListCursorInit(pMemTb->pSlc, &pMemData->sl); - for (iRow = 0;; iRow++) { - if (tDecodeIsEnd(&dc)) break; - - // decode row - if (tDecodeBinary(&dc, (uint8_t **)&tRow.pRow, &tRow.szRow) < 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; + for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= pMemData->sl.level; iLevel--) { + pos[iLevel] = px; } - // move cursor - tsdbMemSkipListCursorMoveTo(pMemTb->pSlc, version, tRow.pRow->ts, 0); - - // encode row - pNode = tsdbMemSkipListNodeCreate(pPool, &pMemData->sl, &tRow); - if (pNode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + if (pMemData->sl.level) { + if (fromPos) px = pos[pMemData->sl.level - 1]; + + for (int8_t iLevel = pMemData->sl.level - 1; iLevel >= 0; iLevel--) { + pn = SL_NODE_FORWARD(px, iLevel); + while (pn != pMemData->sl.pHead) { + pTKey = (TSDBKEY *)SL_NODE_DATA(pn); + + c = tsdbKeyCmprFn(pTKey, pKey); + if (c >= 0) { + break; + } else { + px = pn; + pn = SL_NODE_FORWARD(px, iLevel); + } + } + + pos[iLevel] = px; + } } - - // put the node - tsdbMemSkipListCursorPut(pMemTb->pSlc, pNode); - - // update status - if (tRow.pRow->ts < pMemData->minKey) pMemData->minKey = tRow.pRow->ts; - if (tRow.pRow->ts > pMemData->maxKey) pMemData->maxKey = tRow.pRow->ts; } - tDecoderClear(&dc); - - // update status - if (pMemData->minVer == -1) pMemData->minVer = version; - if (pMemData->maxVer == -1 || pMemData->maxVer < version) pMemData->maxVer = version; - - if (pMemTb->minKey < pMemData->minKey) pMemTb->minKey = pMemData->minKey; - if (pMemTb->maxKey < pMemData->maxKey) pMemTb->maxKey = pMemData->maxKey; - if (pMemTb->minVer == -1) pMemTb->minVer = version; - if (pMemTb->maxVer == -1 || pMemTb->maxVer < version) pMemTb->maxVer = version; - - return 0; } -static FORCE_INLINE int32_t tsdbEncodeRow(SEncoder *pEncoder, const STsdbRow *pRow) { - if (tEncodeI64(pEncoder, pRow->version) < 0) return -1; - if (tEncodeBinary(pEncoder, (const uint8_t *)pRow->pRow, pRow->szRow) < 0) return -1; - return 0; -} - -static FORCE_INLINE int32_t tsdbDecodeRow(SDecoder *pDecoder, STsdbRow *pRow) { - if (tDecodeI64(pDecoder, &pRow->version) < 0) return -1; - if (tDecodeBinary(pDecoder, (uint8_t **)&pRow->pRow, &pRow->szRow) < 0) return -1; - return 0; -} +static int32_t memDataDoPut(SMemTable *pMemTable, SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow, + int8_t forward) { + int32_t code = 0; + int8_t level; + SMemSkipListNode *pNode; + SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; -static int32_t tsdbMemSkipListCursorCreate(int8_t maxLevel, SMemSkipListCurosr **ppSlc) { - *ppSlc = (SMemSkipListCurosr *)taosMemoryCalloc(1, sizeof(**ppSlc) + sizeof(SMemSkipListNode *) * maxLevel); - if (*ppSlc == NULL) { - return -1; + // node + level = tsdbMemSkipListRandLevel(&pMemData->sl); + pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow)); + if (pNode == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pNode->level = level; + for (int8_t iLevel = 0; iLevel < level; iLevel++) { + SL_NODE_FORWARD(pNode, iLevel) = NULL; + SL_NODE_BACKWARD(pNode, iLevel) = NULL; } - return 0; -} - -static void tsdbMemSkipListCursorDestroy(SMemSkipListCurosr *pSlc) { taosMemoryFree(pSlc); } - -static void tsdbMemSkipListCursorInit(SMemSkipListCurosr *pSlc, SMemSkipList *pSl) { - SMemSkipListNode *pHead = SL_HEAD_NODE(pSl); - pSlc->pSl = pSl; - // for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) { - // pSlc->forwards[iLevel] = pHead; - // } -} -static void tsdbMemSkipListCursorPut(SMemSkipListCurosr *pSlc, SMemSkipListNode *pNode) { - SMemSkipList *pSl = pSlc->pSl; - SMemSkipListNode *pNodeNext; + tPutTSDBRow((uint8_t *)SL_NODE_DATA(pNode), pRow); + // put for (int8_t iLevel = 0; iLevel < pNode->level; iLevel++) { - // todo + SMemSkipListNode *px = pos[iLevel]; - ASSERT(0); - } + if (forward) { + SMemSkipListNode *pNext = SL_NODE_FORWARD(px, iLevel); - if (pSl->level < pNode->level) { - pSl->level = pNode->level; - } + SL_NODE_FORWARD(pNode, iLevel) = pNext; + SL_NODE_BACKWARD(pNode, iLevel) = px; - pSl->size += 1; -} + SL_NODE_BACKWARD(pNext, iLevel) = pNode; + SL_NODE_FORWARD(px, iLevel) = pNode; + } else { + SMemSkipListNode *pPrev = SL_NODE_BACKWARD(px, iLevel); -static int32_t tsdbMemSkipListCursorMoveTo(SMemSkipListCurosr *pSlc, int64_t version, TSKEY ts, int32_t flags) { - SMemSkipListNode **pForwards = NULL; - SMemSkipList *pSl = pSlc->pSl; - int8_t maxLevel = pSl->maxLevel; - SMemSkipListNode *pHead = SL_HEAD_NODE(pSl); - SMemSkipListNode *pTail = SL_TAIL_NODE(pSl); + SL_NODE_FORWARD(pNode, iLevel) = px; + SL_NODE_BACKWARD(pNode, iLevel) = pPrev; - if (pSl->size == 0) { - for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) { - pForwards[iLevel] = pHead; + SL_NODE_FORWARD(pPrev, iLevel) = pNode; + SL_NODE_BACKWARD(px, iLevel) = pNode; } } - return 0; + pMemData->sl.size++; + if (pMemData->sl.level < pNode->level) { + pMemData->sl.level = pNode->level; + } + +_exit: + return code; } -static void tsdbMemSkipListCursorMoveToFirst(SMemSkipListCurosr *pSlc) { - SMemSkipList *pSl = pSlc->pSl; - SMemSkipListNode *pHead = SL_HEAD_NODE(pSl); +static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, SMemData *pMemData, int64_t version, + SVSubmitBlk *pSubmitBlk) { + int32_t code = 0; + int32_t n = 0; + uint8_t *p = pSubmitBlk->pData; + int32_t nRow = 0; + TSDBROW row = {.version = version}; - for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) { - pSlc->pNodes[iLevel] = pHead; - } + SMemSkipListNode *pos[SL_MAX_LEVEL]; - tsdbMemSkipListCursorMoveToNext(pSlc); -} + ASSERT(pSubmitBlk->nData); -static void tsdbMemSkipListCursorMoveToLast(SMemSkipListCurosr *pSlc) { - SMemSkipList *pSl = pSlc->pSl; - SMemSkipListNode *pTail = SL_TAIL_NODE(pSl); + // backward put first data + n += tGetTSRow(p + n, &row.tsRow); + ASSERT(n <= pSubmitBlk->nData); - for (int8_t iLevel = 0; iLevel < pSl->maxLevel; iLevel++) { - pSlc->pNodes[iLevel] = pTail; + memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_BACKWARD); + code = memDataDoPut(pMemTable, pMemData, pos, &row, 0); + if (code) { + goto _exit; } + nRow++; - tsdbMemSkipListCursorMoveToPrev(pSlc); -} + if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemData->minKey) < 0) { + pMemData->minKey = *(TSDBKEY *)&row; + } + if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemTable->minKey) < 0) { + pMemTable->minKey = *(TSDBKEY *)&row; + } -static int32_t tsdbMemSkipListCursorMoveToNext(SMemSkipListCurosr *pSlc) { - // TODO - return 0; -} + // forward put rest + for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel; iLevel++) { + pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel); + } + while (n < pSubmitBlk->nData) { + n += tGetTSRow(p + n, &row.tsRow); + ASSERT(n <= pSubmitBlk->nData); -static int32_t tsdbMemSkipListCursorMoveToPrev(SMemSkipListCurosr *pSlc) { - // TODO - return 0; -} + memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_FROM_POS); + code = memDataDoPut(pMemTable, pMemData, pos, &row, 1); + if (code) { + goto _exit; + } -static SMemSkipListNode *tsdbMemSkipListNodeCreate(SVBufPool *pPool, SMemSkipList *pSl, const STsdbRow *pTRow) { - int32_t tsize; - int32_t ret; - int8_t level = tsdbMemSkipListRandLevel(pSl); - SMemSkipListNode *pNode = NULL; - SEncoder ec = {0}; + nRow++; + } - tEncodeSize(tsdbEncodeRow, pTRow, tsize, ret); - pNode = vnodeBufPoolMalloc(pPool, tsize + SL_NODE_SIZE(level)); - if (pNode) { - pNode->level = level; - tEncoderInit(&ec, (uint8_t *)SL_NODE_DATA(pNode), tsize); - tsdbEncodeRow(&ec, pTRow); - tEncoderClear(&ec); + if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemData->maxKey) > 0) { + pMemData->maxKey = *(TSDBKEY *)&row; } + if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemTable->maxKey) > 0) { + pMemTable->maxKey = *(TSDBKEY *)&row; + } + pMemTable->nRows += nRow; - return pNode; +_exit: + return code; } -#endif \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index fa54c811ffc158339fda4b34cad47ba7c4f2fdac..943263e1a3c65ebf980b353e6a1b69ba52868a22 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -74,7 +74,7 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee goto _err; } - tsdbDebug("vgId:%d tsdb is opened for %s, days:%d, keep:%d,%d,%d", TD_VID(pVnode), pTsdb->path, pTsdb->keepCfg.days, + tsdbDebug("vgId:%d, tsdb is opened for %s, days:%d, keep:%d,%d,%d", TD_VID(pVnode), pTsdb->path, pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2); *ppTsdb = pTsdb; @@ -99,7 +99,7 @@ int tsdbClose(STsdb **pTsdb) { int tsdbLockRepo(STsdb *pTsdb) { int code = taosThreadMutexLock(&pTsdb->mutex); if (code != 0) { - tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); + tsdbError("vgId:%d, failed to lock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); return -1; } @@ -112,7 +112,7 @@ int tsdbUnlockRepo(STsdb *pTsdb) { pTsdb->repoLocked = false; int code = taosThreadMutexUnlock(&pTsdb->mutex); if (code != 0) { - tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); + tsdbError("vgId:%d, failed to unlock tsdb since %s", REPO_ID(pTsdb), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 61daa0c9b345837a3406ea7d2947faf61d72fe06..5fdb052ee5662f9d672591389b76c22cb0b0d2f4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -20,10 +20,10 @@ #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) -#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ - ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ - .numOfCols = (_block)->numOfCols, \ - .rows = (_block)->numOfRows, \ +#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ + ((SDataBlockInfo){.window = {.skey = (_block)->minKey.ts, .ekey = (_block)->maxKey.ts}, \ + .numOfCols = (_block)->numOfCols, \ + .rows = (_block)->numOfRows, \ .uid = (_checkInfo)->tableId}) enum { @@ -352,15 +352,15 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, STsdbReadHandle* pReadHandle, } if (level == TSDB_RETENTION_L0) { - tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, + tsdbDebug("vgId:%d, read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L0); return VND_RSMA0(pVnode); } else if (level == TSDB_RETENTION_L1) { - tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, + tsdbDebug("vgId:%d, read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L1); return VND_RSMA1(pVnode); } else { - tsdbDebug("vgId:%d read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, + tsdbDebug("vgId:%d, read handle %p rsma level %d is selected to query", TD_VID(pVnode), pReadHandle, TSDB_RETENTION_L2); return VND_RSMA2(pVnode); } @@ -1105,12 +1105,12 @@ static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY s if (numOfBlocks == 1) break; - if (skey > pBlock[midSlot].keyLast) { + if (skey > pBlock[midSlot].maxKey.ts) { if (numOfBlocks == 2) break; - if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break; + if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].minKey.ts)) break; firstSlot = midSlot + 1; - } else if (skey < pBlock[midSlot].keyFirst) { - if ((order == TSDB_ORDER_ASC) && (skey > pBlock[midSlot - 1].keyLast)) break; + } else if (skey < pBlock[midSlot].minKey.ts) { + if ((order == TSDB_ORDER_ASC) && (skey > pBlock[midSlot - 1].maxKey.ts)) break; lastSlot = midSlot - 1; } else { break; // got the slot @@ -1177,12 +1177,12 @@ static int32_t loadBlockInfo(STsdbReadHandle* pTsdbReadHandle, int32_t index, in int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); int32_t end = start; - if (s > pCompInfo->blocks[start].keyLast) { + if (s > pCompInfo->blocks[start].maxKey.ts) { return 0; } // todo speedup the procedure of located end block - while (end < (int32_t)compIndex->numOfBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { + while (end < (int32_t)compIndex->numOfBlocks && (pCompInfo->blocks[end].minKey.ts <= e)) { end += 1; } @@ -1275,7 +1275,7 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl pBlock->numOfRows = pCols->numOfRows; // Convert from TKEY to TSKEY for primary timestamp column if current block has timestamp before 1970-01-01T00:00:00Z - if (pBlock->keyFirst < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pBlock->minKey.ts < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { int64_t* src = pCols->cols[0].pData; for (int32_t i = 0; i < pBlock->numOfRows; ++i) { src[i] = tdGetKey(src[i]); @@ -1287,7 +1287,7 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl tsdbDebug("%p load file block into buffer, index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%" PRId64 " us, %s", - pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, + pTsdbReadHandle, slotIndex, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->numOfRows, elapsedTime, pTsdbReadHandle->idStr); return TSDB_CODE_SUCCESS; @@ -1295,7 +1295,8 @@ _error: pBlock->numOfRows = 0; tsdbError("%p error occurs in loading file block, index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, %s", - pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pTsdbReadHandle->idStr); + pTsdbReadHandle, slotIndex, pBlock->minKey.ts, pBlock->maxKey.ts, pBlock->numOfRows, + pTsdbReadHandle->idStr); return terrno; } @@ -1423,7 +1424,7 @@ static int32_t loadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBloc if (asc) { // query ended in/started from current block - if (pTsdbReadHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { + if (pTsdbReadHandle->window.ekey < pBlock->maxKey.ts || pCheckInfo->lastKey > pBlock->minKey.ts) { if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { *exists = false; return code; @@ -1432,35 +1433,35 @@ static int32_t loadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBloc SDataCols* pTSCol = pTsdbReadHandle->rhelper.pDCols[0]; assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows); - if (pCheckInfo->lastKey > pBlock->keyFirst) { + if (pCheckInfo->lastKey > pBlock->minKey.ts) { cur->pos = binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order); } else { cur->pos = 0; } - assert(pCheckInfo->lastKey <= pBlock->keyLast); + assert(pCheckInfo->lastKey <= pBlock->maxKey.ts); doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { // the whole block is loaded in to buffer cur->pos = asc ? 0 : (pBlock->numOfRows - 1); code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlock, pCheckInfo); } } else { // desc order, query ended in current block - if (pTsdbReadHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) { + if (pTsdbReadHandle->window.ekey > pBlock->minKey.ts || pCheckInfo->lastKey < pBlock->maxKey.ts) { if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { *exists = false; return code; } SDataCols* pTsCol = pTsdbReadHandle->rhelper.pDCols[0]; - if (pCheckInfo->lastKey < pBlock->keyLast) { + if (pCheckInfo->lastKey < pBlock->maxKey.ts) { cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order); } else { cur->pos = pBlock->numOfRows - 1; } - assert(pCheckInfo->lastKey >= pBlock->keyFirst); + assert(pCheckInfo->lastKey >= pBlock->minKey.ts); doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { cur->pos = asc ? 0 : (pBlock->numOfRows - 1); @@ -1661,7 +1662,11 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa } #ifdef TD_DEBUG_PRINT_ROW - tdSRowPrint(row1, pSchema1, __func__); + char flags[70] = {0}; + STsdb* pTsdb = pTsdbReadHandle->rhelper.pRepo; + snprintf(flags, 70, "%s:%d vgId:%d dir:%s row1%s=NULL,row2%s=NULL", __func__, __LINE__, TD_VID(pTsdb->pVnode), + pTsdb->dir, row1 ? "!" : "", row2 ? "!" : ""); + tdSRowPrint(row1, pSchema1, flags); #endif if (isRow1DataRow) { @@ -1981,8 +1986,8 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf cur->pos >= 0 && cur->pos < pBlock->numOfRows); // Even Multi-Version supported, the records with duplicated TSKEY would be merged inside of tsdbLoadData interface. TSKEY* tsArray = pCols->cols[0].pData; - assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst && - tsArray[pBlock->numOfRows - 1] == pBlock->keyLast); + assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->minKey.ts && + tsArray[pBlock->numOfRows - 1] == pBlock->maxKey.ts); bool ascScan = ASCENDING_TRAVERSE(pTsdbReadHandle->order); int32_t step = ascScan ? 1 : -1; @@ -2829,6 +2834,12 @@ void* tsdbGetIdx(SMeta* pMeta) { } return metaGetIdx(pMeta); } +void* tsdbGetIvtIdx(SMeta* pMeta) { + if (pMeta == NULL) { + return NULL; + } + return metaGetIvtIdx(pMeta); +} int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) { SMCtbCursor* pCur = metaOpenCtbCursor(pMeta, uid); @@ -3555,7 +3566,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SColumnDat return TSDB_CODE_SUCCESS; } - tsdbDebug("vgId:%d succeed to load block statis part for uid %" PRIu64, REPO_ID(pHandle->pTsdb), + tsdbDebug("vgId:%d, succeed to load block statis part for uid %" PRIu64, REPO_ID(pHandle->pTsdb), TSDB_READ_TABLE_UID(&pHandle->rhelper)); int16_t* colIds = pHandle->suppInfo.defaultLoadColumn->pData; @@ -3576,8 +3587,8 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SColumnDat assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_ID); pPrimaryColStatis->numOfNull = 0; - pPrimaryColStatis->min = pBlockInfo->compBlock->keyFirst; - pPrimaryColStatis->max = pBlockInfo->compBlock->keyLast; + pPrimaryColStatis->min = pBlockInfo->compBlock->minKey.ts; + pPrimaryColStatis->max = pBlockInfo->compBlock->maxKey.ts; pHandle->suppInfo.plist[0] = &pHandle->suppInfo.pstatis[0]; // update the number of NULL data rows diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index d51521c41c954821163d17a1eddf4a4ddee7f5ad..a26436156dbb2e8ba1e3d969ce7698d657c5159e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -87,7 +87,7 @@ int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) { TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh)); // if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) { if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), TD_FILE_READ) < 0) { - tsdbError("vgId:%d failed to open file set %d since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FSET_FID(pSet), + tsdbError("vgId:%d, failed to open file set %d since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FSET_FID(pSet), tstrerror(terrno)); return -1; } @@ -107,7 +107,7 @@ int tsdbLoadBlockIdx(SReadH *pReadh) { if (pHeadf->info.offset <= 0) return 0; if (tsdbSeekDFile(pHeadf, pHeadf->info.offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to load SBlockIdx part while seek file %s since %s, offset:%u len :%u", + tsdbError("vgId:%d, failed to load SBlockIdx part while seek file %s since %s, offset:%u len :%u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pHeadf->info.offset, pHeadf->info.len); return -1; @@ -117,7 +117,7 @@ int tsdbLoadBlockIdx(SReadH *pReadh) { int64_t nread = tsdbReadDFile(pHeadf, TSDB_READ_BUF(pReadh), pHeadf->info.len); if (nread < 0) { - tsdbError("vgId:%d failed to load SBlockIdx part while read file %s since %s, offset:%u len :%u", + tsdbError("vgId:%d, failed to load SBlockIdx part while read file %s since %s, offset:%u len :%u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pHeadf->info.offset, pHeadf->info.len); return -1; @@ -125,14 +125,14 @@ int tsdbLoadBlockIdx(SReadH *pReadh) { if (nread < pHeadf->info.len) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d SBlockIdx part in file %s is corrupted, offset:%u expected bytes:%u read bytes: %" PRId64, + tsdbError("vgId:%d, SBlockIdx part in file %s is corrupted, offset:%u expected bytes:%u read bytes: %" PRId64, TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pHeadf->info.offset, pHeadf->info.len, nread); return -1; } if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), pHeadf->info.len)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d SBlockIdx part in file %s is corrupted since wrong checksum, offset:%u len :%u", + tsdbError("vgId:%d, SBlockIdx part in file %s is corrupted since wrong checksum, offset:%u len :%u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pHeadf->info.offset, pHeadf->info.len); return -1; } @@ -209,7 +209,7 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { SBlockIdx *pBlkIdx = pReadh->pBlkIdx; if (tsdbSeekDFile(pHeadf, pBlkIdx->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to load SBlockInfo part while seek file %s since %s, offset:%u len:%u", + tsdbError("vgId:%d, failed to load SBlockInfo part while seek file %s since %s, offset:%u len:%u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len); return -1; } @@ -218,21 +218,21 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { int64_t nread = tsdbReadDFile(pHeadf, (void *)(pReadh->pBlkInfo), pBlkIdx->len); if (nread < 0) { - tsdbError("vgId:%d failed to load SBlockInfo part while read file %s since %s, offset:%u len :%u", + tsdbError("vgId:%d, failed to load SBlockInfo part while read file %s since %s, offset:%u len :%u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len); return -1; } if (nread < pBlkIdx->len) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d SBlockInfo part in file %s is corrupted, offset:%u expected bytes:%u read bytes:%" PRId64, + tsdbError("vgId:%d, SBlockInfo part in file %s is corrupted, offset:%u expected bytes:%u read bytes:%" PRId64, TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len, nread); return -1; } if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkInfo), pBlkIdx->len)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d SBlockInfo part in file %s is corrupted since wrong checksum, offset:%u len :%u", + tsdbError("vgId:%d, SBlockInfo part in file %s is corrupted since wrong checksum, offset:%u len :%u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len); return -1; } @@ -339,8 +339,8 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { } ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows); - ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); - ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); + ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->minKey.ts); + ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->maxKey.ts); return 0; } @@ -457,8 +457,8 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, } ASSERT(pReadh->pDCols[0]->numOfRows <= pBlock->numOfRows); - ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); - ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); + ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->minKey.ts); + ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->maxKey.ts); return 0; } @@ -467,7 +467,7 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { ASSERT(pBlock->numOfSubBlocks <= 1); if (!pBlock->aggrStat) { - tsdbDebug("vgId:%d no need to load block statis part for uid %" PRIu64 " since not exist", REPO_ID(pReadh->pRepo), + tsdbDebug("vgId:%d, no need to load block statis part for uid %" PRIu64 " since not exist", REPO_ID(pReadh->pRepo), TSDB_READ_TABLE_UID(pReadh)); return TSDB_STATIS_NONE; } @@ -475,7 +475,7 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { SDFile *pDFileAggr = pBlock->last ? TSDB_READ_SMAL_FILE(pReadh) : TSDB_READ_SMAD_FILE(pReadh); if (tsdbSeekDFile(pDFileAggr, pBlock->aggrOffset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to load block statis part for uid %" PRIu64 " while seek file %s to offset %" PRIu64 + tsdbError("vgId:%d, failed to load block statis part for uid %" PRIu64 " while seek file %s to offset %" PRIu64 " since %s", TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, tstrerror(terrno)); @@ -487,7 +487,7 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { int64_t nreadAggr = tsdbReadDFile(pDFileAggr, (void *)(pReadh->pAggrBlkData), sizeAggr); if (nreadAggr < 0) { - tsdbError("vgId:%d failed to load block statis part for uid %" PRIu64 + tsdbError("vgId:%d, failed to load block statis part for uid %" PRIu64 " while read file %s since %s, offset:%" PRIu64 " len :%" PRIzu, TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), tstrerror(terrno), (uint64_t)pBlock->aggrOffset, sizeAggr); @@ -496,7 +496,7 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { if (nreadAggr < sizeAggr) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block statis part for uid %" PRIu64 " in file %s is corrupted, offset:%" PRIu64 + tsdbError("vgId:%d, block statis part for uid %" PRIu64 " in file %s is corrupted, offset:%" PRIu64 " expected bytes:%" PRIzu " read bytes: %" PRId64, TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr, nreadAggr); @@ -505,7 +505,7 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pAggrBlkData), (uint32_t)sizeAggr)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block statis part for uid %" PRIu64 + tsdbError("vgId:%d, block statis part for uid %" PRIu64 "in file %s is corrupted since wrong checksum, offset:%" PRIu64 " len :%" PRIzu, TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr); @@ -518,7 +518,7 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) { ASSERT(pBlock->numOfSubBlocks <= 1); SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to load block head part while seek file %s to offset %" PRId64 " since %s", + tsdbError("vgId:%d, failed to load block head part while seek file %s to offset %" PRId64 " since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno)); return -1; } @@ -528,14 +528,14 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) { int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size); if (nread < 0) { - tsdbError("vgId:%d failed to load block head part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu, + tsdbError("vgId:%d, failed to load block head part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu, TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, size); return -1; } if (nread < size) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block head part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu + tsdbError("vgId:%d, block head part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu " read bytes: %" PRId64, TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size, nread); return -1; @@ -543,7 +543,7 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) { if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkData), (uint32_t)size)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block head part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu, + tsdbError("vgId:%d, block head part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu, TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size); return -1; } @@ -559,7 +559,7 @@ int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx) { tlen += taosEncodeFixedU8(buf, pIdx->hasLast); tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks); tlen += taosEncodeFixedU64(buf, pIdx->uid); - tlen += taosEncodeFixedU64(buf, pIdx->maxKey); + tlen += taosEncodeFixedU64(buf, pIdx->maxKey.ts); return tlen; } @@ -579,7 +579,7 @@ void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) { if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; pIdx->uid = (int64_t)value; if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; - pIdx->maxKey = (TSKEY)value; + pIdx->maxKey.ts = (TSKEY)value; return buf; } @@ -671,14 +671,14 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat SBlockData *pBlockData = (SBlockData *)TSDB_READ_BUF(pReadh); if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to load block data part while seek file %s to offset %" PRId64 " since %s", + tsdbError("vgId:%d, failed to load block data part while seek file %s to offset %" PRId64 " since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno)); return -1; } int64_t nread = tsdbReadDFile(pDFile, TSDB_READ_BUF(pReadh), pBlock->len); if (nread < 0) { - tsdbError("vgId:%d failed to load block data part while read file %s since %s, offset:%" PRId64 " len :%d", + tsdbError("vgId:%d, failed to load block data part while read file %s since %s, offset:%" PRId64 " len :%d", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, pBlock->len); return -1; @@ -686,7 +686,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat if (nread < pBlock->len) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block data part in file %s is corrupted, offset:%" PRId64 + tsdbError("vgId:%d, block data part in file %s is corrupted, offset:%" PRId64 " expected bytes:%d read bytes: %" PRId64, TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, pBlock->len, nread); return -1; @@ -695,7 +695,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat int32_t tsize = (int32_t)tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer); if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block head part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d", + tsdbError("vgId:%d, block head part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tsize); return -1; } @@ -726,7 +726,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat if (dcol != 0) { pBlockCol = &(pBlockData->cols[ccol]); tcolId = pBlockCol->colId; - toffset = tsdbGetBlockColOffset(pBlockCol); + toffset = pBlockCol->offset; tlen = pBlockCol->len; pDataCol->bitmap = pBlockCol->blen > 0 ? 1 : 0; } else { @@ -750,7 +750,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat pBlockCol ? pBlockCol->blen : 0, pBlock->algorithm, pBlock->numOfRows, tLenBitmap, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh), (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { - tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u", + tsdbError("vgId:%d, file %s is broken at column %d block offset %" PRId64 " column offset %u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset); return -1; } @@ -942,24 +942,24 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1; - int64_t offset = pBlock->offset + tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer) + - tsdbGetBlockColOffset(pBlockCol); + int64_t offset = + pBlock->offset + tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer) + pBlockCol->offset; if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to load block column data while seek file %s to offset %" PRId64 " since %s", + tsdbError("vgId:%d, failed to load block column data while seek file %s to offset %" PRId64 " since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno)); return -1; } int64_t nread = tsdbReadDFile(pDFile, TSDB_READ_BUF(pReadh), pBlockCol->len); if (nread < 0) { - tsdbError("vgId:%d failed to load block column data while read file %s since %s, offset:%" PRId64 " len :%d", + tsdbError("vgId:%d, failed to load block column data while read file %s since %s, offset:%" PRId64 " len :%d", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), offset, pBlockCol->len); return -1; } if (nread < pBlockCol->len) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block column data in file %s is corrupted, offset:%" PRId64 " expected bytes:%d" PRIzu + tsdbError("vgId:%d, block column data in file %s is corrupted, offset:%" PRId64 " expected bytes:%d" PRIzu " read bytes: %" PRId64, TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, pBlockCol->len, nread); return -1; @@ -968,7 +968,7 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlockCol->blen, pBlock->algorithm, pBlock->numOfRows, tLenBitmap, pCfg->maxRows, pReadh->pCBuf, (int32_t)taosTSizeof(pReadh->pCBuf)) < 0) { - tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), + tsdbError("vgId:%d, file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), pBlockCol->colId, offset); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index aab4da26a37119c6e0044849b1494ede2f33e552..6faf6bd1679c36dd0c9fdc0bed538f74cafc13cd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -28,7 +28,7 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * // scan and convert if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) { if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) { - tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pTsdb), tstrerror(terrno)); + tsdbError("vgId:%d, failed to insert data since %s", REPO_ID(pTsdb), tstrerror(terrno)); } return -1; } @@ -59,7 +59,7 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, STable *pTable, STSRow * TSKEY now) { TSKEY rowKey = TD_ROW_KEY(row); if (rowKey < minKey || rowKey > maxKey) { - tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 + tsdbError("vgId:%d, table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 " maxKey %" PRId64 " row key %" PRId64, REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey, rowKey); @@ -75,7 +75,7 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *ro TSKEY now) { TSKEY rowKey = TD_ROW_KEY(row); if (rowKey < minKey || rowKey > maxKey) { - tsdbError("vgId:%d table uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 + tsdbError("vgId:%d, table uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 " maxKey %" PRId64 " row key %" PRId64, REPO_ID(pTsdb), uid, now, minKey, maxKey, rowKey); terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; @@ -115,7 +115,7 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { #if 0 if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) { - tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid, + tsdbError("vgId:%d, failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid, pBlock->tid); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; return -1; @@ -123,14 +123,14 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { STable *pTable = pMeta->tables[pBlock->tid]; if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) { - tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid, + tsdbError("vgId:%d, failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid, pBlock->tid); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; return -1; } if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable)); + tsdbError("vgId:%d, invalid action trying to insert a super table %s", REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable)); terrno = TSDB_CODE_TDB_INVALID_ACTION; return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 9122913cda69d05889e1f575a5da4b61ef4a03a9..9ca4dd6efb981acdf2ff271635b7e146052c7a40 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -30,7 +30,7 @@ int vnodeOpenBufPool(SVnode *pVnode, int64_t size) { // create pool ret = vnodeBufPoolCreate(size, &pPool); if (ret < 0) { - vError("vgId:%d failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); vnodeCloseBufPool(pVnode); return -1; } @@ -40,7 +40,7 @@ int vnodeOpenBufPool(SVnode *pVnode, int64_t size) { pVnode->pPool = pPool; } - vDebug("vgId:%d vnode buffer pool is opened, pool size: %" PRId64, TD_VID(pVnode), size); + vDebug("vgId:%d, vnode buffer pool is opened, pool size: %" PRId64, TD_VID(pVnode), size); return 0; } @@ -53,7 +53,7 @@ int vnodeCloseBufPool(SVnode *pVnode) { vnodeBufPoolDestroy(pPool); } - vDebug("vgId:%d vnode buffer pool is closed", TD_VID(pVnode)); + vDebug("vgId:%d, vnode buffer pool is closed", TD_VID(pVnode)); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index a0db3cfe2da57c9a9b8ebc7e743ff69554763dd6..3715866bb88f3ae030d5f65c7ce938e69120f466 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -42,29 +42,29 @@ int vnodeBegin(SVnode *pVnode) { // begin meta if (metaBegin(pVnode->pMeta) < 0) { - vError("vgId:%d failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } // begin tsdb if (pVnode->pSma) { if (tsdbBegin(VND_RSMA0(pVnode)) < 0) { - vError("vgId:%d failed to begin rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to begin rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbBegin(VND_RSMA1(pVnode)) < 0) { - vError("vgId:%d failed to begin rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to begin rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbBegin(VND_RSMA2(pVnode)) < 0) { - vError("vgId:%d failed to begin rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to begin rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } } else { if (tsdbBegin(pVnode->pTsdb) < 0) { - vError("vgId:%d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } } @@ -110,7 +110,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { // free info binary taosMemoryFree(data); - vInfo("vgId:%d vnode info is saved, fname: %s", pInfo->config.vgId, fname); + vInfo("vgId:%d, vnode info is saved, fname: %s", pInfo->config.vgId, fname); return 0; @@ -132,7 +132,7 @@ int vnodeCommitInfo(const char *dir, const SVnodeInfo *pInfo) { return -1; } - vInfo("vgId:%d vnode info is committed", pInfo->config.vgId); + vInfo("vgId:%d, vnode info is committed", pInfo->config.vgId); return 0; } @@ -210,7 +210,7 @@ int vnodeCommit(SVnode *pVnode) { SVnodeInfo info = {0}; char dir[TSDB_FILENAME_LEN]; - vInfo("vgId:%d start to commit, version: %" PRId64, TD_VID(pVnode), pVnode->state.applied); + vInfo("vgId:%d, start to commit, version: %" PRId64, TD_VID(pVnode), pVnode->state.applied); pVnode->onCommit = pVnode->inUse; pVnode->inUse = NULL; @@ -268,7 +268,7 @@ int vnodeCommit(SVnode *pVnode) { pVnode->pPool = pVnode->onCommit; pVnode->onCommit = NULL; - vInfo("vgId:%d commit over", TD_VID(pVnode)); + vInfo("vgId:%d, commit over", TD_VID(pVnode)); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 7a26c16e4ba0d4137e44262569a457d87b1a5adf..a85b8306165326bc07f643718e9b67201e668de6 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -23,13 +23,13 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { // check config if (vnodeCheckCfg(pCfg) < 0) { - vError("vgId:%d failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d, failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); return -1; } // create vnode env if (tfsMkdir(pTfs, path) < 0) { - vError("vgId:%d failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d, failed to create vnode since: %s", pCfg->vgId, tstrerror(terrno)); return -1; } @@ -39,11 +39,11 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { info.state.applied = -1; if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) { - vError("vgId:%d failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno)); + vError("vgId:%d, failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno)); return -1; } - vInfo("vgId:%d vnode is created", pCfg->vgId); + vInfo("vgId:%d, vnode is created", pCfg->vgId); return 0; } @@ -70,7 +70,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { pVnode = (SVnode *)taosMemoryCalloc(1, sizeof(*pVnode) + strlen(path) + 1); if (pVnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - vError("vgId:%d failed to open vnode since %s", info.config.vgId, tstrerror(terrno)); + vError("vgId:%d, failed to open vnode since %s", info.config.vgId, tstrerror(terrno)); return NULL; } @@ -86,25 +86,25 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // open buffer pool if (vnodeOpenBufPool(pVnode, pVnode->config.isHeap ? 0 : pVnode->config.szBuf / 3) < 0) { - vError("vgId:%d failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open meta if (metaOpen(pVnode, &pVnode->pMeta) < 0) { - vError("vgId:%d failed to open vnode meta since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to open vnode meta since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open tsdb if (!VND_IS_RSMA(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, NULL) < 0) { - vError("vgId:%d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open sma if (smaOpen(pVnode)) { - vError("vgId:%d failed to open vnode sma since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to open vnode sma since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } @@ -113,7 +113,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { taosRealPath(tdir, NULL, sizeof(tdir)); pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg)); if (pVnode->pWal == NULL) { - vError("vgId:%d failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } @@ -122,27 +122,27 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { taosRealPath(tdir, NULL, sizeof(tdir)); pVnode->pTq = tqOpen(tdir, pVnode, pVnode->pWal); if (pVnode->pTq == NULL) { - vError("vgId:%d failed to open vnode tq since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to open vnode tq since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; } // open query if (vnodeQueryOpen(pVnode)) { - vError("vgId:%d failed to open vnode query since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to open vnode query since %s", TD_VID(pVnode), tstrerror(terrno)); terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } // vnode begin if (vnodeBegin(pVnode) < 0) { - vError("vgId:%d failed to begin since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to begin since %s", TD_VID(pVnode), tstrerror(terrno)); terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } // open sync if (vnodeSyncOpen(pVnode, dir)) { - vError("vgId:%d failed to open sync since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to open sync since %s", TD_VID(pVnode), tstrerror(terrno)); terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index ab879c5dbbbff95a663747d8c1ce09f987f5dc8f..4bcd5fc095a5e71f5e87e7ee374993f557bf3000 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -15,16 +15,18 @@ #include "vnd.h" -static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); -static int vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp); -static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) { + int32_t code = 0; SDecoder dc = {0}; switch (pMsg->msgType) { @@ -89,22 +91,22 @@ int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) { } break; case TDMT_VND_ALTER_REPLICA: { - vnodeSyncAlter(pVnode, pMsg); + code = vnodeSyncAlter(pVnode, pMsg); } break; default: break; } - return 0; + return code; } -int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) { - void *ptr = NULL; - void *pReq; - int len; - int ret; +int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp) { + void *ptr = NULL; + void *pReq; + int32_t len; + int32_t ret; - vTrace("vgId:%d start to process write request %s, version %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + vTrace("vgId:%d, start to process write request %s, version %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); pVnode->state.applied = version; @@ -157,6 +159,9 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg pMsg->contLen - sizeof(SMsgHead)) < 0) { } } break; + case TDMT_VND_ALTER_CONFIRM: + vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp); + break; case TDMT_VND_ALTER_CONFIG: break; default: @@ -164,16 +169,16 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg break; } - vDebug("vgId:%d process %s request success, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); + vTrace("vgId:%d, process %s request success, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { - vError("vgId:%d failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } // commit if need if (vnodeShouldCommit(pVnode)) { - vInfo("vgId:%d commit at version %" PRId64, TD_VID(pVnode), version); + vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); // commit current change vnodeCommit(pVnode); @@ -184,12 +189,20 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg return 0; _err: - vDebug("vgId:%d process %s request failed since %s, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + vError("vgId:%d, process %s request failed since %s, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), tstrerror(terrno), version); return -1; } -int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { +int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { + if (TDMT_VND_QUERY != pMsg->msgType) { + return 0; + } + + return qWorkerPreprocessQueryMsg(pVnode->pQuery, pMsg); +} + +int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in vnode query queue is processing"); SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; switch (pMsg->msgType) { @@ -203,7 +216,7 @@ int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { } } -int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { +int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("message in fetch queue is processing"); char *msgstr = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); @@ -258,7 +271,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { pMetaRsp->precision = pVnode->config.tsdbCfg.precision; } -int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int32_t ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; if (syncEnvIsStart()) { @@ -278,56 +291,56 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SRpcMsg *pRpcMsg = pMsg; - if (pRpcMsg->msgType == TDMT_VND_SYNC_TIMEOUT) { + if (pRpcMsg->msgType == TDMT_SYNC_TIMEOUT) { SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); ret = syncNodeOnTimeoutCb(pSyncNode, pSyncMsg); syncTimeoutDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) { + } else if (pRpcMsg->msgType == TDMT_SYNC_PING) { SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); ret = syncNodeOnPingCb(pSyncNode, pSyncMsg); syncPingDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) { + } else if (pRpcMsg->msgType == TDMT_SYNC_PING_REPLY) { SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); ret = syncNodeOnPingReplyCb(pSyncNode, pSyncMsg); syncPingReplyDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) { + } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); ret = syncNodeOnClientRequestCb(pSyncNode, pSyncMsg); syncClientRequestDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) { + } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); ret = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); syncRequestVoteDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) { + } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); ret = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); syncRequestVoteReplyDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) { + } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); ret = syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg); syncAppendEntriesDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) { + } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -348,7 +361,7 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { return ret; } -static int vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateStbReq req = {0}; SDecoder coder; @@ -380,9 +393,9 @@ _err: return -1; } -static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SDecoder decoder = {0}; - int rcode = 0; + int32_t rcode = 0; SVCreateTbBatchReq req = {0}; SVCreateTbReq *pCreateReq; SVCreateTbBatchRsp rsp = {0}; @@ -413,7 +426,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pReq, } // loop to create table - for (int iReq = 0; iReq < req.nReqs; iReq++) { + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; // validate hash @@ -468,7 +481,7 @@ _exit: return rcode; } -static int vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateStbReq req = {0}; SDecoder dc = {0}; @@ -497,9 +510,9 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, return 0; } -static int vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVDropStbReq req = {0}; - int rcode = TSDB_CODE_SUCCESS; + int32_t rcode = TSDB_CODE_SUCCESS; SDecoder decoder = {0}; pRsp->msgType = TDMT_VND_CREATE_STB_RSP; @@ -526,12 +539,12 @@ _exit: return 0; } -static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVAlterTbReq vAlterTbReq = {0}; SVAlterTbRsp vAlterTbRsp = {0}; SDecoder dc = {0}; - int rcode = 0; - int ret; + int32_t rcode = 0; + int32_t ret; SEncoder ec = {0}; STableMetaRsp vMetaRsp = {0}; @@ -573,12 +586,12 @@ _exit: return 0; } -static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVDropTbBatchReq req = {0}; SVDropTbBatchRsp rsp = {0}; SDecoder decoder = {0}; SEncoder encoder = {0}; - int ret; + int32_t ret; SArray *tbUids = NULL; pRsp->msgType = TDMT_VND_DROP_TABLE_RSP; @@ -600,7 +613,7 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in rsp.pArray = taosArrayInit(req.nReqs, sizeof(SVDropTbRsp)); if (tbUids == NULL || rsp.pArray == NULL) goto _exit; - for (int iReq = 0; iReq < req.nReqs; iReq++) { + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { SVDropTbReq *pDropTbReq = req.pReqs + iReq; SVDropTbRsp dropTbRsp = {0}; @@ -632,7 +645,8 @@ _exit: return 0; } -static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSubmitMsgIter *msgIter, const char *tags) { +static int32_t vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSubmitMsgIter *msgIter, + const char *tags) { SSubmitBlkIter blkIter = {0}; STSchema *pSchema = NULL; tb_uid_t suid = 0; @@ -666,7 +680,7 @@ static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSub return TSDB_CODE_SUCCESS; } -static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char *tags) { +static int32_t vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char *tags) { ASSERT(pMsg != NULL); SSubmitMsgIter msgIter = {0}; SMeta *pMeta = pVnode->pMeta; @@ -683,7 +697,7 @@ static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char return 0; } -static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; SSubmitRsp submitRsp = {0}; SSubmitMsgIter msgIter = {0}; @@ -714,8 +728,8 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in goto _exit; } - submitRsp.pArray = taosArrayInit(pSubmitReq->numOfBlocks, sizeof(SSubmitBlkRsp)); - newTbUids = taosArrayInit(pSubmitReq->numOfBlocks, sizeof(int64_t)); + submitRsp.pArray = taosArrayInit(msgIter.numOfBlocks, sizeof(SSubmitBlkRsp)); + newTbUids = taosArrayInit(msgIter.numOfBlocks, sizeof(int64_t)); if (!submitRsp.pArray) { pRsp->code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -799,7 +813,7 @@ _exit: return 0; } -static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { +static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SVCreateTSmaReq req = {0}; SDecoder coder; @@ -828,13 +842,13 @@ static int vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq } tDecoderClear(&coder); - vDebug("vgId:%d success to create tsma %s:%" PRIi64 " version %" PRIi64 " for table %" PRIi64, TD_VID(pVnode), + vDebug("vgId:%d, success to create tsma %s:%" PRIi64 " version %" PRIi64 " for table %" PRIi64, TD_VID(pVnode), req.indexName, req.indexUid, version, req.tableUid); return 0; _err: tDecoderClear(&coder); - vError("vgId:%d failed to create tsma %s:%" PRIi64 " version %" PRIi64 "for table %" PRIi64 " since %s", + vError("vgId:%d, failed to create tsma %s:%" PRIi64 " version %" PRIi64 "for table %" PRIi64 " since %s", TD_VID(pVnode), req.indexName, req.indexUid, version, req.tableUid, terrstr(terrno)); return -1; } @@ -850,3 +864,13 @@ _err: int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen) { return vnodeProcessCreateTSmaReq(pVnode, 1, pCont, contLen, NULL); } + +static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + vInfo("vgId:%d, alter replica confim msg is processed", TD_VID(pVnode)); + pRsp->msgType = TDMT_VND_ALTER_CONFIRM_RSP; + pRsp->code = TSDB_CODE_SUCCESS; + pRsp->pCont = NULL; + pRsp->contLen = 0; + + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 8792fbbb0c60d52a06b7af28ca040a1a0f22eb9b..8f98e9cec57b810adcbacb989b6179ac7fc9b2ad 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -50,13 +50,11 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { return 0; } -void vnodeSyncAlter(SVnode *pVnode, SRpcMsg *pMsg) { +int32_t vnodeSyncAlter(SVnode *pVnode, SRpcMsg *pMsg) { SAlterVnodeReq req = {0}; if (tDeserializeSAlterVnodeReq((char *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead), &req) != 0) { terrno = TSDB_CODE_INVALID_MSG; - vError("vgId:%d, failed to alter replica since %s", TD_VID(pVnode), terrstr()); - SRpcMsg rsp = {.info = pMsg->info, .code = terrno}; - tmsgSendRsp(&rsp); + return TSDB_CODE_INVALID_MSG; } vInfo("vgId:%d, start to alter vnode replica to %d", TD_VID(pVnode), req.replica); @@ -68,11 +66,15 @@ void vnodeSyncAlter(SVnode *pVnode, SRpcMsg *pMsg) { vInfo("vgId:%d, replica:%d %s:%u", TD_VID(pVnode), r, pNode->nodeFqdn, pNode->nodePort); } - if (syncReconfig(pVnode->sync, &cfg) != 0) { - vError("vgId:%d, failed to propose sync reconfig since %s", TD_VID(pVnode), terrstr()); - SRpcMsg rsp = {.info = pMsg->info, .code = terrno}; + int32_t code = syncReconfig(pVnode->sync, &cfg); + if (code == TAOS_SYNC_PROPOSE_SUCCESS) { + // todo refactor + SRpcMsg rsp = {.info = pMsg->info, .code = 0}; tmsgSendRsp(&rsp); + return TSDB_CODE_ACTION_IN_PROGRESS; } + + return code; } void vnodeSyncStart(SVnode *pVnode) { diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 4d2741f751066b62ebb463b4ff8d2930f057a318..3b8c94e413ee866441f5d7514e13986d31fc4137 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -147,8 +147,8 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { // resource release taosMemoryFreeClear(pSW); - tdDestroyTSma(&tSma); - tdDestroyTSmaWrapper(&dstTSmaWrapper); + tDestroyTSma(&tSma); + tDestroyTSmaWrapper(&dstTSmaWrapper); } #endif @@ -218,7 +218,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { printf("tagsFilter1 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1); EXPECT_EQ(qSmaCfg->tableUid, tSma.tableUid); - tdDestroyTSma(qSmaCfg); + tDestroyTSma(qSmaCfg); taosMemoryFreeClear(qSmaCfg); qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid2, true); @@ -229,7 +229,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { printf("tagsFilter2 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2); EXPECT_EQ(qSmaCfg->interval, tSma.interval); - tdDestroyTSma(qSmaCfg); + tDestroyTSma(qSmaCfg); taosMemoryFreeClear(qSmaCfg); // get index name by table uid @@ -265,7 +265,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { EXPECT_EQ((pSW->tSma + 1)->indexUid, indexUid2); EXPECT_EQ((pSW->tSma + 1)->tableUid, tbUid); - tdDestroyTSmaWrapper(pSW); + tDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); // get all sma table uids @@ -282,7 +282,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { metaRemoveSmaFromDb(pMeta, indexUid1); metaRemoveSmaFromDb(pMeta, indexUid2); - tdDestroyTSma(&tSma); + tDestroyTSma(&tSma); metaClose(pMeta); } #endif @@ -576,7 +576,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { taosArrayDestroy(pDataBlocks); // release meta - tdDestroyTSma(&tSma); + tDestroyTSma(&tSma); tfsClose(pTsdb->pTfs); tsdbClose(pTsdb); metaClose(pMeta); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 239d719fa80eb3b4fbf9b075e165dc50051b3e52..9219a382e4865312fd8b546a0a9c3ea106fe76a2 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -52,6 +52,7 @@ enum { CTG_OP_UPDATE_VGROUP = 0, CTG_OP_UPDATE_TB_META, CTG_OP_DROP_DB_CACHE, + CTG_OP_DROP_DB_VGROUP, CTG_OP_DROP_STB_META, CTG_OP_DROP_TB_META, CTG_OP_UPDATE_USER, @@ -266,26 +267,32 @@ typedef struct SCtgUpdateTblMsg { STableMetaOutput* output; } SCtgUpdateTblMsg; -typedef struct SCtgRemoveDBMsg { +typedef struct SCtgDropDBMsg { SCatalog* pCtg; char dbFName[TSDB_DB_FNAME_LEN]; uint64_t dbId; -} SCtgRemoveDBMsg; +} SCtgDropDBMsg; -typedef struct SCtgRemoveStbMsg { +typedef struct SCtgDropDbVgroupMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; +} SCtgDropDbVgroupMsg; + + +typedef struct SCtgDropStbMetaMsg { SCatalog* pCtg; char dbFName[TSDB_DB_FNAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; uint64_t dbId; uint64_t suid; -} SCtgRemoveStbMsg; +} SCtgDropStbMetaMsg; -typedef struct SCtgRemoveTblMsg { +typedef struct SCtgDropTblMetaMsg { SCatalog* pCtg; char dbFName[TSDB_DB_FNAME_LEN]; char tbName[TSDB_TABLE_NAME_LEN]; uint64_t dbId; -} SCtgRemoveTblMsg; +} SCtgDropTblMetaMsg; typedef struct SCtgUpdateUserMsg { SCatalog* pCtg; @@ -451,6 +458,7 @@ int32_t ctgGetTbMetaFromCache(CTG_PARAMS, SCtgTbMetaCtx* ctx, STableMeta** pTabl int32_t ctgOpUpdateVgroup(SCtgCacheOperation *action); int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *action); int32_t ctgOpDropDbCache(SCtgCacheOperation *action); +int32_t ctgOpDropDbVgroup(SCtgCacheOperation *action); int32_t ctgOpDropStbMeta(SCtgCacheOperation *action); int32_t ctgOpDropTbMeta(SCtgCacheOperation *action); int32_t ctgOpUpdateUser(SCtgCacheOperation *action); @@ -464,6 +472,7 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName); int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass); int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId); +int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char *dbFName, bool syncReq); int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq); int32_t ctgDropTbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq); int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq); @@ -490,7 +499,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(CTG_PARAMS, char *dbFName, char* tbName, STabl int32_t ctgGetTbMetaFromMnode(CTG_PARAMS, const SName* pTableName, STableMetaOutput* out, SCtgTask* pTask); int32_t ctgGetTbMetaFromVnode(CTG_PARAMS, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTask* pTask); -int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param); +int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int32_t* taskNum); int32_t ctgLaunchJob(SCtgJob *pJob); int32_t ctgMakeAsyncRes(SCtgJob *pJob); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 7e0efe22dbfa2dd61a5c662c1103a1b98f579da8..040156eca20f4a1d9f6f8c24a8a4eafd9a256988 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1028,16 +1028,21 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmt CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - int32_t code = 0; + int32_t code = 0, taskNum = 0; SCtgJob *pJob = NULL; - CTG_ERR_JRET(ctgInitJob(CTG_PARAMS_LIST(), &pJob, reqId, pReq, fp, param)); + CTG_ERR_JRET(ctgInitJob(CTG_PARAMS_LIST(), &pJob, reqId, pReq, fp, param, &taskNum)); + if (taskNum <= 0) { + SMetaData* pMetaData = taosMemoryCalloc(1, sizeof(SMetaData)); + fp(pMetaData, param, TSDB_CODE_SUCCESS); + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + } CTG_ERR_JRET(ctgLaunchJob(pJob)); - *jobId = pJob->refId; + // NOTE: here the assignment of jobId is invalid, may over-write the true scheduler created query job. +// *jobId = pJob->refId; _return: - if (pJob) { taosReleaseRef(gCtgMgmt.jobPool, pJob->refId); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index eb84bf00a444fb6bc57652ee32abdf44035a0426..6998d8c778ad16af9bd8e30671315cb876109812 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -44,7 +44,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, task.type, name->tname); + qDebug("QID:0x%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, task.type, name->tname); return TSDB_CODE_SUCCESS; } @@ -143,7 +143,7 @@ int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, task.type, name->tname); + qDebug("QID:0x%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, task.type, name->tname); return TSDB_CODE_SUCCESS; } @@ -233,7 +233,7 @@ int32_t ctgInitGetUserTask(SCtgJob *pJob, int32_t taskIdx, SUserAuthInfo *user) } -int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param) { +int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int32_t* taskNum) { int32_t code = 0; int32_t tbMetaNum = (int32_t)taosArrayGetSize(pReq->pTableMeta); int32_t dbVgNum = (int32_t)taosArrayGetSize(pReq->pDbVgroup); @@ -245,15 +245,15 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* int32_t userNum = (int32_t)taosArrayGetSize(pReq->pUser); int32_t dbInfoNum = (int32_t)taosArrayGetSize(pReq->pDbInfo); - int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dbCfgNum + indexNum + userNum + dbInfoNum; - if (taskNum <= 0) { - ctgError("empty input for job, taskNum:%d", taskNum); - CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + *taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dbCfgNum + indexNum + userNum + dbInfoNum; + if (*taskNum <= 0) { + ctgDebug("Empty input for job, no need to retrieve meta, reqId:0x%" PRIx64, reqId); + return TSDB_CODE_SUCCESS; } - + *job = taosMemoryCalloc(1, sizeof(SCtgJob)); if (NULL == *job) { - ctgError("calloc %d failed", (int32_t)sizeof(SCtgJob)); + ctgError("failed to calloc, size:%d, reqId:0x%" PRIx64, (int32_t)sizeof(SCtgJob), reqId); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -275,52 +275,58 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* pJob->indexNum = indexNum; pJob->userNum = userNum; pJob->dbInfoNum = dbInfoNum; - - pJob->pTasks = taosArrayInit(taskNum, sizeof(SCtgTask)); - + + pJob->pTasks = taosArrayInit(*taskNum, sizeof(SCtgTask)); + if (NULL == pJob->pTasks) { - ctgError("taosArrayInit %d tasks failed", taskNum); + ctgError("taosArrayInit %d tasks failed", *taskNum); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } int32_t taskIdx = 0; for (int32_t i = 0; i < dbVgNum; ++i) { - char *dbFName = taosArrayGet(pReq->pDbVgroup, i); + char* dbFName = taosArrayGet(pReq->pDbVgroup, i); + if (pReq->forceUpdate) { + ctgDropDbVgroupEnqueue(pCtg, dbFName, true); + } CTG_ERR_JRET(ctgInitGetDbVgTask(pJob, taskIdx++, dbFName)); } for (int32_t i = 0; i < dbCfgNum; ++i) { - char *dbFName = taosArrayGet(pReq->pDbCfg, i); + char* dbFName = taosArrayGet(pReq->pDbCfg, i); CTG_ERR_JRET(ctgInitGetDbCfgTask(pJob, taskIdx++, dbFName)); } for (int32_t i = 0; i < dbInfoNum; ++i) { - char *dbFName = taosArrayGet(pReq->pDbInfo, i); + char* dbFName = taosArrayGet(pReq->pDbInfo, i); CTG_ERR_JRET(ctgInitGetDbInfoTask(pJob, taskIdx++, dbFName)); } for (int32_t i = 0; i < tbMetaNum; ++i) { - SName *name = taosArrayGet(pReq->pTableMeta, i); + SName* name = taosArrayGet(pReq->pTableMeta, i); + if (pReq->forceUpdate) { + catalogRemoveTableMeta(pCtg, name); + } CTG_ERR_JRET(ctgInitGetTbMetaTask(pJob, taskIdx++, name)); } for (int32_t i = 0; i < tbHashNum; ++i) { - SName *name = taosArrayGet(pReq->pTableHash, i); + SName* name = taosArrayGet(pReq->pTableHash, i); CTG_ERR_JRET(ctgInitGetTbHashTask(pJob, taskIdx++, name)); } for (int32_t i = 0; i < indexNum; ++i) { - char *indexName = taosArrayGet(pReq->pIndex, i); + char* indexName = taosArrayGet(pReq->pIndex, i); CTG_ERR_JRET(ctgInitGetIndexTask(pJob, taskIdx++, indexName)); } for (int32_t i = 0; i < udfNum; ++i) { - char *udfName = taosArrayGet(pReq->pUdf, i); + char* udfName = taosArrayGet(pReq->pUdf, i); CTG_ERR_JRET(ctgInitGetUdfTask(pJob, taskIdx++, udfName)); } for (int32_t i = 0; i < userNum; ++i) { - SUserAuthInfo *user = taosArrayGet(pReq->pUser, i); + SUserAuthInfo* user = taosArrayGet(pReq->pUser, i); CTG_ERR_JRET(ctgInitGetUserTask(pJob, taskIdx++, user)); } @@ -336,14 +342,12 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* taosAcquireRef(gCtgMgmt.jobPool, pJob->refId); - qDebug("QID:%" PRIx64 ", job %" PRIx64 " initialized, task num %d", pJob->queryId, pJob->refId, taskNum); - + qDebug("QID:%" PRIx64 ", job %" PRIx64 " initialized, task num %d", pJob->queryId, pJob->refId, *taskNum); return TSDB_CODE_SUCCESS; -_return: +_return: taosMemoryFreeClear(*job); - CTG_RET(code); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 0f1344c3432b2540c2daa33de33c2f8c570658f0..8332c7b068225fe63469973063e3ff703d9ccfe0 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -35,6 +35,11 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = { "drop DB", ctgOpDropDbCache }, + { + CTG_OP_DROP_DB_VGROUP, + "drop DBVgroup", + ctgOpDropDbVgroup + }, { CTG_OP_DROP_STB_META, "drop stbMeta", @@ -563,9 +568,9 @@ int32_t ctgEnqueue(SCatalog* pCtg, SCtgCacheOperation *operation) { int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) { int32_t code = 0; SCtgCacheOperation action= {.opId = CTG_OP_DROP_DB_CACHE}; - SCtgRemoveDBMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveDBMsg)); + SCtgDropDBMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDBMsg)); if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveDBMsg)); + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropDBMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -590,13 +595,43 @@ _return: CTG_RET(code); } +int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char *dbFName, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation action= {.opId = CTG_OP_DROP_DB_VGROUP, .syncOp = syncOp}; + SCtgDropDbVgroupMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDbVgroupMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropDbVgroupMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + char *p = strchr(dbFName, '.'); + if (p && CTG_IS_SYS_DBNAME(p + 1)) { + dbFName = p + 1; + } + + msg->pCtg = pCtg; + strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + + action.data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(action.data); + CTG_RET(code); +} + + int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncOp) { int32_t code = 0; SCtgCacheOperation action= {.opId = CTG_OP_DROP_STB_META, .syncOp = syncOp}; - SCtgRemoveStbMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveStbMsg)); + SCtgDropStbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropStbMetaMsg)); if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropStbMetaMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -623,9 +658,9 @@ _return: int32_t ctgDropTbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncOp) { int32_t code = 0; SCtgCacheOperation action= {.opId = CTG_OP_DROP_TB_META, .syncOp = syncOp}; - SCtgRemoveTblMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveTblMsg)); + SCtgDropTblMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTblMetaMsg)); if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg)); + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTblMetaMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1281,7 +1316,7 @@ _return: int32_t ctgOpDropDbCache(SCtgCacheOperation *operation) { int32_t code = 0; - SCtgRemoveDBMsg *msg = operation->data; + SCtgDropDBMsg *msg = operation->data; SCatalog* pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; @@ -1304,6 +1339,33 @@ _return: CTG_RET(code); } +int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgDropDbVgroupMsg *msg = operation->data; + SCatalog* pCtg = msg->pCtg; + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + goto _return; + } + + CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); + + ctgFreeVgInfo(dbCache->vgInfo); + dbCache->vgInfo = NULL; + + ctgDebug("db vgInfo removed, dbFName:%s", msg->dbFName); + + ctgWReleaseVgInfo(dbCache); + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { int32_t code = 0; @@ -1353,7 +1415,7 @@ _return: int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { int32_t code = 0; - SCtgRemoveStbMsg *msg = operation->data; + SCtgDropStbMetaMsg *msg = operation->data; SCatalog* pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; @@ -1399,7 +1461,7 @@ _return: int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { int32_t code = 0; - SCtgRemoveTblMsg *msg = operation->data; + SCtgDropTblMetaMsg *msg = operation->data; SCatalog* pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index fdab50db0f65fd67d16d6f5b134f847dc0f882bc..42baa530ce69a7b5c37ee48a8b51cd4969f23b58 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -132,7 +132,22 @@ void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { } } -int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId) { + +/* +prepare SQL: +create database db1; +use db1; +create stable st1 (ts timestamp, f1 int) tags(t1 int); +create table tb1 using st1 tags(1); +insert into tb1 values (now, 1); +create qnode on dnode 1; +create user user1 pass "abc"; +create database db2; +grant write on db2.* to user1; +create function udf1 as '/tmp/libudf1.so' outputtype int; +create aggregate function udf2 as '/tmp/libudf2.so' outputtype int; +*/ +int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, uint64_t reqId, bool forceUpdate) { int32_t code = 0; SCatalogReq req = {0}; req.pTableMeta = taosArrayInit(2, sizeof(SName)); @@ -144,6 +159,7 @@ int32_t ctgdLaunchAsyncCall(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps req.pIndex = NULL;//taosArrayInit(2, TSDB_INDEX_FNAME_LEN); req.pUser = taosArrayInit(2, sizeof(SUserAuthInfo)); req.qNodeRequired = true; + req.forceUpdate = forceUpdate; SName name = {0}; char dbFName[TSDB_DB_FNAME_LEN] = {0}; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 831b7017b2632a3e52e3050c08b2c29ffa463eeb..98ae5e2fb167548edf744a334a8303e5f2eece25 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -153,7 +153,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo pPhysiChildren = pSortNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: { SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; pPhysiChildren = pIntNode->window.node.pChildren; break; @@ -164,12 +164,12 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo break; } case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: { - SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*) pNode; + SStateWinodwPhysiNode *pStateNode = (SStateWinodwPhysiNode *)pNode; pPhysiChildren = pStateNode->window.node.pChildren; break; } case QUERY_NODE_PHYSICAL_PLAN_PARTITION: { - SPartitionPhysiNode* partitionPhysiNode = (SPartitionPhysiNode*) pNode; + SPartitionPhysiNode *partitionPhysiNode = (SPartitionPhysiNode *)pNode; pPhysiChildren = partitionPhysiNode->node.pChildren; break; } @@ -397,7 +397,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i int32_t nodeNum = taosArrayGetSize(pResNode->pExecInfo); for (int32_t i = 0; i < nodeNum; ++i) { - SExplainExecInfo * execInfo = taosArrayGet(pResNode->pExecInfo, i); + SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, i); STableScanAnalyzeInfo *pScanInfo = (STableScanAnalyzeInfo *)execInfo->verboseInfo; EXPLAIN_ROW_APPEND("total_blocks=%d", pScanInfo->totalBlocks); @@ -429,7 +429,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, + pTblScanNode->scanRange.ekey); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); @@ -641,7 +642,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); } - SDataBlockDescNode* pDescNode = pSortNode->node.pOutputDataBlockDesc; + SDataBlockDescNode *pDescNode = pSortNode->node.pOutputDataBlockDesc; EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, nodesGetOutputNumFromSlotList(pDescNode->pSlots)); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pDescNode->totalRowSize); @@ -667,7 +668,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i int32_t nodeNum = taosArrayGetSize(pResNode->pExecInfo); SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, 0); - SSortExecInfo * pExecInfo = (SSortExecInfo *)execInfo->verboseInfo; + SSortExecInfo *pExecInfo = (SSortExecInfo *)execInfo->verboseInfo; EXPLAIN_ROW_APPEND("%s", pExecInfo->sortMethod == SORT_QSORT_T ? "quicksort" : "merge sort"); if (pExecInfo->sortBuffer > 1024 * 1024) { EXPLAIN_ROW_APPEND(" Buffers:%.2f Mb", pExecInfo->sortBuffer / (1024 * 1024.0)); @@ -701,7 +702,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: { SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); @@ -784,7 +785,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: { SStateWinodwPhysiNode *pStateNode = (SStateWinodwPhysiNode *)pNode; - EXPLAIN_ROW_NEW(level, EXPLAIN_STATE_WINDOW_FORMAT, nodesGetNameFromColumnNode(((STargetNode*)pStateNode->pStateKey)->pExpr)); + EXPLAIN_ROW_NEW(level, EXPLAIN_STATE_WINDOW_FORMAT, + nodesGetNameFromColumnNode(((STargetNode *)pStateNode->pStateKey)->pExpr)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); @@ -823,7 +825,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i case QUERY_NODE_PHYSICAL_PLAN_PARTITION: { SPartitionPhysiNode *pPartNode = (SPartitionPhysiNode *)pNode; - SNode* p = nodesListGetNode(pPartNode->pPartitionKeys, 0); + SNode *p = nodesListGetNode(pPartNode->pPartitionKeys, 0); EXPLAIN_ROW_NEW(level, EXPLAIN_PARITION_FORMAT, nodesGetNameFromColumnNode(p)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 63c398618f38446260124978a803b2a63c6f0688..b8975854c9446eab43cd4a7d8c3ccb6e38b93016 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -75,15 +75,15 @@ typedef struct SResultRowInfo { int32_t size; // number of result set int32_t capacity; // max capacity SResultRowPosition cur; + SList* openWindow; } SResultRowInfo; struct SqlFunctionCtx; -size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); +size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size); void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); -int32_t numOfClosedResultRows(SResultRowInfo* pResultRowInfo); void closeAllResultRows(SResultRowInfo* pResultRowInfo); void initResultRow(SResultRow *pResultRow); @@ -92,15 +92,6 @@ bool isResultRowClosed(SResultRow* pResultRow); struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, const int32_t* offset); -static FORCE_INLINE SResultRow *getResultRow(SDiskbasedBuf* pBuf, SResultRowInfo *pResultRowInfo, int32_t slot) { - ASSERT(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); - SResultRowPosition* pos = &pResultRowInfo->pPosition[slot]; - - SFilePage* bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId); - SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset); - return pRow; -} - static FORCE_INLINE SResultRow *getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) { SFilePage* bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId); SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset); diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index f2f0bc20551336b8bfa14414697b9b5299ea6a49..88f308710ec5022c1ad1b83a38bfa0e4dd4e53a6 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -27,6 +27,10 @@ typedef struct { int32_t bytes; } SGroupKeys, SStateKeys; +int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList); +uint64_t calcGroupId(char* pData, int32_t len); +void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex); +int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals); #ifdef __cplusplus } #endif diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 8139e71f63dfa2a58be128fbc2ced05ab81466df..7d597217ee939d43259ca234ca99174e99629320 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -303,13 +303,17 @@ typedef struct SScanInfo { int32_t numOfDesc; } SScanInfo; +typedef struct SSampleExecInfo { + double sampleRatio; // data block sample ratio, 1 by default + uint32_t seed; // random seed value +} SSampleExecInfo; + typedef struct STableScanInfo { void* dataReader; SReadHandle readHandle; SFileBlockLoadRecorder readRecorder; int64_t numOfRows; - int64_t elapsedTime; // int32_t prevGroupId; // previous table group id SScanInfo scanInfo; int32_t scanTimes; @@ -330,9 +334,15 @@ typedef struct STableScanInfo { SQueryTableDataCond cond; int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t dataBlockLoadFlag; - double sampleRatio; // data block sample ratio, 1 by default SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded. + SArray* pGroupCols; + SArray* pGroupColVals; // current group column values, SArray + char* keyBuf; // group by keys for hash + int32_t groupKeyLen; // total group by column width + SHashObj* pGroupSet; // quick locate the window object for each result + + SSampleExecInfo sample; // sample execution info int32_t curTWinIdx; } STableScanInfo; @@ -360,16 +370,18 @@ typedef struct SCatchSupporter { } SCatchSupporter; typedef struct SStreamAggSupporter { - SArray* pResultRows; // SResultWindowInfo + SArray* pResultRows; int32_t keySize; char* pKeyBuf; // window key buffer SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + SArray* pScanWindow; } SStreamAggSupporter; typedef struct SessionWindowSupporter { SStreamAggSupporter* pStreamAggSup; int64_t gap; + uint8_t parentType; } SessionWindowSupporter; typedef struct SStreamBlockScanInfo { @@ -400,6 +412,7 @@ typedef struct SStreamBlockScanInfo { SArray* childIds; SessionWindowSupporter sessionSup; bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. + int32_t scanWinIndex; } SStreamBlockScanInfo; typedef struct SSysTableScanInfo { @@ -454,13 +467,14 @@ typedef struct SIntervalAggOperatorInfo { STimeWindow win; // query time range bool timeWindowInterpo; // interpolation needed or not char** pRow; // previous row/tuple of already processed datablock + SArray* pInterpCols; // interpolation columns STableQueryInfo* pCurrent; // current tableQueryInfo struct int32_t order; // current SSDataBlock scan order EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] SArray* pUpdatedWindow; // updated time window due to the input data block from the downstream operator. STimeWindowAggSupp twAggSup; - struct SFillInfo* pFillInfo; // fill info bool invertible; + SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. } SIntervalAggOperatorInfo; typedef struct SStreamFinalIntervalOperatorInfo { @@ -496,7 +510,7 @@ typedef struct SProjectOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode SOptrBasicInfo binfo; SAggSupporter aggSup; - + SNode* pFilterNode; // filter info, which is push down by optimizer SSDataBlock* existDataBlock; SArray* pPseudoColInfo; SLimit limit; @@ -586,6 +600,11 @@ typedef struct SResultWindowInfo { bool isClosed; } SResultWindowInfo; +typedef struct SStateWindowInfo { + SResultWindowInfo winInfo; + SStateKeys stateKey; +} SStateWindowInfo; + typedef struct SStreamSessionAggOperatorInfo { SOptrBasicInfo binfo; SStreamAggSupporter streamAggSup; @@ -599,7 +618,7 @@ typedef struct SStreamSessionAggOperatorInfo { SSDataBlock* pDelRes; SHashObj* pStDeleted; void* pDelIterator; - SArray* pChildren; // cache for children's result; + SArray* pChildren; // cache for children's result; final stream operator } SStreamSessionAggOperatorInfo; typedef struct STimeSliceOperatorInfo { @@ -623,6 +642,22 @@ typedef struct SStateWindowOperatorInfo { // bool reptScan; } SStateWindowOperatorInfo; +typedef struct SStreamStateAggOperatorInfo { + SOptrBasicInfo binfo; + SStreamAggSupporter streamAggSup; + SGroupResInfo groupResInfo; + int32_t primaryTsIndex; // primary timestamp slot id + int32_t order; // current SSDataBlock scan order + STimeWindowAggSupp twAggSup; + SColumn stateCol; // start row index + SqlFunctionCtx* pDummyCtx; // for combine + SSDataBlock* pDelRes; + SHashObj* pSeDeleted; + void* pDelIterator; + SArray* pScanWindow; + SArray* pChildren; // cache for children's result; +} SStreamStateAggOperatorInfo; + typedef struct SSortedMergeOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode SOptrBasicInfo binfo; @@ -691,7 +726,7 @@ void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWin int32_t setGroupResultOutputBuf(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, int16_t bytes, int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, SAggSupporter* pAggSup); void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput); -int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, +int32_t setDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList); void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, STimeWindow* win); @@ -699,7 +734,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scan int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); void doSetOperatorCompleted(SOperatorInfo* pOperator); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchInfo); +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset); void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols); void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); @@ -726,20 +761,21 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR char* pData, int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup); -SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SArray* groupKyes, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SLimit* pLimit, SLimit* pSlimit, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SLimit* pLimit, SLimit* pSlimit, SNode* pCondition, SExecTaskInfo* pTaskInfo); SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols, SArray* pIndexMap, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName, - SNode* pCondition, SEpSet epset, SArray* colList, - SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); @@ -757,9 +793,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, - SArray* pTableIdList, STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, - STimeWindowAggSupp* pTwSup); +SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, SArray* pTableIdList, + STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, @@ -775,11 +810,14 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, SSDataBlock* pResBlock, SArray* pColMatchInfo, STableListInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, + SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); + #if 0 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); #endif @@ -789,8 +827,6 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol); -void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); - bool isTaskKilled(SExecTaskInfo* pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); @@ -825,16 +861,15 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result); int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length); -STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, - SInterval* pInterval, int32_t precision, STimeWindow* win); -int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, - int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, - int32_t order); +STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, + int32_t precision, STimeWindow* win); +int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, + __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey); -SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize); -SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap, - int32_t* pIndex); +int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey); +int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey); +SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize); +SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap, int32_t* pIndex); int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted); bool functionNeedToExecute(SqlFunctionCtx* pCtx); @@ -842,6 +877,7 @@ int64_t getSmaWaterMark(int64_t interval, double filesFactor); bool isSmaStream(int8_t triggerType); int32_t compareTimeWindow(const void* p1, const void* p2, const void* param); + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index aea9d70f316806286fbd69f0d13dc72ccae54dfe..484ff35e3a7193f8eb51c49eec38aff2eb391654 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -28,18 +28,6 @@ typedef struct SCompSupporter { int32_t order; } SCompSupporter; -int32_t getRowNumForMultioutput(STaskAttr* pQueryAttr, bool topBottomQuery, bool stable) { - if (pQueryAttr && (!stable)) { - for (int16_t i = 0; i < pQueryAttr->numOfOutput; ++i) { -// if (pQueryAttr->pExpr1[i].base. == FUNCTION_TOP || pQueryAttr->pExpr1[i].base.functionId == FUNCTION_BOTTOM) { -// return (int32_t)pQueryAttr->pExpr1[i].base.param[0].i; -// } - } - } - - return 1; -} - int32_t getOutputInterResultBufSize(STaskAttr* pQueryAttr) { int32_t size = 0; @@ -101,20 +89,8 @@ void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow pResultRowInfo->size = 0; } -int32_t numOfClosedResultRows(SResultRowInfo *pResultRowInfo) { - int32_t i = 0; -// while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) { -// ++i; -// } - - return i; -} - void closeAllResultRows(SResultRowInfo *pResultRowInfo) { - assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); - - for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - } +// do nothing } bool isResultRowClosed(SResultRow* pRow) { @@ -125,35 +101,6 @@ void closeResultRow(SResultRow* pResultRow) { pResultRow->closed = true; } -void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow) { - if (pResultRow == NULL) { - return; - } - - // the result does not put into the SDiskbasedBuf, ignore it. - if (pResultRow->pageId >= 0) { - SFilePage *page = getBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId); - - int16_t offset = 0; - for (int32_t i = 0; i < pRuntimeEnv->pQueryAttr->numOfOutput; ++i) { - struct SResultRowEntryInfo *pEntryInfo = NULL;//pResultRow->pEntryInfo[i]; - -// int16_t size = pRuntimeEnv->pQueryAttr->pExpr1[i].base.resSchema.bytes; -// char * s = getPosInResultPage(pRuntimeEnv->pQueryAttr, page, pResultRow->offset, offset); -// memset(s, 0, size); - -// offset += size; - cleanupResultRowEntry(pEntryInfo); - } - } - - pResultRow->numOfRows = 0; - pResultRow->pageId = -1; - pResultRow->offset = -1; - pResultRow->closed = false; - pResultRow->win = TSWINDOW_INITIALIZER; -} - // TODO refactor: use macro SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, const int32_t* offset) { assert(index >= 0 && offset != NULL); @@ -258,32 +205,6 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { return (int32_t) taosArrayGetSize(pGroupResInfo->pRows); } -static int64_t getNumOfResultWindowRes(STaskRuntimeEnv* pRuntimeEnv, SResultRowPosition *pos, int32_t* rowCellInfoOffset) { - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - ASSERT(0); - - for (int32_t j = 0; j < pQueryAttr->numOfOutput; ++j) { - int32_t functionId = 0;//pQueryAttr->pExpr1[j].base.functionId; - - /* - * ts, tag, tagprj function can not decide the output number of current query - * the number of output result is decided by main output - */ - if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ) { - continue; - } - -// SResultRowEntryInfo *pResultInfo = getResultCell(pResultRow, j, rowCellInfoOffset); -// assert(pResultInfo != NULL); -// -// if (pResultInfo->numOfRes > 0) { -// return pResultInfo->numOfRes; -// } - } - - return 0; -} - static int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) { int32_t left = *(int32_t *)pLeft; int32_t right = *(int32_t *)pRight; @@ -381,7 +302,7 @@ static int32_t mergeIntoGroupResultImplRv(STaskRuntimeEnv *pRuntimeEnv, SGroupRe } - int64_t num = getNumOfResultWindowRes(pRuntimeEnv, &pResultRowCell->pos, rowCellInfoOffset); + int64_t num = 0;//getNumOfResultWindowRes(pRuntimeEnv, &pResultRowCell->pos, rowCellInfoOffset); if (num <= 0) { continue; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6f2be4e14ef0aee31912a20640f70faa94e73ae1..92b3195f36cf880294a947e74b6f2d880bb8f5c7 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -239,36 +239,6 @@ static bool hasNull(SColumn* pColumn, SColumnDataAgg* pStatis) { return true; } -static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env) { - int64_t newCapacity = 0; - - // more than the capacity, reallocate the resources - if (pResultRowInfo->size < pResultRowInfo->capacity) { - return; - } - - if (pResultRowInfo->capacity > 10000) { - newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25); - } else { - newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); - } - - if (newCapacity <= pResultRowInfo->capacity) { - newCapacity += 4; - } - - char* p = taosMemoryRealloc(pResultRowInfo->pPosition, newCapacity * sizeof(SResultRowPosition)); - if (p == NULL) { - longjmp(env, TSDB_CODE_OUT_OF_MEMORY); - } - - pResultRowInfo->pPosition = (SResultRowPosition*)p; - - int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; - memset(&pResultRowInfo->pPosition[pResultRowInfo->capacity], 0, sizeof(SResultRowPosition) * inc); - pResultRowInfo->capacity = (int32_t)newCapacity; -} - static bool chkResultRowFromKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, char* pData, int16_t bytes, bool masterscan, uint64_t uid) { bool existed = false; @@ -306,7 +276,7 @@ static bool chkResultRowFromKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pR return p1 != NULL; } -SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize) { +SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize) { SFilePage* pData = NULL; // in the first scan, new space needed for results @@ -375,6 +345,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR // In case of group by column query, the required SResultRow object must be existInCurrentResusltRowInfo in the // pResultRowInfo object. if (p1 != NULL) { + // todo pResult = getResultRowByPos(pResultBuf, p1); ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset); } @@ -383,19 +354,16 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR // 1. close current opened time window if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId && pResult->offset != pResultRowInfo->cur.offset))) { - // todo extract function SResultRowPosition pos = pResultRowInfo->cur; SFilePage* pPage = getBufPage(pResultBuf, pos.pageId); - SResultRow* pRow = (SResultRow*)((char*)pPage + pos.offset); - closeResultRow(pRow); releaseBufPage(pResultBuf, pPage); } // allocate a new buffer page - prepareResultListBuffer(pResultRowInfo, pTaskInfo->env); if (pResult == NULL) { ASSERT(pSup->resultRowSize > 0); - pResult = getNewResultRow_rv(pResultBuf, groupId, pSup->resultRowSize); + pResult = getNewResultRow(pResultBuf, groupId, pSup->resultRowSize); + initResultRow(pResult); // add a new result set for a new group @@ -405,12 +373,10 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR } // 2. set the new time window to be the new active time window - pResultRowInfo->pPosition[pResultRowInfo->size++] = - (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; pResultRowInfo->cur = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; // too many time window in query - if (pResultRowInfo->size > MAX_INTERVAL_TIME_WINDOW) { + if (taosHashGetSize(pSup->pResultRowHashTable) > MAX_INTERVAL_TIME_WINDOW) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } @@ -590,6 +556,7 @@ void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { // keep it temporarily + // todo no need this?? bool hasAgg = pCtx[k].input.colDataAggIsSet; int32_t numOfRows = pCtx[k].input.numOfRows; int32_t startOffset = pCtx[k].input.startRowIndex; @@ -609,7 +576,8 @@ void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow if (fmIsWindowPseudoColumnFunc(pCtx[k].functionId)) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[k]); - char* p = GET_ROWCELL_INTERBUF(pEntryInfo); + + char* p = GET_ROWCELL_INTERBUF(pEntryInfo); SColumnInfoData idata = {0}; idata.info.type = TSDB_DATA_TYPE_BIGINT; @@ -620,22 +588,23 @@ void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData}; pCtx[k].sfp.process(&tw, 1, &out); pEntryInfo->numOfRes = 1; - continue; - } - int32_t code = TSDB_CODE_SUCCESS; - if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) { - code = pCtx[k].fpSet.process(&pCtx[k]); - if (code != TSDB_CODE_SUCCESS) { - qError("%s apply functions error, code: %s", GET_TASKID(taskInfo), tstrerror(code)); - taskInfo->code = code; - longjmp(taskInfo->env, code); + } else { + int32_t code = TSDB_CODE_SUCCESS; + if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) { + code = pCtx[k].fpSet.process(&pCtx[k]); + + if (code != TSDB_CODE_SUCCESS) { + qError("%s apply functions error, code: %s", GET_TASKID(taskInfo), tstrerror(code)); + taskInfo->code = code; + longjmp(taskInfo->env, code); + } } - } - // restore it - pCtx[k].input.colDataAggIsSet = hasAgg; - pCtx[k].input.startRowIndex = startOffset; - pCtx[k].input.numOfRows = numOfRows; + // restore it + pCtx[k].input.colDataAggIsSet = hasAgg; + pCtx[k].input.startRowIndex = startOffset; + pCtx[k].input.numOfRows = numOfRows; + } } } @@ -774,12 +743,14 @@ static int32_t doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunct for (int32_t k = 0; k < pOperator->numOfExprs; ++k) { if (functionNeedToExecute(&pCtx[k])) { // todo add a dummy funtion to avoid process check - if (pCtx[k].fpSet.process != NULL) { - int32_t code = pCtx[k].fpSet.process(&pCtx[k]); - if (code != TSDB_CODE_SUCCESS) { - qError("%s aggregate function error happens, code: %s", GET_TASKID(pOperator->pTaskInfo), tstrerror(code)); - return code; - } + if (pCtx[k].fpSet.process == NULL) { + continue; + } + + int32_t code = pCtx[k].fpSet.process(&pCtx[k]); + if (code != TSDB_CODE_SUCCESS) { + qError("%s aggregate function error happens, code: %s", GET_TASKID(pOperator->pTaskInfo), tstrerror(code)); + return code; } } } @@ -1218,7 +1189,6 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { taosVariantDestroy(&pCtx[i].param[j].param); } - taosVariantDestroy(&pCtx[i].tag); taosMemoryFreeClear(pCtx[i].subsidiaries.pCtx); taosMemoryFree(pCtx[i].input.pData); taosMemoryFree(pCtx[i].input.pColumnDataAgg); @@ -1248,9 +1218,9 @@ void setTaskKilled(SExecTaskInfo* pTaskInfo) { pTaskInfo->code = TSDB_CODE_TSC_Q static bool isCachedLastQuery(STaskAttr* pQueryAttr) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); - if (functionId == FUNCTION_LAST || functionId == FUNCTION_LAST_DST) { - continue; - } + // if (functionId == FUNCTION_LAST || functionId == FUNCTION_LAST_DST) { + // continue; + // } return false; } @@ -1300,7 +1270,7 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQuery->pExpr1[i]); - +#if 0 if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY || functionId == FUNCTION_TAG || functionId == FUNCTION_TAG_DUMMY) { continue; @@ -1311,6 +1281,7 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { } else { hasOtherFunc = true; } +#endif } if (hasFirstLastFunc && status == BLK_DATA_NOT_LOAD) { @@ -1786,41 +1757,12 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity, int32_t numOf // set the correct pointer after the memory buffer reallocated. int32_t functionId = pBInfo->pCtx[i].functionId; - +#if 0 if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) { // if (i > 0) pBInfo->pCtx[i].pTsOutput = pBInfo->pCtx[i - 1].pOutput; } - } -} - -void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput) { - bool needCopyTs = false; - int32_t tsNum = 0; - char* src = NULL; - for (int32_t i = 0; i < numOfOutput; i++) { - int32_t functionId = pCtx[i].functionId; - if (functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) { - needCopyTs = true; - if (i > 0 && pCtx[i - 1].functionId == FUNCTION_TS_DUMMY) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i - 1); // find ts data - src = pColRes->pData; - } - } else if (functionId == FUNCTION_TS_DUMMY) { - tsNum++; - } - } - - if (!needCopyTs) return; - if (tsNum < 2) return; - if (src == NULL) return; - - for (int32_t i = 0; i < numOfOutput; i++) { - int32_t functionId = pCtx[i].functionId; - if (functionId == FUNCTION_TS_DUMMY) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i); - memcpy(pColRes->pData, src, pColRes->info.bytes * pRes->info.rows); - } +#endif } } @@ -1879,7 +1821,8 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO } static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SArray* pColMatchInfo) { + +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { if (pFilterNode == NULL) { return; } @@ -2472,6 +2415,9 @@ int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) pRsp->compLen = htonl(pRsp->compLen); pRsp->numOfCols = htonl(pRsp->numOfCols); pRsp->useconds = htobe64(pRsp->useconds); + + ASSERT(pSourceDataInfo->pRsp != NULL); + qDebug("fetch rsp received, index:%d, rows:%d", pSourceDataInfo->index, pRsp->numOfRows); } else { pSourceDataInfo->code = code; } @@ -2520,6 +2466,8 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); + ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY); + qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, sourceIndex, totalSources); @@ -2573,51 +2521,11 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray } } -int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, +int32_t setDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList) { if (pColList == NULL) { // data from other sources - blockDataEnsureCapacity(pRes, numOfRows); - - int32_t dataLen = *(int32_t*)pData; - pData += sizeof(int32_t); - - pRes->info.groupId = *(uint64_t*)pData; - pData += sizeof(uint64_t); - - int32_t* colLen = (int32_t*)pData; - - char* pStart = pData + sizeof(int32_t) * numOfOutput; - for (int32_t i = 0; i < numOfOutput; ++i) { - colLen[i] = htonl(colLen[i]); - ASSERT(colLen[i] >= 0); - - SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i); - if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - pColInfoData->varmeta.length = colLen[i]; - pColInfoData->varmeta.allocLen = colLen[i]; - - memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows); - pStart += sizeof(int32_t) * numOfRows; - - if (colLen[i] > 0) { - taosMemoryFreeClear(pColInfoData->pData); - pColInfoData->pData = taosMemoryMalloc(colLen[i]); - } - } else { - memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); - pStart += BitmapLen(numOfRows); - } - - if (colLen[i] > 0) { - memcpy(pColInfoData->pData, pStart, colLen[i]); - } - - // TODO setting this flag to true temporarily so aggregate function on stable will - // examine NULL value for non-primary key column - pColInfoData->hasNull = true; - pStart += colLen[i]; - } + blockCompressDecode(pRes, numOfOutput, numOfRows, pData); } else { // extract data according to pColList ASSERT(numOfOutput == taosArrayGetSize(pColList)); char* pStart = pData; @@ -2737,7 +2645,6 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx int32_t completed = 0; for (int32_t i = 0; i < totalSources; ++i) { SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i); - if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) { completed += 1; continue; @@ -2747,6 +2654,11 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx continue; } + if (pDataInfo->code != TSDB_CODE_SUCCESS) { + code = pDataInfo->code; + goto _error; + } + SRetrieveTableRsp* pRsp = pDataInfo->pRsp; SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, i); @@ -2764,7 +2676,7 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx } SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; - code = setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, + code = setDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); if (code != 0) { taosMemoryFreeClear(pDataInfo->pRsp); @@ -2784,6 +2696,8 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx pLoadInfo->totalSize); } + taosMemoryFreeClear(pDataInfo->pRsp); + if (pDataInfo->status != EX_SOURCE_DATA_EXHAUSTED) { pDataInfo->status = EX_SOURCE_DATA_NOT_READY; code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); @@ -2793,7 +2707,6 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx } } - taosMemoryFreeClear(pDataInfo->pRsp); return pExchangeInfo->pResult; } @@ -2807,34 +2720,6 @@ _error: return NULL; } -static SSDataBlock* concurrentlyLoadRemoteData(SOperatorInfo* pOperator) { - SExchangeInfo* pExchangeInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - if (pOperator->status == OP_RES_TO_RETURN) { - return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); - } - - size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - int64_t startTs = taosGetTimestampUs(); - - // Asynchronously send all fetch requests to all sources. - for (int32_t i = 0; i < totalSources; ++i) { - int32_t code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); - if (code != TSDB_CODE_SUCCESS) { - return NULL; - } - } - - int64_t endTs = taosGetTimestampUs(); - qDebug("%s send all fetch request to %" PRIzu " sources completed, elapsed:%" PRId64, GET_TASKID(pTaskInfo), - totalSources, endTs - startTs); - - tsem_wait(&pExchangeInfo->ready); - pOperator->status = OP_RES_TO_RETURN; - return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); -} - static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2852,10 +2737,11 @@ static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { } int64_t endTs = taosGetTimestampUs(); - qDebug("%s send all fetch request to %" PRIzu " sources completed, elapsed:%" PRId64, GET_TASKID(pTaskInfo), + qDebug("%s send all fetch requests to %" PRIzu " sources completed, elapsed:%" PRId64, GET_TASKID(pTaskInfo), totalSources, endTs - startTs); tsem_wait(&pExchangeInfo->ready); + pOperator->status = OP_RES_TO_RETURN; pOperator->cost.openCost = taosGetTimestampUs() - startTs; return TSDB_CODE_SUCCESS; @@ -2903,7 +2789,7 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { SSDataBlock* pRes = pExchangeInfo->pResult; SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp; int32_t code = - setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, + setDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data, pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL); if (pRsp->completed == 1) { @@ -2969,7 +2855,8 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { if (pExchangeInfo->seqLoadData) { return seqLoadRemoteData(pOperator); } else { - return concurrentlyLoadRemoteData(pOperator); + return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); +// return concurrentlyLoadRemoteData(pOperator); } } @@ -2995,52 +2882,55 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo) { return TSDB_CODE_SUCCESS; } -SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, const SNodeList* pSources, SSDataBlock* pBlock, - SExecTaskInfo* pTaskInfo) { - SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } +static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo) { + size_t numOfSources = LIST_LENGTH(pExNode->pSrcEndPoints); - size_t numOfSources = LIST_LENGTH(pSources); pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); if (pInfo->pSourceDataInfo == NULL || pInfo->pSources == NULL) { - goto _error; + return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; i < numOfSources; ++i) { - SNodeListNode* pNode = nodesListGetNode((SNodeList*)pSources, i); + SNodeListNode* pNode = nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i); taosArrayPush(pInfo->pSources, pNode); } - int32_t code = initDataSource(numOfSources, pInfo); - if (code != TSDB_CODE_SUCCESS) { + return initDataSource(numOfSources, pInfo); +} + +SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo) { + SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->pResult = pBlock; - pInfo->seqLoadData = true; + int32_t code = initExchangeOperator(pExNode, pInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + pInfo->seqLoadData = false; + pInfo->pTransporter = pTransporter; + pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); tsem_init(&pInfo->ready, 0, 0); - pOperator->name = "ExchangeOperator"; + pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfExprs = pBlock->info.numOfCols; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfExprs = pInfo->pResult->info.numOfCols; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo, NULL, NULL, NULL); - pInfo->pTransporter = pTransporter; return pOperator; _error: if (pInfo != NULL) { - destroyExchangeOperatorInfo(pInfo, numOfSources); + destroyExchangeOperatorInfo(pInfo, LIST_LENGTH(pExNode->pSrcEndPoints)); } taosMemoryFreeClear(pInfo); @@ -3587,7 +3477,7 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result) { offset += sizeof(int32_t); uint64_t tableGroupId = *(uint64_t*)(result + offset); - SResultRow* resultRow = getNewResultRow_rv(pSup->pResultBuf, tableGroupId, pSup->resultRowSize); + SResultRow* resultRow = getNewResultRow(pSup->pResultBuf, tableGroupId, pSup->resultRowSize); if (!resultRow) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -3610,10 +3500,6 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result) { offset += valueLen; initResultRow(resultRow); - prepareResultListBuffer(&pInfo->resultRowInfo, pOperator->pTaskInfo->env); - // pInfo->resultRowInfo.cur = pInfo->resultRowInfo.size; - // pInfo->resultRowInfo.pPosition[pInfo->resultRowInfo.size++] = - // (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset}; pInfo->resultRowInfo.cur = (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset}; } @@ -3768,6 +3654,8 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, code); } + doFilter(pProjectInfo->pFilterNode, pBlock); + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false); blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); @@ -3905,18 +3793,6 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { } } -// todo set the attribute of query scan count -static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); - if (functionId == FUNCTION_STDDEV || functionId == FUNCTION_PERCT) { - return 2; - } - } - - return 1; -} - static void destroyOperatorInfo(SOperatorInfo* pOperator) { if (pOperator == NULL) { return; @@ -4155,7 +4031,7 @@ static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) } SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, - SSDataBlock* pResBlock, SLimit* pLimit, SLimit* pSlimit, + SSDataBlock* pResBlock, SLimit* pLimit, SLimit* pSlimit, SNode* pCondition, SExecTaskInfo* pTaskInfo) { SProjectOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SProjectOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -4167,8 +4043,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p pInfo->slimit = *pSlimit; pInfo->curOffset = pLimit->offset; pInfo->curSOffset = pSlimit->offset; - pInfo->binfo.pRes = pResBlock; + pInfo->pFilterNode = pCondition; int32_t numOfCols = num; int32_t numOfRows = 4096; @@ -4370,7 +4246,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pValNode->node.aliasName); pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE; - valueNodeToVariant(pValNode, &pExp->base.pParam[0].param); + nodesValueNodeToVariant(pValNode, &pExp->base.pParam[0].param); } else if (type == QUERY_NODE_FUNCTION) { pExp->pExpr->nodeType = QUERY_NODE_FUNCTION; SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; @@ -4413,7 +4289,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* } else if (p1->type == QUERY_NODE_VALUE) { SValueNode* pvn = (SValueNode*)p1; pExp->base.pParam[j].type = FUNC_PARAM_TYPE_VALUE; - valueNodeToVariant(pvn, &pExp->base.pParam[j].param); + nodesValueNodeToVariant(pvn, &pExp->base.pParam[j].param); } } } else if (type == QUERY_NODE_OPERATOR) { @@ -4498,23 +4374,22 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (pDataReader == NULL && terrno != 0) { return NULL; } - + SArray* groupKyes = extractPartitionColInfo(pTableScanNode->pPartitionKeys); extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); - SOperatorInfo* pOperator = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); + SOperatorInfo* pOperator = + createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, groupKyes, pTaskInfo); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { - SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; - SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc); - return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, pExchange->pSrcEndPoints, pResBlock, pTaskInfo); + return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STimeWindowAggSupp twSup = { - .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN}; + .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN}; tsdbReaderT pDataReader = NULL; if (pHandle->vnode) { pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond); @@ -4528,50 +4403,25 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else { qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo)); } - SArray* tableIdList = extractTableIdList(pTableListInfo); + SArray* tableIdList = extractTableIdList(pTableListInfo); - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pDataReader, pHandle, - tableIdList, pTableScanNode, pTaskInfo, &twSup); + SOperatorInfo* pOperator = + createStreamScanOperatorInfo(pDataReader, pHandle, tableIdList, pTableScanNode, pTaskInfo, &twSup); taosArrayDestroy(tableIdList); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; - SScanPhysiNode* pScanNode = &pSysScanPhyNode->scan; - - SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; - - SSDataBlock* pResBlock = createResDataBlock(pDescNode); - - int32_t numOfOutputCols = 0; - SArray* colList = - extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_COL_ID); - SOperatorInfo* pOperator = createSysTableScanOperatorInfo( - pHandle, pResBlock, &pScanNode->tableName, pScanNode->node.pConditions, pSysScanPhyNode->mgmtEpSet, colList, - pTaskInfo, pSysScanPhyNode->showRewrite, pSysScanPhyNode->accountId); - return pOperator; + return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; - SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; - - SSDataBlock* pResBlock = createResDataBlock(pDescNode); - int32_t code = getTableList(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableListInfo, pTagCond); if (code != TSDB_CODE_SUCCESS) { return NULL; } - int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(pScanPhyNode->pScanPseudoCols, NULL, &num); - - int32_t numOfOutputCols = 0; - SArray* colList = extractColMatchInfo(pScanPhyNode->pScanPseudoCols, pDescNode, &numOfOutputCols, pTaskInfo, - COL_MATCH_FROM_COL_ID); - - SOperatorInfo* pOperator = - createTagScanOperatorInfo(pHandle, pExprInfo, num, pResBlock, colList, pTableListInfo, pTaskInfo); - return pOperator; + return createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo); } else { ASSERT(0); } @@ -4597,7 +4447,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); SLimit limit = {.limit = pProjPhyNode->limit, .offset = pProjPhyNode->offset}; SLimit slimit = {.limit = pProjPhyNode->slimit, .offset = pProjPhyNode->soffset}; - pOptr = createProjectOperatorInfo(ops[0], pExprInfo, num, pResBlock, &limit, &slimit, pTaskInfo); + pOptr = createProjectOperatorInfo(ops[0], pExprInfo, num, pResBlock, &limit, &slimit, + pProjPhyNode->node.pConditions, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_AGG == type) { SAggPhysiNode* pAggNode = (SAggPhysiNode*)pPhyNode; SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); @@ -4617,7 +4468,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pScalarExprInfo, numOfScalarExpr, pTaskInfo); } - } else if (QUERY_NODE_PHYSICAL_PLAN_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { + } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); @@ -4630,18 +4481,19 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo .offset = pIntervalPhyNode->offset, .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision}; - STimeWindowAggSupp as = {.waterMark = pIntervalPhyNode->window.watermark, - .calTrigger = pIntervalPhyNode->window.triggerType, - .maxTs = INT64_MIN, - .winMap = NULL,}; + STimeWindowAggSupp as = { + .waterMark = pIntervalPhyNode->window.watermark, + .calTrigger = pIntervalPhyNode->window.triggerType, + .maxTs = INT64_MIN, + .winMap = NULL, + }; if (isSmaStream(pIntervalPhyNode->window.triggerType)) { if (FLT_LESS(pIntervalPhyNode->window.filesFactor, 1.000000)) { as.calTrigger = STREAM_TRIGGER_AT_ONCE_SMA; } else { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP); as.winMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK); - as.waterMark = getSmaWaterMark(interval.interval, - pIntervalPhyNode->window.filesFactor); + as.waterMark = getSmaWaterMark(interval.interval, pIntervalPhyNode->window.filesFactor); as.calTrigger = STREAM_TRIGGER_WINDOW_CLOSE_SMA; } } @@ -4709,6 +4561,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; SColumn col = extractColumnFromColumnNode(pColNode); pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, tsSlotId, &col, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW == type) { + pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_JOIN == type) { SJoinPhysiNode* pJoinNode = (SJoinPhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); @@ -4943,7 +4797,8 @@ int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STa if (tableType == TSDB_SUPER_TABLE) { if (pTagCond) { - SIndexMetaArg metaArg = {.metaEx = metaHandle, .metaHandle = tsdbGetIdx(metaHandle), .suid = tableUid}; + SIndexMetaArg metaArg = { + .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid}; SArray* res = taosArrayInit(8, sizeof(uint64_t)); code = doFilterTag(pTagCond, &metaArg, res); @@ -5129,45 +4984,6 @@ _complete: return code; } -void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) { - const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); - - // the minimum number of rows for projection query - const int32_t MIN_ROWS_FOR_PRJ_QUERY = 8192; - const int32_t DEFAULT_MIN_ROWS = 4096; - - const float THRESHOLD_RATIO = 0.85f; - - // if (isProjQuery(pQueryAttr)) { - // int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize; - // if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) { - // numOfRes = MIN_ROWS_FOR_PRJ_QUERY; - // } - // - // pResultInfo->capacity = numOfRes; - // } else { // in case of non-prj query, a smaller output buffer will be used. - // pResultInfo->capacity = DEFAULT_MIN_ROWS; - // } - - pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO); - pResultInfo->totalRows = 0; -} - -// TODO refactor -void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) { - if (pFilter == NULL || numOfFilters == 0) { - return; - } - - for (int32_t i = 0; i < numOfFilters; i++) { - if (pFilter[i].filterstr && pFilter[i].pz) { - taosMemoryFree((void*)(pFilter[i].pz)); - } - } - - taosMemoryFree(pFilter); -} - static void doDestroyTableList(STableListInfo* pTableqinfoList) { taosArrayDestroy(pTableqinfoList->pTableList); taosHashCleanup(pTableqinfoList->map); @@ -5294,14 +5110,16 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo return TSDB_CODE_SUCCESS; } -int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey) { +int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, size_t size) { pSup->keySize = sizeof(int64_t) + sizeof(TSKEY); pSup->pKeyBuf = taosMemoryCalloc(1, pSup->keySize); - pSup->pResultRows = taosArrayInit(1024, sizeof(SResultWindowInfo)); + pSup->pResultRows = taosArrayInit(1024, size); if (pSup->pKeyBuf == NULL || pSup->pResultRows == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + pSup->pScanWindow = taosArrayInit(4, sizeof(STimeWindow)); + int32_t pageSize = 4096; while (pageSize < pSup->resultRowSize * 4) { pageSize <<= 1u; @@ -5314,16 +5132,23 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey) { return createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, pKey, TD_TMP_DIR_PATH); } +int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey) { + return initStreamAggSupporter(pSup, pKey, sizeof(SResultWindowInfo)); +} + +int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey) { + return initStreamAggSupporter(pSup, pKey, sizeof(SStateWindowInfo)); +} + int64_t getSmaWaterMark(int64_t interval, double filesFactor) { int64_t waterMark = 0; - ASSERT(FLT_GREATEREQUAL(filesFactor,0.000000)); + ASSERT(FLT_GREATEREQUAL(filesFactor, 0.000000)); waterMark = -1 * filesFactor; return waterMark; } bool isSmaStream(int8_t triggerType) { - if (triggerType == STREAM_TRIGGER_AT_ONCE || - triggerType == STREAM_TRIGGER_WINDOW_CLOSE) { + if (triggerType == STREAM_TRIGGER_AT_ONCE || triggerType == STREAM_TRIGGER_WINDOW_CLOSE) { return false; } return true; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index d388b802f3ae22c7c0cf135c9eb70111be1d79a4..132f93a6a533caa2f676009a47afdea8ee6ee869 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -24,10 +24,10 @@ #include "tcompare.h" #include "thash.h" #include "ttypes.h" +#include "executorInt.h" static int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity); static void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len); -static uint64_t calcGroupId(char* pData, int32_t len); static void destroyGroupOperatorInfo(void* param, int32_t numOfOutput) { SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*)param; @@ -37,7 +37,7 @@ static void destroyGroupOperatorInfo(void* param, int32_t numOfOutput) { taosArrayDestroy(pInfo->pGroupColVals); } -static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { +int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { *pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); if ((*pGroupColVals) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -93,7 +93,15 @@ static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlo char* val = colDataGetData(pColInfoData, rowIndex); - if (IS_VAR_DATA_TYPE(pkey->type)) { + if (pkey->type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(val); + + if (memcmp(pkey->pData, val, dataLen) == 0){ + continue; + } else { + return false; + } + } else if (IS_VAR_DATA_TYPE(pkey->type)) { int32_t len = varDataLen(val); if (len == varDataLen(pkey->pData) && memcmp(varDataVal(pkey->pData), varDataVal(val), len) == 0) { continue; @@ -110,9 +118,11 @@ static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlo return true; } -static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { +void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex) { SColumnDataAgg* pColAgg = NULL; + size_t numOfGroupCols = taosArrayGetSize(pGroupCols); + for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = taosArrayGet(pGroupCols, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); @@ -127,7 +137,10 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData } else { pkey->isNull = false; char* val = colDataGetData(pColInfoData, rowIndex); - if (IS_VAR_DATA_TYPE(pkey->type)) { + if (pkey->type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(val); + memcpy(pkey->pData, val, dataLen); + } else if (IS_VAR_DATA_TYPE(pkey->type)) { memcpy(pkey->pData, val, varDataTLen(val)); ASSERT(varDataTLen(val) <= pkey->bytes); } else { @@ -137,7 +150,7 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData } } -static int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) { +int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) { ASSERT(pKey != NULL); size_t numOfGroupCols = taosArrayGetSize(pGroupColVals); @@ -151,7 +164,11 @@ static int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) { } isNull[i] = 0; - if (IS_VAR_DATA_TYPE(pkey->type)) { + if (pkey->type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(pkey->pData); + memcpy(pStart, (pkey->pData), dataLen); + pStart += dataLen; + } else if (IS_VAR_DATA_TYPE(pkey->type)) { varDataCopy(pStart, pkey->pData); pStart += varDataTLen(pkey->pData); ASSERT(varDataTLen(pkey->pData) <= pkey->bytes); @@ -176,7 +193,10 @@ static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t char* dest = GET_ROWCELL_INTERBUF(pEntryInfo); char* data = colDataGetData(pColInfoData, rowIndex); - if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { + int32_t dataLen = getJsonValueLen(data); + memcpy(dest, data, dataLen); + } else if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { varDataCopy(dest, data); } else { memcpy(dest, data, pColInfoData->info.bytes); @@ -208,7 +228,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { for (int32_t j = 0; j < pBlock->info.rows; ++j) { // Compare with the previous row of this column, and do not set the output buffer again if they are identical. if (!pInfo->isInit) { - recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); + recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j); pInfo->isInit = true; num++; continue; @@ -223,7 +243,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { // The first row of a new block does not belongs to the previous existed group if (j == 0) { num++; - recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); + recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j); continue; } @@ -238,7 +258,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { // assign the group keys or user input constant values if required doAssignGroupKeys(pCtx, pOperator->numOfExprs, pBlock->info.rows, rowIndex); - recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); + recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j); num = 1; } @@ -339,7 +359,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { while(1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pRes, NULL); + doFilter(pInfo->pCondition, pRes); bool hasRemain = hashRemainDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { @@ -407,9 +427,8 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { SPartitionOperatorInfo* pInfo = pOperator->info; - int32_t numOfGroupCols = taosArrayGetSize(pInfo->pGroupCols); for (int32_t j = 0; j < pBlock->info.rows; ++j) { - recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); + recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j); int32_t len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); SDataGroupInfo* pGInfo = NULL; @@ -446,6 +465,16 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { if (colDataIsNull_s(pColInfoData, j)) { offset[(*rows)] = -1; contentLen = 0; + } else if(pColInfoData->info.type == TSDB_DATA_TYPE_JSON){ + offset[*rows] = (*columnLen); + char* src = colDataGetData(pColInfoData, j); + int32_t dataLen = getJsonValueLen(src); + + memcpy(data + (*columnLen), src, dataLen); + int32_t v = (data + (*columnLen) + dataLen - (char*)pPage); + ASSERT(v > 0); + + contentLen = dataLen; } else { offset[*rows] = (*columnLen); char* src = colDataGetData(pColInfoData, j); @@ -620,8 +649,13 @@ static void destroyPartitionOperatorInfo(void* param, int32_t numOfOutput) { SPartitionOperatorInfo* pInfo = (SPartitionOperatorInfo*)param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); taosArrayDestroy(pInfo->pGroupCols); + for(int i = 0; i < taosArrayGetSize(pInfo->pGroupColVals); i++){ + SGroupKeys key = *(SGroupKeys*)taosArrayGet(pInfo->pGroupColVals, i); + taosMemoryFree(key.pData); + } taosArrayDestroy(pInfo->pGroupColVals); taosMemoryFree(pInfo->keyBuf); + taosHashCleanup(pInfo->pGroupSet); taosMemoryFree(pInfo->columnOffset); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0e682682a11330fa07f876097a9f93b0108e442d..ba0397fd3439e252c3de115459205dd534c7f78a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -33,18 +33,31 @@ #include "ttypes.h" #include "vnode.h" +#include "executorInt.h" + #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) -typedef struct SWindowPosition { - int32_t pageId; - int32_t rowId; -} SWindowPosition; - static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, const char* dbName); +static void addTagPseudoColumnData(SReadHandle *pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock); +static bool processBlockWithProbability(const SSampleExecInfo *pInfo); + +bool processBlockWithProbability(const SSampleExecInfo *pInfo) { +#if 0 + if (pInfo->sampleRatio == 1) { + return true; + } + + uint32_t val = taosRandR((uint32_t*) &pInfo->seed); + return (val % ((uint32_t)(1/pInfo->sampleRatio))) == 0; +#else + return true; +#endif +} + static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t i = 0; i < numOfOutput; ++i) { SWITCH_ORDER(pCtx[i].order); @@ -158,8 +171,6 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn return false; } -static void addTagPseudoColumnData(SReadHandle *pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock); - static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -254,7 +265,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } int64_t st = taosGetTimestampMs(); - doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo); + doFilter(pTableScanInfo->pFilterNode, pBlock); int64_t et = taosGetTimestampMs(); pTableScanInfo->readRecorder.filterTime += (et - st); @@ -279,12 +290,9 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction STimeWindow* pTWindow = &pTableScanInfo->cond.twindows[i]; TSWAP(pTWindow->skey, pTWindow->ekey); } - SQueryTableDataCond *pCond = &pTableScanInfo->cond; - taosqsort(pCond->twindows, - pCond->numOfTWindows, - sizeof(STimeWindow), - pCond, - compareTimeWindow); + + SQueryTableDataCond* pCond = &pTableScanInfo->cond; + taosqsort(pCond->twindows, pCond->numOfTWindows, sizeof(STimeWindow), pCond, compareTimeWindow); } void addTagPseudoColumnData(SReadHandle *pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock) { @@ -315,18 +323,19 @@ void addTagPseudoColumnData(SReadHandle *pHandle, SExprInfo* pPseudoExpr, int32_ } else { // these are tags STagVal tagVal = {0}; tagVal.cid = pExpr->base.pParam[0].pCol->colId; - const char *p = metaGetTableTagVal(&mr.me, pColInfoData->info.type, &tagVal); + const char* p = metaGetTableTagVal(&mr.me, pColInfoData->info.type, &tagVal); - char *data = NULL; - if(pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL){ - data = tTagValToData((const STagVal *)p, false); - }else { + char* data = NULL; + if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { + data = tTagValToData((const STagVal*)p, false); + } else { data = (char*)p; } for (int32_t i = 0; i < pBlock->info.rows; ++i) { colDataAppend(pColInfoData, i, data, (data == NULL)); } + if (data && (pColInfoData->info.type != TSDB_DATA_TYPE_JSON) && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type)) { taosMemoryFree(data); @@ -364,6 +373,12 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } + // process this data block based on the probabilities + bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample); + if (!processThisBlock) { + continue; + } + tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info); uint32_t status = 0; @@ -373,6 +388,17 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { longjmp(pOperator->pTaskInfo->env, code); } + recordNewGroupKeys(pTableScanInfo->pGroupCols, pTableScanInfo->pGroupColVals, pBlock, 0); + int32_t len = buildGroupKeys(pTableScanInfo->keyBuf, pTableScanInfo->pGroupColVals); + + uint64_t *groupId = taosHashGet(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len); + if (groupId) { + pBlock->info.groupId = *groupId; + }else if(len != 0){ + pBlock->info.groupId = calcGroupId(pTableScanInfo->keyBuf, len); + taosHashPut(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len, &pBlock->info.groupId, sizeof(uint64_t)); + } + // current block is filter out according to filter condition, continue load the next block if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) { continue; @@ -438,6 +464,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STimeWindow* pWin = &pTableScanInfo->cond.twindows[i]; qDebug("%s\t qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pWin->skey, pWin->ekey); } + while (pTableScanInfo->scanTimes < total) { while (pTableScanInfo->curTWinIdx < pTableScanInfo->cond.numOfTWindows) { SSDataBlock* p = doTableScanImpl(pOperator); @@ -497,21 +524,25 @@ static void destroyTableScanOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFree(pTableScanInfo->pResBlock); tsdbCleanupReadHandle(pTableScanInfo->dataReader); + taosArrayDestroy(pTableScanInfo->pGroupCols); + for(int i = 0; i < taosArrayGetSize(pTableScanInfo->pGroupColVals); i++){ + SGroupKeys key = *(SGroupKeys*)taosArrayGet(pTableScanInfo->pGroupColVals, i); + taosMemoryFree(key.pData); + } + taosArrayDestroy(pTableScanInfo->pGroupColVals); + taosMemoryFree(pTableScanInfo->keyBuf); + taosHashCleanup(pTableScanInfo->pGroupSet); if (pTableScanInfo->pColMatchInfo != NULL) { taosArrayDestroy(pTableScanInfo->pColMatchInfo); } } SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, - SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { + SReadHandle* readHandle, SArray* groupKyes, SExecTaskInfo* pTaskInfo) { STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; + goto _error; } SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc; @@ -522,7 +553,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { - return NULL; + goto _error; } if (pTableScanNode->scan.pScanPseudoCols != NULL) { @@ -531,34 +562,54 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; - // pInfo->scanInfo = (SScanInfo){.numOfAsc = 0, .numOfDesc = 1}; // for debug purpose +// pInfo->scanInfo = (SScanInfo){.numOfAsc = 0, .numOfDesc = 1}; // for debug purpose + + pInfo->readHandle = *readHandle; + pInfo->interval = extractIntervalInfo(pTableScanNode); + pInfo->sample.sampleRatio= pTableScanNode->ratio; + pInfo->sample.seed = taosGetTimestampSec(); - pInfo->readHandle = *readHandle; - pInfo->interval = extractIntervalInfo(pTableScanNode); - pInfo->sampleRatio = pTableScanNode->ratio; pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; - pInfo->pResBlock = createResDataBlock(pDescNode); - pInfo->pFilterNode = pTableScanNode->scan.node.pConditions; - pInfo->dataReader = pDataReader; - pInfo->scanFlag = MAIN_SCAN; - pInfo->pColMatchInfo = pColList; - pInfo->curTWinIdx = 0; - - pOperator->name = "TableScanOperator"; // for debug purpose + pInfo->pResBlock = createResDataBlock(pDescNode); + pInfo->pFilterNode = pTableScanNode->scan.node.pConditions; + pInfo->dataReader = pDataReader; + pInfo->scanFlag = MAIN_SCAN; + pInfo->pColMatchInfo = pColList; + pInfo->curTWinIdx = 0; + + pOperator->name = "TableScanOperator"; // for debug purpose pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfExprs = numOfCols; + pOperator->pTaskInfo = pTaskInfo; + + // for table group + pInfo->pGroupCols = groupKyes; + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pGroupSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); + if (pInfo->pGroupSet == NULL) { + goto _error; + } + code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, groupKyes); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, destroyTableScanOperatorInfo, NULL, NULL, getTableScannerExecInfo); // for non-blocking operator, the open cost is always 0 pOperator->cost.openCost = 0; - return pOperator; + + _error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; } SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pTaskInfo) { @@ -671,16 +722,23 @@ static void doClearBufferedBlocks(SStreamBlockScanInfo* pInfo) { taosArrayClear(pInfo->pBlockLists); } -static bool isSessionWindow(SStreamBlockScanInfo* pInfo) { return pInfo->sessionSup.pStreamAggSup != NULL; } +static bool isSessionWindow(SStreamBlockScanInfo* pInfo) { + return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION_WINDOW; +} + +static bool isStateWindow(SStreamBlockScanInfo* pInfo) { + return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW; +} static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { SSDataBlock* pSDB = pInfo->pUpdateRes; - if (pInfo->updateResIndex < pSDB->info.rows) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, 0); + STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX,}; + bool needRead = false; + if (!isStateWindow(pInfo) && pInfo->updateResIndex < pSDB->info.rows) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, pInfo->primaryTsIndex); TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win; if (isSessionWindow(pInfo)) { SStreamAggSupporter* pAggSup = pInfo->sessionSup.pStreamAggSup; int64_t gap = pInfo->sessionSup.gap; @@ -696,15 +754,28 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { pInfo->updateResIndex += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, pInfo->updateResIndex, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); } - STableScanInfo* pTableScanInfo = pInfo->pOperatorDumy->info; - pTableScanInfo->cond.twindows[0] = win; - pTableScanInfo->curTWinIdx = 0; - tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); - pTableScanInfo->scanTimes = 0; - return true; - } else { + needRead = true; + } else if (isStateWindow(pInfo)) { + SArray* pWins = pInfo->sessionSup.pStreamAggSup->pScanWindow; + int32_t size = taosArrayGetSize(pWins); + if (pInfo->scanWinIndex < size) { + win = *(STimeWindow *)taosArrayGet(pWins, pInfo->scanWinIndex); + pInfo->scanWinIndex++; + needRead = true; + } else { + pInfo->scanWinIndex = 0; + taosArrayClear(pWins); + } + } + if (!needRead) { return false; } + STableScanInfo* pTableScanInfo = pInfo->pOperatorDumy->info; + pTableScanInfo->cond.twindows[0] = win; + pTableScanInfo->curTWinIdx = 0; + tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); + pTableScanInfo->scanTimes = 0; + return true; } static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) { @@ -719,36 +790,39 @@ static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) { return pResult; } -static SSDataBlock* getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool invertible) { - SColumnInfoData* pColDataInfo = taosArrayGet(pInfo->pRes->pDataBlock, pInfo->primaryTsIndex); +static void getUpdateDataBlock(SStreamBlockScanInfo* pInfo, bool invertible, + SSDataBlock* pBlock, SSDataBlock* pUpdateBlock) { + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); + ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); TSKEY* ts = (TSKEY*)pColDataInfo->pData; - for (int32_t i = 0; i < pInfo->pRes->info.rows; i++) { - if (updateInfoIsUpdated(pInfo->pUpdateInfo, pInfo->pRes->info.uid, ts[i])) { + for (int32_t i = 0; i < pBlock->info.rows; i++) { + if (updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, ts[i])) { taosArrayPush(pInfo->tsArray, ts + i); } } + if (!pUpdateBlock) { + taosArrayClear(pInfo->tsArray); + return; + } int32_t size = taosArrayGetSize(pInfo->tsArray); if (size > 0 && invertible) { - // TODO(liuyao) get from tsdb - // SSDataBlock* p = createOneDataBlock(pInfo->pRes, true); + // Todo(liuyao) get from tsdb + // SSDataBlock* p = createOneDataBlock(pBlock, true); // p->info.type = STREAM_INVERT; // taosArrayClear(pInfo->tsArray); // return p; - SSDataBlock* pDataBlock = createOneDataBlock(pInfo->pRes, false); - SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pDataBlock->pDataBlock, 0); + SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); colInfoDataEnsureCapacity(pCol, 0, size); for (int32_t i = 0; i < size; i++) { TSKEY* pTs = (TSKEY*)taosArrayGet(pInfo->tsArray, i); colDataAppend(pCol, i, (char*)pTs, false); } - pDataBlock->info.rows = size; - pDataBlock->info.type = STREAM_REPROCESS; - blockDataUpdateTsWindow(pDataBlock, 0); + pUpdateBlock->info.rows = size; + pUpdateBlock->info.type = STREAM_REPROCESS; + blockDataUpdateTsWindow(pUpdateBlock, 0); taosArrayClear(pInfo->tsArray); - return pDataBlock; } - return NULL; } static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { @@ -780,14 +854,25 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } else if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) { blockDataCleanup(pInfo->pRes); pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; - prepareDataScan(pInfo); + if (!isStateWindow(pInfo)) { + prepareDataScan(pInfo); + } return pInfo->pUpdateRes; - } else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) { - SSDataBlock* pSDB = doDataScan(pInfo); - if (pSDB == NULL) { - pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - } else { - return pSDB; + } else { + if (isStateWindow(pInfo) && + taosArrayGetSize(pInfo->sessionSup.pStreamAggSup->pScanWindow) > 0) { + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; + pInfo->updateResIndex = pInfo->pUpdateRes->info.rows; + prepareDataScan(pInfo); + } + if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) { + SSDataBlock* pSDB = doDataScan(pInfo); + if (pSDB == NULL) { + pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + } else { + getUpdateDataBlock(pInfo, true, pSDB, NULL); + return pSDB; + } } } @@ -859,7 +944,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes); } - doFilter(pInfo->pCondition, pInfo->pRes, NULL); + doFilter(pInfo->pCondition, pInfo->pRes); blockDataUpdateTsWindow(pInfo->pRes, 0); break; } @@ -871,7 +956,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { if (rows == 0) { pOperator->status = OP_EXEC_DONE; } else if (pInfo->pUpdateInfo) { - SSDataBlock* upRes = getUpdateDataBlock(pInfo, true); + SSDataBlock* upRes = createOneDataBlock(pInfo->pRes, false); + getUpdateDataBlock(pInfo, true, pInfo->pRes, upRes); if (upRes) { pInfo->pUpdateRes = upRes; if (upRes->info.type == STREAM_REPROCESS) { @@ -901,7 +987,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan SScanPhysiNode* pScanPhyNode = &pTableScanNode->scan; SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; - SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); + SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, NULL, pTaskInfo); STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanDummy->info; @@ -915,6 +1001,9 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan int16_t colId = id->colId; taosArrayPush(pColIds, &colId); + if (id->colId == pTableScanNode->tsColId) { + pInfo->primaryTsIndex = id->targetSlotId; + } } // set the extract column id to streamHandle @@ -939,7 +1028,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan pTwSup->waterMark = getSmaWaterMark(pSTInfo->interval.interval, pTableScanNode->filesFactor); } - pInfo->primaryTsIndex = 0; // pTableScanNode->tsColId; + if (pSTInfo->interval.interval > 0 && pDataReader) { pInfo->pUpdateInfo = updateInfoInitP(&pSTInfo->interval, pTwSup->waterMark); } else { @@ -1356,7 +1445,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } SRetrieveMetaTableRsp* pTableRsp = pInfo->pRsp; - setSDataBlockFromFetchRsp(pInfo->pRes, &pInfo->loadInfo, pTableRsp->numOfRows, pTableRsp->data, + setDataBlockFromFetchRsp(pInfo->pRes, &pInfo->loadInfo, pTableRsp->numOfRows, pTableRsp->data, pTableRsp->compLen, pOperator->numOfExprs, startTs, NULL, pInfo->scanCols); // todo log the filter info @@ -1430,48 +1519,59 @@ int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbT return numOfRows; } -SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSDataBlock* pResBlock, const SName* pName, - SNode* pCondition, SEpSet epset, SArray* colList, - SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId) { +SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo) { SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; + goto _error; } - pInfo->accountId = accountId; - pInfo->showRewrite = showRewrite; - pInfo->pRes = pResBlock; - pInfo->pCondition = pCondition; - pInfo->scanCols = colList; + SScanPhysiNode* pScanNode = &pScanPhyNode->scan; + + SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; + SSDataBlock* pResBlock = createResDataBlock(pDescNode); + + int32_t num = 0; + SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, pTaskInfo, COL_MATCH_FROM_COL_ID); + + pInfo->accountId = pScanPhyNode->accountId; + pInfo->showRewrite = pScanPhyNode->showRewrite; + pInfo->pRes = pResBlock; + pInfo->pCondition = pScanNode->node.pConditions; + pInfo->scanCols = colList; initResultSizeInfo(pOperator, 4096); - tNameAssign(&pInfo->name, pName); + tNameAssign(&pInfo->name, &pScanNode->tableName); const char* name = tNameGetTableName(&pInfo->name); + if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { pInfo->readHandle = *(SReadHandle*)readHandle; blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); } else { tsem_init(&pInfo->ready, 0, 0); - pInfo->epSet = epset; + pInfo->epSet = pScanPhyNode->mgmtEpSet; pInfo->readHandle = *(SReadHandle*)readHandle; } - pOperator->name = "SysTableScanOperator"; + pOperator->name = "SysTableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfExprs = pResBlock->info.numOfCols; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfExprs = pResBlock->info.numOfCols; + pOperator->pTaskInfo = pTaskInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, NULL, NULL, NULL); - pOperator->pTaskInfo = pTaskInfo; return pOperator; + + _error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; } static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { @@ -1626,28 +1726,34 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, - SSDataBlock* pResBlock, SArray* pColMatchInfo, STableListInfo* pTableListInfo, - SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->pTableList = pTableListInfo; - pInfo->pColMatchInfo = pColMatchInfo; - pInfo->pRes = pResBlock; - pInfo->readHandle = *pReadHandle; - pInfo->curPos = 0; - pOperator->name = "TagScanOperator"; + SDataBlockDescNode* pDescNode = pPhyNode->node.pOutputDataBlockDesc; + + int32_t numOfExprs = 0; + SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs); + + int32_t num = 0; + SArray* colList = extractColMatchInfo(pPhyNode->pScanPseudoCols, pDescNode, &num, pTaskInfo, COL_MATCH_FROM_COL_ID); + + pInfo->pTableList = pTableListInfo; + pInfo->pColMatchInfo = colList; + pInfo->pRes = createResDataBlock(pDescNode);; + pInfo->readHandle = *pReadHandle; + pInfo->curPos = 0; + pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfExprs = numOfOutput; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pExpr = pExprInfo; + pOperator->numOfExprs = numOfExprs; + pOperator->pTaskInfo = pTaskInfo; initResultSizeInfo(pOperator, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 8f5fa88070fde1625385fd6e691ccccf424c2094..5e1b47cc9115bd10231ceaa854d4453e5b9223df 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include "tdatablock.h" #include "executorimpl.h" diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 018f0016a22db237b70bdd43135492d835147ee1..110c2d6a8f896ee040d03402780564e44df3216b 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -22,24 +22,25 @@ #include "thash.h" #include "ttime.h" -#include "function.h" -#include "tdatablock.h" #include "executorInt.h" +#include "function.h" #include "querynodes.h" +#include "tdatablock.h" #include "tfill.h" #define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC) -#define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1)))) +#define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) \ + ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1)))) static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { - for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) { + for (int32_t j = 0; j < pFillInfo->numOfCols; ++j) { SFillColInfo* pCol = &pFillInfo->pFillCol[j]; if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || TSDB_COL_IS_UD_COL(pCol->flag)) { continue; } SResSchema* pSchema = &pCol->pExpr->base.resSchema; - char* val1 = elePtrAt(data[j], pSchema->bytes, genRows); + char* val1 = elePtrAt(data[j], pSchema->bytes, genRows); assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags); SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; @@ -55,21 +56,22 @@ static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) } } -#define GET_DEST_SLOT_ID(_p) ((_p)->pExpr->base.resSchema.slotId) +#define GET_DEST_SLOT_ID(_p) ((_p)->pExpr->base.resSchema.slotId) #define GET_SRC_SLOT_ID(_p) ((_p)->pExpr->base.pParam[0].pCol->slotId) static void doSetVal(SColumnInfoData* pDstColInfoData, int32_t rowIndex, const SGroupKeys* pKey); -static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock *pBlock, SSDataBlock* pSrcBlock, int64_t ts, bool outOfBound) { - SPoint point1, point2, point; +static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock* pBlock, SSDataBlock* pSrcBlock, int64_t ts, + bool outOfBound) { + SPoint point1, point2, point; int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); // set the primary timestamp column value - int32_t index = pFillInfo->numOfCurrent; - SColumnInfoData *pCol0 = taosArrayGet(pBlock->pDataBlock, 0); - char* val = colDataGetData(pCol0, index); - - *(TSKEY*) val = pFillInfo->currentKey; + int32_t index = pFillInfo->numOfCurrent; + SColumnInfoData* pCol0 = taosArrayGet(pBlock->pDataBlock, 0); + char* val = colDataGetData(pCol0, index); + + *(TSKEY*)val = pFillInfo->currentKey; // set the other values if (pFillInfo->type == TSDB_FILL_PREV) { @@ -81,7 +83,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock *pBlock, SSData continue; } - SGroupKeys* pKey = taosArrayGet(p, i); + SGroupKeys* pKey = taosArrayGet(p, i); SColumnInfoData* pDstColInfoData = taosArrayGet(pBlock->pDataBlock, GET_DEST_SLOT_ID(pCol)); doSetVal(pDstColInfoData, index, pKey); } @@ -94,7 +96,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock *pBlock, SSData continue; } - SGroupKeys* pKey = taosArrayGet(p, i); + SGroupKeys* pKey = taosArrayGet(p, i); SColumnInfoData* pDstColInfoData = taosArrayGet(pBlock->pDataBlock, GET_DEST_SLOT_ID(pCol)); doSetVal(pDstColInfoData, index, pKey); } @@ -111,10 +113,10 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock *pBlock, SSData int32_t srcSlotId = GET_SRC_SLOT_ID(pCol); - int32_t dstSlotId = GET_DEST_SLOT_ID(pCol); + int32_t dstSlotId = GET_DEST_SLOT_ID(pCol); SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); - int16_t type = pCol->pExpr->base.resSchema.type; + int16_t type = pCol->pExpr->base.resSchema.type; SGroupKeys* pKey = taosArrayGet(pFillInfo->prev, i); if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pKey->isNull) { colDataAppendNULL(pDstCol, index); @@ -122,10 +124,10 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock *pBlock, SSData } SGroupKeys* pKey1 = taosArrayGet(pFillInfo->prev, 0); - int64_t prevTs = *(int64_t*)pKey1->pData; + int64_t prevTs = *(int64_t*)pKey1->pData; SColumnInfoData* pSrcCol = taosArrayGet(pSrcBlock->pDataBlock, srcSlotId); - char* data = colDataGetData(pSrcCol, pFillInfo->index); + char* data = colDataGetData(pSrcCol, pFillInfo->index); point1 = (SPoint){.key = prevTs, .val = pKey->pData}; point2 = (SPoint){.key = ts, .val = data}; @@ -139,10 +141,10 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock *pBlock, SSData } } else if (pFillInfo->type == TSDB_FILL_NULL) { // fill with NULL setNullRow(pBlock, pFillInfo->numOfCols, index); - } else { // fill with user specified value for each column + } else { // fill with user specified value for each column for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->schema.type)*/) { + if (TSDB_COL_IS_TAG(pCol->flag) /* || IS_VAR_DATA_TYPE(pCol->schema.type)*/) { continue; } @@ -165,9 +167,10 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, SSDataBlock *pBlock, SSData } } -// setTagsValue(pFillInfo, data, index); + // setTagsValue(pFillInfo, data, index); SInterval* pInterval = &pFillInfo->interval; - pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision); + pFillInfo->currentKey = + taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision); pFillInfo->numOfCurrent++; } @@ -187,12 +190,12 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo) { for (int i = 0; i < pFillInfo->numOfCols; i++) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - SGroupKeys key = {0}; + SGroupKeys key = {0}; SResSchema* pSchema = &pCol->pExpr->base.resSchema; - key.pData = taosMemoryMalloc(pSchema->bytes); + key.pData = taosMemoryMalloc(pSchema->bytes); key.isNull = true; key.bytes = pSchema->bytes; - key.type = pSchema->type; + key.type = pSchema->type; taosArrayPush(pFillInfo->next, &key); @@ -209,7 +212,7 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray SColumnInfoData* pSrcCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, srcSlotId); - bool isNull = colDataIsNull_s(pSrcCol, rowIndex); + bool isNull = colDataIsNull_s(pSrcCol, rowIndex); char* p = colDataGetData(pSrcCol, rowIndex); saveColData(pRow, i, p, isNull); } @@ -222,7 +225,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t SColumnInfoData* pTsCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, 0); int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order); - bool ascFill = FILL_IS_ASC_FILL(pFillInfo); + bool ascFill = FILL_IS_ASC_FILL(pFillInfo); #if 0 ASSERT(ascFill && (pFillInfo->currentKey >= pFillInfo->start) || (!ascFill && (pFillInfo->currentKey <= pFillInfo->start))); @@ -236,9 +239,11 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pFillInfo->next); } - if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && pFillInfo->numOfCurrent < outputRows) { + if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && + pFillInfo->numOfCurrent < outputRows) { // fill the gap between two input rows - while (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && pFillInfo->numOfCurrent < outputRows) { + while (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && + pFillInfo->numOfCurrent < outputRows) { doFillOneRowResult(pFillInfo, pBlock, pFillInfo->pSrcBlock, ts, false); } @@ -259,7 +264,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t // assign rows to dst buffer for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->schema.type)*/) { + if (TSDB_COL_IS_TAG(pCol->flag) /* || IS_VAR_DATA_TYPE(pCol->schema.type)*/) { continue; } @@ -277,7 +282,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t saveColData(pFillInfo->prev, i, src, isNull); } else { // i > 0 and data is null , do interpolation if (pFillInfo->type == TSDB_FILL_PREV) { - SGroupKeys *pKey = taosArrayGet(pFillInfo->prev, i); + SGroupKeys* pKey = taosArrayGet(pFillInfo->prev, i); doSetVal(pDst, pFillInfo->numOfCurrent, pKey); } else if (pFillInfo->type == TSDB_FILL_LINEAR) { bool isNull = colDataIsNull_s(pSrc, pFillInfo->index); @@ -286,7 +291,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t } else if (pFillInfo->type == TSDB_FILL_NULL) { colDataAppendNULL(pDst, pFillInfo->numOfCurrent); } else if (pFillInfo->type == TSDB_FILL_NEXT) { - SGroupKeys *pKey = taosArrayGet(pFillInfo->next, i); + SGroupKeys* pKey = taosArrayGet(pFillInfo->next, i); doSetVal(pDst, pFillInfo->numOfCurrent, pKey); } else { SVariant* pVar = &pFillInfo->pFillCol[i].fillVal; @@ -296,9 +301,10 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t } // set the tag value for final result -// setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent); - SInterval *pInterval = &pFillInfo->interval; - pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision); + // setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent); + SInterval* pInterval = &pFillInfo->interval; + pFillInfo->currentKey = + taosTimeAdd(pFillInfo->currentKey, pInterval->sliding * step, pInterval->slidingUnit, pInterval->precision); pFillInfo->index += 1; pFillInfo->numOfCurrent += 1; @@ -306,9 +312,9 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) { /* the raw data block is exhausted, next value does not exists */ -// if (pFillInfo->index >= pFillInfo->numOfRows) { -// taosMemoryFreeClear(*next); -// } + // if (pFillInfo->index >= pFillInfo->numOfRows) { + // taosMemoryFreeClear(*next); + // } pFillInfo->numOfTotal += pFillInfo->numOfCurrent; return pFillInfo->numOfCurrent; } @@ -318,7 +324,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t } static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull) { - SGroupKeys *pKey = taosArrayGet(rowBuf, columnIndex); + SGroupKeys* pKey = taosArrayGet(rowBuf, columnIndex); if (isNull) { pKey->isNull = true; } else { @@ -351,12 +357,12 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t int32_t k = 0; for (int32_t i = 0; i < numOfCols; ++i) { SFillColInfo* pColInfo = &pFillInfo->pFillCol[i]; - SResSchema* pSchema = &pColInfo->pExpr->base.resSchema; + SResSchema* pSchema = &pColInfo->pExpr->base.resSchema; if (TSDB_COL_IS_TAG(pColInfo->flag) || pSchema->type == TSDB_DATA_TYPE_BINARY) { numOfTags += 1; - bool exists = false; + bool exists = false; int32_t index = -1; for (int32_t j = 0; j < k; ++j) { if (pFillInfo->pTags[j].col.colId == pSchema->slotId) { @@ -369,7 +375,7 @@ static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t if (!exists) { SSchema* pSchema1 = &pFillInfo->pTags[k].col; pSchema1->colId = pSchema->slotId; - pSchema1->type = pSchema->type; + pSchema1->type = pSchema->type; pSchema1->bytes = pSchema->bytes; pFillInfo->pTags[k].tagVal = taosMemoryCalloc(1, pSchema->bytes); @@ -399,7 +405,8 @@ static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { } struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, - SInterval* pInterval, int32_t fillType, struct SFillColInfo* pCol, const char* id) { + SInterval* pInterval, int32_t fillType, struct SFillColInfo* pCol, + const char* id) { if (fillType == TSDB_FILL_NONE) { return NULL; } @@ -413,32 +420,44 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag taosResetFillInfo(pFillInfo, skey); pFillInfo->order = order; - switch(fillType) { - case FILL_MODE_NONE: pFillInfo->type = TSDB_FILL_NONE; break; - case FILL_MODE_PREV: pFillInfo->type = TSDB_FILL_PREV; break; - case FILL_MODE_NULL: pFillInfo->type = TSDB_FILL_NULL; break; - case FILL_MODE_LINEAR: pFillInfo->type = TSDB_FILL_LINEAR;break; - case FILL_MODE_NEXT: pFillInfo->type = TSDB_FILL_NEXT; break; - case FILL_MODE_VALUE: pFillInfo->type = TSDB_FILL_SET_VALUE; break; + switch (fillType) { + case FILL_MODE_NONE: + pFillInfo->type = TSDB_FILL_NONE; + break; + case FILL_MODE_PREV: + pFillInfo->type = TSDB_FILL_PREV; + break; + case FILL_MODE_NULL: + pFillInfo->type = TSDB_FILL_NULL; + break; + case FILL_MODE_LINEAR: + pFillInfo->type = TSDB_FILL_LINEAR; + break; + case FILL_MODE_NEXT: + pFillInfo->type = TSDB_FILL_NEXT; + break; + case FILL_MODE_VALUE: + pFillInfo->type = TSDB_FILL_SET_VALUE; + break; default: terrno = TSDB_CODE_INVALID_PARA; return NULL; } - pFillInfo->type = fillType; - pFillInfo->pFillCol = pCol; + pFillInfo->type = fillType; + pFillInfo->pFillCol = pCol; pFillInfo->numOfTags = numOfTags; pFillInfo->numOfCols = numOfCols; - pFillInfo->alloc = capacity; - pFillInfo->id = id; - pFillInfo->interval = *pInterval; - -// if (numOfTags > 0) { - pFillInfo->pTags = taosMemoryCalloc(numOfCols, sizeof(SFillTagColInfo)); - for (int32_t i = 0; i < numOfCols; ++i) { - pFillInfo->pTags[i].col.colId = -2; // TODO - } -// } + pFillInfo->alloc = capacity; + pFillInfo->id = id; + pFillInfo->interval = *pInterval; + + // if (numOfTags > 0) { + pFillInfo->pTags = taosMemoryCalloc(numOfCols, sizeof(SFillTagColInfo)); + for (int32_t i = 0; i < numOfCols; ++i) { + pFillInfo->pTags[i].col.colId = -2; // TODO + } + // } pFillInfo->next = taosArrayInit(numOfCols, sizeof(SGroupKeys)); pFillInfo->prev = taosArrayInit(numOfCols, sizeof(SGroupKeys)); @@ -451,13 +470,13 @@ struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTag } void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) { - pFillInfo->start = startTimestamp; - pFillInfo->currentKey = startTimestamp; - pFillInfo->end = startTimestamp; - pFillInfo->index = -1; - pFillInfo->numOfRows = 0; + pFillInfo->start = startTimestamp; + pFillInfo->currentKey = startTimestamp; + pFillInfo->end = startTimestamp; + pFillInfo->index = -1; + pFillInfo->numOfRows = 0; pFillInfo->numOfCurrent = 0; - pFillInfo->numOfTotal = 0; + pFillInfo->numOfTotal = 0; } void* taosDestroyFillInfo(SFillInfo* pFillInfo) { @@ -468,7 +487,7 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) { taosArrayDestroy(pFillInfo->prev); taosArrayDestroy(pFillInfo->next); - for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) { + for (int32_t i = 0; i < pFillInfo->numOfTags; ++i) { taosMemoryFreeClear(pFillInfo->pTags[i].tagVal); } @@ -488,12 +507,12 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->interval.precision); } - pFillInfo->index = 0; + pFillInfo->index = 0; pFillInfo->numOfRows = numOfRows; } void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) { - pFillInfo->pSrcBlock = (SSDataBlock*) pInput; + pFillInfo->pSrcBlock = (SSDataBlock*)pInput; } bool taosFillHasMoreResults(SFillInfo* pFillInfo) { @@ -513,8 +532,8 @@ bool taosFillHasMoreResults(SFillInfo* pFillInfo) { int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) { SColumnInfoData* pCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, 0); - int64_t* tsList = (int64_t*) pCol->pData; - int32_t numOfRows = taosNumOfRemainRows(pFillInfo); + int64_t* tsList = (int64_t*)pCol->pData; + int32_t numOfRows = taosNumOfRemainRows(pFillInfo); TSKEY ekey1 = ekey; if (!FILL_IS_ASC_FILL(pFillInfo)) { @@ -524,32 +543,25 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma int64_t numOfRes = -1; if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set. TSKEY lastKey = tsList[pFillInfo->numOfRows - 1]; - numOfRes = taosTimeCountInterval( - lastKey, - pFillInfo->currentKey, - pFillInfo->interval.sliding, - pFillInfo->interval.slidingUnit, - pFillInfo->interval.precision); + numOfRes = taosTimeCountInterval(lastKey, pFillInfo->currentKey, pFillInfo->interval.sliding, + pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); numOfRes += 1; assert(numOfRes >= numOfRows); - } else { // reach the end of data + } else { // reach the end of data if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { return 0; } - numOfRes = taosTimeCountInterval( - ekey1, - pFillInfo->currentKey, - pFillInfo->interval.sliding, - pFillInfo->interval.slidingUnit, - pFillInfo->interval.precision); + numOfRes = taosTimeCountInterval(ekey1, pFillInfo->currentKey, pFillInfo->interval.sliding, + pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); numOfRes += 1; } return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes; } -int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType) { +int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, + int32_t inputType) { double v1 = -1, v2 = -1; GET_TYPED_DATA(v1, double, inputType, point1->val); GET_TYPED_DATA(v2, double, inputType, point2->val); @@ -570,20 +582,20 @@ int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, SSDataBlock* p, int32_t ca if (remain == 0) { appendFilledResult(pFillInfo, p, numOfRes); } else { - fillResultImpl(pFillInfo, p, (int32_t) numOfRes); + fillResultImpl(pFillInfo, p, (int32_t)numOfRes); assert(numOfRes == pFillInfo->numOfCurrent); } -// qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%"PRId64"-%"PRId64", currentKey:%"PRId64", current:%d, total:%d, %p", -// pFillInfo, pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey, pFillInfo->numOfCurrent, -// pFillInfo->numOfTotal, pFillInfo->handle); + // qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%"PRId64"-%"PRId64", currentKey:%"PRId64", + // current:%d, total:%d, %p", + // pFillInfo, pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey, + // pFillInfo->numOfCurrent, + // pFillInfo->numOfTotal, pFillInfo->handle); return numOfRes; } -int64_t getFillInfoStart(struct SFillInfo *pFillInfo) { - return pFillInfo->start; -} +int64_t getFillInfoStart(struct SFillInfo* pFillInfo) { return pFillInfo->start; } SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const struct SNodeListNode* pValNode) { SFillColInfo* pFillCol = taosMemoryCalloc(numOfOutput, sizeof(SFillColInfo)); @@ -591,8 +603,8 @@ SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const str return NULL; } - size_t len = (pValNode != NULL)? LIST_LENGTH(pValNode->pNodeList):0; - for(int32_t i = 0; i < numOfOutput; ++i) { + size_t len = (pValNode != NULL) ? LIST_LENGTH(pValNode->pNodeList) : 0; + for (int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExprInfo = &pExpr[i]; pFillCol[i].pExpr = pExprInfo; pFillCol[i].tagIndex = -2; @@ -600,14 +612,14 @@ SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const str // todo refactor if (len > 0) { // if the user specified value is less than the column, alway use the last one as the fill value - int32_t index = (i >= len)? (len - 1):i; + int32_t index = (i >= len) ? (len - 1) : i; SValueNode* pv = (SValueNode*)nodesListGetNode(pValNode->pNodeList, index); - valueNodeToVariant(pv, &pFillCol[i].fillVal); + nodesValueNodeToVariant(pv, &pFillCol[i].fillVal); } if (pExprInfo->base.numOfParams > 0) { - pFillCol[i].flag = pExprInfo->base.pParam[0].pCol->flag; // always be the normal column for table query + pFillCol[i].flag = pExprInfo->base.pParam[0].pCol->flag; // always be the normal column for table query } } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 14344daf8150799683bb8d5fac605347d611819e..b621d729e03a31175b2f06069fa4ffdea4145f9a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1,4 +1,5 @@ #include "executorimpl.h" +#include "function.h" #include "functionMgt.h" #include "tdatablock.h" #include "ttime.h" @@ -9,7 +10,12 @@ typedef enum SResultTsInterpType { } SResultTsInterpType; static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator); -static SSDataBlock* doStreamSessionWindowAgg(SOperatorInfo* pOperator); +static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator); + +static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo); + +static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult); +static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult); /* * There are two cases to handle: @@ -21,48 +27,10 @@ static SSDataBlock* doStreamSessionWindowAgg(SOperatorInfo* pOperator); * is a previous result generated or not. */ static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) { - // SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; - // if (pResultRowInfo->curPos != -1) { - // return; - // } - - // pTableQueryInfo->win.skey = key; - // STimeWindow win = {.skey = key, .ekey = pQRange->ekey}; - - /** - * In handling the both ascending and descending order super table query, we need to find the first qualified - * timestamp of this table, and then set the first qualified start timestamp. - * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional - * operations involve. - */ - // STimeWindow w = TSWINDOW_INITIALIZER; - // - // TSKEY sk = TMIN(win.skey, win.ekey); - // TSKEY ek = TMAX(win.skey, win.ekey); - // getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); - - // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { - // if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { - // assert(win.ekey == pQueryAttr->window.ekey); - // } - // - // pResultRowInfo->prevSKey = w.skey; - // } - - // pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; + // do nothing } -static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows, bool ascQuery) { - TSKEY ts = TSKEY_INITIAL_VAL; - if (tsCols == NULL) { - ts = ascQuery ? win->skey : win->ekey; - } else { -// int32_t offset = ascQuery ? 0 : rows - 1; - ts = tsCols[0]; - } - - return ts; -} +static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; } static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, bool ascQuery) { @@ -85,8 +53,8 @@ static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, T } // get the correct time window according to the handled timestamp -STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, - SInterval* pInterval, int32_t precision, STimeWindow* win) { +STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, + int32_t precision, STimeWindow* win) { STimeWindow w = {0}; if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value @@ -134,8 +102,10 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo // set time window for current result pResultRow->win = (*win); + *pResult = pResultRow; setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); + return TSDB_CODE_SUCCESS; } @@ -163,38 +133,38 @@ static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsL static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, int16_t order, int64_t* pData) { - int32_t forwardStep = 0; + int32_t forwardRows = 0; if (order == TSDB_ORDER_ASC) { int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order); if (end >= 0) { - forwardStep = end; + forwardRows = end; if (pData[end + pos] == ekey) { - forwardStep += 1; + forwardRows += 1; } } } else { int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order); if (end >= 0) { - forwardStep = end; + forwardRows = end; if (pData[end + pos] == ekey) { - forwardStep += 1; + forwardRows += 1; } } -// int32_t end = searchFn((char*)pData, pos + 1, ekey, order); -// if (end >= 0) { -// forwardStep = pos - end; -// -// if (pData[end] == ekey) { -// forwardStep += 1; -// } -// } + // int32_t end = searchFn((char*)pData, pos + 1, ekey, order); + // if (end >= 0) { + // forwardRows = pos - end; + // + // if (pData[end] == ekey) { + // forwardRows += 1; + // } + // } } - assert(forwardStep >= 0); - return forwardStep; + assert(forwardRows >= 0); + return forwardRows; } int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { @@ -268,9 +238,8 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { return midPos; } -int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, - TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, - int32_t order) { +int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, + __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) { assert(startPos >= 0 && startPos < pDataBlockInfo->rows); int32_t num = -1; @@ -339,34 +308,39 @@ static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t o tw->ekey -= 1; } -void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, +void doTimeWindowInterpolation(SIntervalAggOperatorInfo* pInfo, int32_t numOfExprs, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { - SExprInfo* pExpr = pOperator->pExpr; - - SqlFunctionCtx* pCtx = pInfo->pCtx; + SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; - for (int32_t k = 0; k < pOperator->numOfExprs; ++k) { - int32_t functionId = pCtx[k].functionId; - if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { + int32_t index = 1; + for (int32_t k = 0; k < numOfExprs; ++k) { + // todo use flag instead of function name + if (strcmp(pCtx[k].pExpr->pExpr->_function.functionName, "twa") != 0) { pCtx[k].start.key = INT64_MIN; continue; } - SColIndex* pColIndex = NULL /*&pExpr[k].base.colInfo*/; - int16_t index = pColIndex->colIndex; - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, index); + // if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { + // pCtx[k].start.key = INT64_MIN; + // continue; + // } - // assert(pColInfo->info.colId == pColIndex->info.colId && curTs != windowKey); - double v1 = 0, v2 = 0, v = 0; + SFunctParam* pParam = &pCtx[k].param[0]; + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId); + + ASSERT(pColInfo->info.colId == pParam->pCol->colId && curTs != windowKey); + double v1 = 0, v2 = 0, v = 0; if (prevRowIndex == -1) { - // GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pRuntimeEnv->prevRow[index]); + SGroupKeys* p = taosArrayGet(pInfo->pPrevValues, index); + GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData); } else { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); + GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex)); } - GET_TYPED_DATA(v2, double, pColInfo->info.type, (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes); + GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex)); +#if 0 if (functionId == FUNCTION_INTERP) { if (type == RESULT_ROW_START_INTERP) { pCtx[k].start.key = prevTs; @@ -386,21 +360,27 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, } } } else if (functionId == FUNCTION_TWA) { - SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; - SPoint point2 = (SPoint){.key = curTs, .val = &v2}; - SPoint point = (SPoint){.key = windowKey, .val = &v}; +#endif - taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; + SPoint point2 = (SPoint){.key = curTs, .val = &v2}; + SPoint point = (SPoint){.key = windowKey, .val = &v}; - if (type == RESULT_ROW_START_INTERP) { - pCtx[k].start.key = point.key; - pCtx[k].start.val = v; - } else { - pCtx[k].end.key = point.key; - pCtx[k].end.val = v; - } + taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = point.key; + pCtx[k].start.val = v; + } else { + pCtx[k].end.key = point.key; + pCtx[k].end.val = v; } + + index += 1; } +#if 0 + } +#endif } static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { @@ -415,69 +395,67 @@ static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, in } } -static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos, - int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, - STimeWindow* win) { - bool ascQuery = true; +static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfExprs, + int32_t pos, SSDataBlock* pBlock, const TSKEY* tsCols, STimeWindow* win) { + bool ascQuery = (pInfo->order == TSDB_ORDER_ASC); + TSKEY curTs = tsCols[pos]; - TSKEY lastTs = 0; //*(TSKEY*)pRuntimeEnv->prevRow[0]; + + SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0); + TSKEY lastTs = *(int64_t*)pTsKey->pData; // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. // start exactly from this point, no need to do interpolation TSKEY key = ascQuery ? win->skey : win->ekey; if (key == curTs) { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_START_INTERP); + setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_START_INTERP); return true; } - if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_START_INTERP); - return true; + // it is the first time window, no need to do interpolation + if (pTsKey->isNull && pos == 0) { + setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_START_INTERP); + } else { + TSKEY prevTs = ((pos == 0) ? lastTs : tsCols[pos - 1]); + doTimeWindowInterpolation(pInfo, numOfExprs, pBlock->pDataBlock, prevTs, pos - 1, curTs, pos, key, + RESULT_ROW_START_INTERP); } - int32_t step = 1; // GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery)) ? lastTs : tsCols[pos - step]; - - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, - RESULT_ROW_START_INTERP); return true; } -static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex, - SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, - STimeWindow* win) { - int32_t order = TSDB_ORDER_ASC; - int32_t numOfOutput = pOperatorInfo->numOfExprs; +static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfExprs, + int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, + TSKEY blockEkey, STimeWindow* win) { + int32_t order = pInfo->order; TSKEY actualEndKey = tsCols[endRowIndex]; - TSKEY key = order ? win->ekey : win->skey; + TSKEY key = (order == TSDB_ORDER_ASC) ? win->ekey : win->skey; // not ended in current data block, do not invoke interpolation - if ((key > blockEkey /*&& QUERY_IS_ASC_QUERY(pQueryAttr)*/) || - (key < blockEkey /*&& !QUERY_IS_ASC_QUERY(pQueryAttr)*/)) { - setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + if ((key > blockEkey && (order == TSDB_ORDER_ASC)) || (key < blockEkey && (order == TSDB_ORDER_DESC))) { + setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_END_INTERP); return false; } - // there is actual end point of current time window, no interpolation need + // there is actual end point of current time window, no interpolation needs if (key == actualEndKey) { - setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_END_INTERP); return true; } - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - int32_t nextRowIndex = endRowIndex + step; + int32_t nextRowIndex = endRowIndex + 1; assert(nextRowIndex >= 0); TSKEY nextKey = tsCols[nextRowIndex]; - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey, - nextRowIndex, key, RESULT_ROW_END_INTERP); + doTimeWindowInterpolation(pInfo, numOfExprs, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key, + RESULT_ROW_END_INTERP); return true; } static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, TSKEY* primaryKeys, int32_t prevPosition, int32_t order) { - bool ascQuery = (order == TSDB_ORDER_ASC); + bool ascQuery = (order == TSDB_ORDER_ASC); int32_t precision = pInterval->precision; getNextTimeWindow(pInterval, precision, order, pNext); @@ -542,8 +520,8 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, return startPos; } -static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { - assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); +static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { + ASSERT(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); if (type == RESULT_ROW_START_INTERP) { return pResult->startInterp == true; } else { @@ -560,34 +538,30 @@ static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { } } -static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SqlFunctionCtx* pCtx, - SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep, - int32_t order, bool timeWindowInterpo) { - if (!timeWindowInterpo) { +static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t numOfExprs, + SqlFunctionCtx* pCtx, SResultRow* pResult, STimeWindow* win, int32_t startPos, + int32_t forwardRows) { + if (!pInfo->timeWindowInterpo) { return; } - assert(pBlock != NULL); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - + ASSERT(pBlock != NULL); if (pBlock->pDataBlock == NULL) { // tscError("pBlock->pDataBlock == NULL"); return; } - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); TSKEY* tsCols = (TSKEY*)(pColInfo->pData); - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + bool done = isResultRowInterpolated(pResult, RESULT_ROW_START_INTERP); if (!done) { // it is not interpolated, now start to generated the interpolated value - int32_t startRowIndex = startPos; - bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, - pBlock->pDataBlock, tsCols, win); + bool interp = setTimeWindowInterpolationStartTs(pInfo, pCtx, numOfExprs, startPos, pBlock, tsCols, win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); } } else { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_START_INTERP); + setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_START_INTERP); } // point interpolation does not require the end key time window interpolation. @@ -596,38 +570,116 @@ static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBloc // } // interpolation query does not generate the time window end interpolation - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + done = isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP); if (!done) { - int32_t endRowIndex = startPos + (forwardStep - 1) * step; + int32_t endRowIndex = startPos + forwardRows - 1; - TSKEY endKey = (order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey; + TSKEY endKey = (pInfo->order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey; bool interp = - setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); + setTimeWindowInterpolationEndTs(pInfo, pCtx, numOfExprs, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); } } else { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_END_INTERP); } } -static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowIndex, int32_t numOfCols) { - if (pDataBlock == NULL) { +static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, SArray* pCols) { + if (pBlock->pDataBlock == NULL) { return; } - for (int32_t k = 0; k < numOfCols; ++k) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, k); - memcpy(pRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); + size_t num = taosArrayGetSize(pPrevKeys); + for (int32_t k = 0; k < num; ++k) { + SColumn* pc = taosArrayGet(pCols, k); + + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pc->slotId); + + SGroupKeys* pkey = taosArrayGet(pPrevKeys, k); + for (int32_t i = pBlock->info.rows - 1; i >= 0; --i) { + if (colDataIsNull_s(pColInfo, i)) { + continue; + } + + char* val = colDataGetData(pColInfo, i); + if (IS_VAR_DATA_TYPE(pkey->type)) { + memcpy(pkey->pData, val, varDataTLen(val)); + ASSERT(varDataTLen(val) <= pkey->bytes); + } else { + memcpy(pkey->pData, val, pkey->bytes); + } + + break; + } + } +} + +static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t numOfExprs, SResultRowInfo* pResultRowInfo, + SSDataBlock* pBlock, int32_t scanFlag, int64_t* tsCols, SResultRowPosition* p) { + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + + SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; + + int32_t startPos = 0; + int32_t numOfOutput = pOperatorInfo->numOfExprs; + uint64_t groupId = pBlock->info.groupId; + + SResultRow* pResult = NULL; + + while (1) { + SListNode* pn = tdListGetHead(pResultRowInfo->openWindow); + + SResultRowPosition* p1 = (SResultRowPosition*)pn->data; + if (p->pageId == p1->pageId && p->offset == p1->offset) { + break; + } + + SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1); + ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId); + + if (pr->closed) { + ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) && + isResultRowInterpolated(pr, RESULT_ROW_END_INTERP)); + tdListPopHead(pResultRowInfo->openWindow); + continue; + } + + STimeWindow w = pr->win; + int32_t ret = + setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + + SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0); + int64_t prevTs = *(int64_t*)pTsKey->pData; + doTimeWindowInterpolation(pInfo, numOfOutput, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey, + RESULT_ROW_END_INTERP); + + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pInfo->binfo.pCtx, numOfExprs, RESULT_ROW_START_INTERP); + + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &w, &pInfo->twAggSup.timeWindowData, startPos, 0, tsCols, + pBlock->info.rows, numOfExprs, pInfo->order); + + if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) { + closeResultRow(pr); + tdListPopHead(pResultRowInfo->openWindow); + } else { // the remains are can not be closed yet. + break; + } } } typedef int64_t (*__get_value_fn_t)(void* data, int32_t index); -int32_t binarySearch(void* keyList, int num, TSKEY key, int order, - __get_value_fn_t getValuefn) { - int firstPos = 0, lastPos = num - 1, midPos = -1; - int numOfRows = 0; +int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_fn_t getValuefn) { + int firstPos = 0, lastPos = num - 1, midPos = -1; + int numOfRows = 0; if (num <= 0) return -1; if (order == TSDB_ORDER_DESC) { @@ -680,7 +732,7 @@ int32_t binarySearch(void* keyList, int num, TSKEY key, int order, } int64_t getReskey(void* data, int32_t index) { - SArray* res = (SArray*) data; + SArray* res = (SArray*)data; SResKeyPos* pos = taosArrayGetP(res, index); return *(int64_t*)pos->key; } @@ -698,7 +750,7 @@ static int32_t saveResult(SResultRow* result, uint64_t groupId, SArray* pUpdated return TSDB_CODE_SUCCESS; } } - + SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); if (newPos == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -706,44 +758,32 @@ static int32_t saveResult(SResultRow* result, uint64_t groupId, SArray* pUpdated newPos->groupId = groupId; newPos->pos = (SResultRowPosition){.pageId = result->pageId, .offset = result->offset}; *(int64_t*)newPos->key = result->win.skey; - if (taosArrayInsert(pUpdated, index, &newPos) == NULL ){ + if (taosArrayInsert(pUpdated, index, &newPos) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } return TSDB_CODE_SUCCESS; } static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, - uint64_t tableGroupId, SArray* pUpdated) { + int32_t scanFlag, SArray* pUpdated) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - int32_t numOfOutput = pOperatorInfo->numOfExprs; - - int32_t step = 1; - bool ascScan = (pInfo->order == TSDB_ORDER_ASC); - - // int32_t prevIndex = pResultRowInfo->curPos; - - TSKEY* tsCols = NULL; - if (pBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); - tsCols = (int64_t*)pColDataInfo->pData; - - if (tsCols != NULL) { - blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex); - } - } - int32_t startPos = 0; - TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols, pBlock->info.rows, ascScan); + int32_t startPos = 0; + int32_t numOfOutput = pOperatorInfo->numOfExprs; + int64_t* tsCols = extractTsCol(pBlock, pInfo); + uint64_t tableGroupId = pBlock->info.groupId; + bool ascScan = (pInfo->order == TSDB_ORDER_ASC); + TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); + SResultRow* pResult = NULL; STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, &pInfo->win); - bool masterScan = true; - SResultRow* pResult = NULL; - int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t ret = + setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -758,72 +798,46 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } } - int32_t forwardStep = 0; - TSKEY ekey = ascScan? win.ekey:win.skey; - forwardStep = + TSKEY ekey = ascScan ? win.ekey : win.skey; + int32_t forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); - ASSERT(forwardStep > 0); + ASSERT(forwardRows > 0); // prev time window not interpolation yet. - // int32_t curIndex = pResultRowInfo->curPos; - -#if 0 - if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) { - for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. - SResultRow* pRes = getResultRow(pResultRowInfo, j); - if (pRes->closed) { - assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); - continue; - } - - STimeWindow w = pRes->win; - ret = setTimeWindowOutputBuf(pResultRowInfo, pBlock->info.uid, &w, masterScan, &pResult, tableGroupId, - pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, - pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - doTimeWindowInterpolation(pOperatorInfo, &pInfo->binfo, pBlock->pDataBlock, *(TSKEY*)pInfo->pRow[0], -1, - tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); - - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pInfo->binfo.pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_START_INTERP); - - doApplyFunctions(pInfo->binfo.pCtx, &w, &pInfo->timeWindowData, startPos, 0, tsCols, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - } + if (pInfo->timeWindowInterpo) { + SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult); + doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos); // restore current time window - ret = setTimeWindowOutputBuf(pResultRowInfo, pBlock->info.uid, &win, masterScan, &pResult, tableGroupId, - pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, - pTaskInfo); + ret = + setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - } -#endif - // window start key interpolation - doWindowBorderInterpolation(pOperatorInfo, pBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, - pInfo->order, false); + // window start key interpolation + doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &win, startPos, forwardRows); + } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); - doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, - pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, + pBlock->info.rows, numOfOutput, pInfo->order); + + doCloseWindow(pResultRowInfo, pInfo, pResult); STimeWindow nextWin = win; while (1) { - int32_t prevEndPos = (forwardStep - 1) * step + startPos; + int32_t prevEndPos = forwardRows - 1 + startPos; startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->order); if (startPos < 0) { break; } // null data, failed to allocate more memory buffer - int32_t code = - setTimeWindowOutputBuf(pResultRowInfo, &nextWin, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, + &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -838,25 +852,61 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } } - ekey = ascScan? nextWin.ekey:nextWin.skey; - forwardStep = + ekey = ascScan ? nextWin.ekey : nextWin.skey; + forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, - pInfo->order, false); + doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, + forwardRows); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, - pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, + tsCols, pBlock->info.rows, numOfOutput, pInfo->order); + doCloseWindow(pResultRowInfo, pInfo, pResult); } if (pInfo->timeWindowInterpo) { - int32_t rowIndex = ascScan ? (pBlock->info.rows - 1) : 0; - saveDataBlockLastRow(pInfo->pRow, pBlock->pDataBlock, rowIndex, pBlock->info.numOfCols); + saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols); } +} - // updateResultRowInfoActiveIndex(pResultRowInfo, &pInfo->win, pRuntimeEnv->current->lastKey, true, false); +void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) { + // current result is done in computing final results. + if (pInfo->timeWindowInterpo && isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) { + closeResultRow(pResult); + tdListPopHead(pResultRowInfo->openWindow); + } +} + +SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult) { + SResultRowPosition pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + SListNode* pn = tdListGetTail(pResultRowInfo->openWindow); + if (pn == NULL) { + tdListAppend(pResultRowInfo->openWindow, &pos); + return pos; + } + + SResultRowPosition* px = (SResultRowPosition*)pn->data; + if (px->pageId != pos.pageId || px->offset != pos.offset) { + tdListAppend(pResultRowInfo->openWindow, &pos); + } + + return pos; +} + +int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) { + TSKEY* tsCols = NULL; + if (pBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + + if (tsCols != NULL) { + blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex); + } + } + + return tsCols; } static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { @@ -869,7 +919,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { int32_t scanFlag = MAIN_SCAN; - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -885,7 +935,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId, NULL); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL); #if 0 // test for encode/decode result info if(pOperator->fpSet.encodeResultRow){ @@ -941,7 +991,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI struct SColumnDataAgg* pAgg = NULL; for (int32_t j = 0; j < pBlock->info.rows; ++j) { - pAgg = (pBlock->pBlockAgg != NULL)? pBlock->pBlockAgg[pInfo->stateCol.slotId]: NULL; + pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL; if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) { continue; } @@ -1016,8 +1066,9 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { } SStateWindowOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SOptrBasicInfo* pBInfo = &pInfo->binfo; + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); @@ -1045,7 +1096,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { doStateWindowAggImpl(pOperator, pInfo, pBlock); } - pOperator->cost.openCost = (taosGetTimestampUs() - st)/1000.0; + pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); @@ -1060,7 +1111,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { size_t rows = pBInfo->pRes->info.rows; pOperator->resultInfo.totalRows += rows; - return (rows == 0)? NULL : pBInfo->pRes; + return (rows == 0) ? NULL : pBInfo->pRes; } static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { @@ -1091,7 +1142,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { size_t rows = pBlock->info.rows; pOperator->resultInfo.totalRows += rows; - return (rows == 0)? NULL:pBlock; + return (rows == 0) ? NULL : pBlock; } } @@ -1126,9 +1177,8 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type } } -void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, - SOptrBasicInfo* pBinfo, int32_t numOfOutput) { - SResultRow* pResult = getResultRowByPos(pResultBuf, p1); +void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SOptrBasicInfo* pBinfo, int32_t numOfOutput) { + SResultRow* pResult = getResultRowByPos(pResultBuf, p1); SqlFunctionCtx* pCtx = pBinfo->pCtx; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultCell(pResult, i, pBinfo->rowCellInfoOffset); @@ -1143,28 +1193,24 @@ void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, } } -void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData, - int16_t bytes, uint64_t groupId, int32_t numOfOutput) { +void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData, int16_t bytes, uint64_t groupId, + int32_t numOfOutput) { SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = - (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, - GET_RES_WINDOW_KEY_LEN(bytes)); + (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); doClearWindowImpl(p1, pSup->pResultBuf, pBinfo, numOfOutput); } -static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, - SInterval* pInterval, int32_t tsIndex, int32_t numOfOutput, SSDataBlock* pBlock, - SArray* pUpWins) { +static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, SInterval* pInterval, int32_t tsIndex, + int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) { SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); - TSKEY *tsCols = (TSKEY*)pColDataInfo->pData; - int32_t step = 0; + TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; + int32_t step = 0; for (int32_t i = 0; i < pBlock->info.rows; i += step) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, - pInterval->precision, NULL); - step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, - win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, NULL); + step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); doClearWindow(pSup, pBinfo, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput); if (pUpWins) { taosArrayPush(pUpWins, &win); @@ -1172,19 +1218,18 @@ static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, } } -static int32_t closeIntervalWindow(SHashObj *pHashMap, STimeWindowAggSupp *pSup, - SInterval* pInterval, SArray* closeWins) { - void *pIte = NULL; +static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, + SArray* closeWins) { + void* pIte = NULL; size_t keyLen = 0; - while((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { - void* key = taosHashGetKey(pIte, &keyLen); - uint64_t groupId = *(uint64_t*) key; + while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { + void* key = taosHashGetKey(pIte, &keyLen); + uint64_t groupId = *(uint64_t*)key; ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); - TSKEY ts = *(int64_t*) ((char*)key + sizeof(uint64_t)); + TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, - pInterval->precision, NULL); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL); if (win.ekey < pSup->maxTs - pSup->waterMark) { if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE_SMA) { if (taosHashGet(pSup->winMap, &win.skey, sizeof(TSKEY))) { @@ -1193,8 +1238,7 @@ static int32_t closeIntervalWindow(SHashObj *pHashMap, STimeWindowAggSupp *pSup, } char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId); - if (pSup->calTrigger != STREAM_TRIGGER_AT_ONCE_SMA && - pSup->calTrigger != STREAM_TRIGGER_WINDOW_CLOSE_SMA) { + if (pSup->calTrigger != STREAM_TRIGGER_AT_ONCE_SMA && pSup->calTrigger != STREAM_TRIGGER_WINDOW_CLOSE_SMA) { taosHashRemove(pHashMap, keyBuf, keyLen); } SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); @@ -1202,7 +1246,7 @@ static int32_t closeIntervalWindow(SHashObj *pHashMap, STimeWindowAggSupp *pSup, return TSDB_CODE_OUT_OF_MEMORY; } pos->groupId = groupId; - pos->pos = *(SResultRowPosition*) pIte; + pos->pos = *(SResultRowPosition*)pIte; *(int64_t*)pos->key = ts; if (!taosArrayPush(closeWins, &pos)) { taosMemoryFree(pos); @@ -1216,7 +1260,7 @@ static int32_t closeIntervalWindow(SHashObj *pHashMap, STimeWindowAggSupp *pSup, static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SIntervalAggOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; pInfo->order = TSDB_ORDER_ASC; @@ -1243,7 +1287,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { break; } - // The timewindows that overlaps the timestamps of the input pBlock need to be recalculated and return to the + // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the // caller. Note that all the time window are not close till now. // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true); @@ -1252,26 +1296,24 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } if (pBlock->info.type == STREAM_REPROCESS) { - doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, 0, - pOperator->numOfExprs, pBlock, NULL); + doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, 0, pOperator->numOfExprs, pBlock, NULL); qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); continue; } - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId, pUpdated); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated); } - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, - &pInfo->interval, pClosed); - finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pClosed, - pInfo->binfo.rowCellInfoOffset); + + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pClosed); + finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pClosed, pInfo->binfo.rowCellInfoOffset); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE || pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE_SMA) { taosArrayAddAll(pUpdated, pClosed); } + taosArrayDestroy(pClosed); - finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, - pInfo->binfo.rowCellInfoOffset); + finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); @@ -1295,7 +1337,7 @@ void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { } void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { - SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo *)param; + SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); cleanupAggSup(&pInfo->aggSup); if (pInfo->pChildren) { @@ -1309,7 +1351,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { } } -bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { +static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { for (int32_t i = 0; i < numOfCols; i++) { if (!fmIsInvertible(pFCtx[i].functionId)) { return false; @@ -1318,6 +1360,50 @@ bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { return true; } +static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) { + // the primary timestamp column + bool needed = false; + pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys)); + + { // ts column + SColumn c = {0}; + c.colId = 1; + c.slotId = pInfo->primaryTsIndex; + c.type = TSDB_DATA_TYPE_TIMESTAMP; + c.bytes = sizeof(int64_t); + taosArrayPush(pInfo->pInterpCols, &c); + + SGroupKeys key = {0}; + key.bytes = c.bytes; + key.type = c.type; + key.isNull = true; // to denote no value is assigned yet + key.pData = taosMemoryCalloc(1, c.bytes); + taosArrayPush(pInfo->pPrevValues, &key); + } + + for (int32_t i = 0; i < numOfCols; ++i) { + SExprInfo* pExpr = pCtx[i].pExpr; + + if (strcmp(pExpr->pExpr->_function.functionName, "twa") == 0) { + SFunctParam* pParam = &pExpr->base.pParam[0]; + + SColumn c = *pParam->pCol; + taosArrayPush(pInfo->pInterpCols, &c); + needed = true; + + SGroupKeys key = {0}; + key.bytes = c.bytes; + key.type = c.type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, c.bytes); + taosArrayPush(pInfo->pPrevValues, &key); + } + } + + return needed; +} + SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { @@ -1327,11 +1413,12 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } + pInfo->win = pTaskInfo->window; pInfo->order = TSDB_ORDER_ASC; pInfo->interval = *pInterval; pInfo->execModel = pTaskInfo->execModel; - pInfo->win = pTaskInfo->window; pInfo->twAggSup = *pTwAggSupp; + pInfo->primaryTsIndex = primaryTsSlotId; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; @@ -1341,17 +1428,24 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); + pInfo->invertible = allInvertible(pInfo->binfo.pCtx, numOfCols); - pInfo->invertible = false; // Todo(liuyao): Dependent TSDB API + pInfo->invertible = false; // Todo(liuyao): Dependent TSDB API - if (code != TSDB_CODE_SUCCESS) { + pInfo->timeWindowInterpo = timeWindowinterpNeeded(pInfo->binfo.pCtx, numOfCols, pInfo); + if (pInfo->timeWindowInterpo) { + pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition)); + } + + // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { goto _error; } initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); pOperator->name = "TimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->pExpr = pExprInfo; @@ -1378,10 +1472,11 @@ _error: } SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { + SSDataBlock* pResBlock, SInterval* pInterval, + int32_t primaryTsSlotId, STimeWindowAggSupp* pTwAggSupp, + SExecTaskInfo* pTaskInfo) { SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -1392,19 +1487,18 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, 4096); int32_t code = - initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, - keyBufSize, pTaskInfo->id.str); + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); if (code != TSDB_CODE_SUCCESS) { goto _error; } initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - int32_t numOfChild = 8;// Todo(liuyao) get it from phy plan + int32_t numOfChild = 8; // Todo(liuyao) get it from phy plan pInfo->pChildren = taosArrayInit(numOfChild, sizeof(SOperatorInfo)); for (int32_t i = 0; i < numOfChild; i++) { - SSDataBlock* chRes = createOneDataBlock(pResBlock, false); - SOperatorInfo* pChildOp = createIntervalOperatorInfo(NULL, pExprInfo, numOfCols, - chRes, pInterval, primaryTsSlotId, pTwAggSupp, pTaskInfo); + SSDataBlock* chRes = createOneDataBlock(pResBlock, false); + SOperatorInfo* pChildOp = createIntervalOperatorInfo(NULL, pExprInfo, numOfCols, chRes, pInterval, primaryTsSlotId, + pTwAggSupp, pTaskInfo); if (pChildOp && chRes) { taosArrayPush(pInfo->pChildren, &pChildOp); continue; @@ -1421,9 +1515,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->numOfExprs = numOfCols; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, - destroyStreamFinalIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, - NULL); + pOperator->fpSet = + createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -1471,7 +1565,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExpr initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); pOperator->name = "StreamTimeIntervalAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->pExpr = pExprInfo; @@ -1620,7 +1714,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { size_t rows = pBInfo->pRes->info.rows; pOperator->resultInfo.totalRows += rows; - return (rows == 0)? NULL : pBInfo->pRes; + return (rows == 0) ? NULL : pBInfo->pRes; } static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator) { @@ -1726,15 +1820,15 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf pInfo->twAggSup = *pTwAggSup; initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - pInfo->tsSlotId = tsSlotId; - pOperator->name = "StateWindowOperator"; + pInfo->tsSlotId = tsSlotId; + pOperator->name = "StateWindowOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExpr; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExpr; pOperator->numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL, destroyStateWindowOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); @@ -1805,18 +1899,17 @@ _error: return NULL; } -static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, - int32_t tableGroupId) { +static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; - SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - int32_t numOfOutput = pOperatorInfo->numOfExprs; - SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); - int32_t step = 1; - bool ascScan = true; - TSKEY* tsCols = NULL; - SResultRow* pResult = NULL; - int32_t forwardStep = 0; + SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + int32_t numOfOutput = pOperatorInfo->numOfExprs; + SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); + int32_t step = 1; + bool ascScan = true; + TSKEY* tsCols = NULL; + SResultRow* pResult = NULL; + int32_t forwardRows = 0; if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); @@ -1825,14 +1918,13 @@ static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataB return pUpdated; } - int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan); - STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, - &pInfo->interval, pInfo->interval.precision, NULL); + int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); + STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, + pInfo->interval.precision, NULL); while (1) { - int32_t code = - setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pInfo->binfo.pCtx, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1841,15 +1933,16 @@ static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataB pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; *(int64_t*)pos->key = pResult->win.skey; taosArrayPush(pUpdated, &pos); - forwardStep = - getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, + TSDB_ORDER_ASC); // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, - pInfo->order, false); + // disable it temporarily + // doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, + // forwardRows); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, - pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - int32_t prevEndPos = (forwardStep - 1) * step + startPos; + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, + tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + int32_t prevEndPos = (forwardRows - 1) * step + startPos; startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order); if (startPos < 0) { break; @@ -1858,12 +1951,10 @@ static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataB return pUpdated; } -bool isFinalInterval(SStreamFinalIntervalOperatorInfo* pInfo) { - return pInfo->pChildren != NULL; -} +bool isFinalInterval(SStreamFinalIntervalOperatorInfo* pInfo) { return pInfo->pChildren != NULL; } -void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, - int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { +void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput, + SExecTaskInfo* pTaskInfo) { for (int32_t k = 0; k < numOfOutput; ++k) { if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) { continue; @@ -1880,24 +1971,22 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, } } -static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SArray *pWinArray, - int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { +static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId, + int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { int32_t size = taosArrayGetSize(pWinArray); ASSERT(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { STimeWindow* pParentWin = taosArrayGet(pWinArray, i); - SResultRow* pCurResult = NULL; - setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, pParentWin, true, &pCurResult, 0, - pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, - pTaskInfo); + SResultRow* pCurResult = NULL; + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, pParentWin, true, &pCurResult, 0, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); for (int32_t j = 0; j < numOfChildren; j++) { - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, j); + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, j); SIntervalAggOperatorInfo* pChInfo = pChildOp->info; - SResultRow* pChResult = NULL; - setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, pParentWin, true, &pChResult, - 0, pChInfo->binfo.pCtx, pChildOp->numOfExprs, pChInfo->binfo.rowCellInfoOffset, - &pChInfo->aggSup, pTaskInfo); + SResultRow* pChResult = NULL; + setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, pParentWin, true, &pChResult, 0, pChInfo->binfo.pCtx, + pChildOp->numOfExprs, pChInfo->binfo.rowCellInfoOffset, &pChInfo->aggSup, pTaskInfo); compactFunctions(pInfo->binfo.pCtx, pChInfo->binfo.pCtx, numOfOutput, pTaskInfo); } } @@ -1905,8 +1994,8 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SArra static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = NULL; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + SArray* pUpdated = NULL; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -1926,23 +2015,23 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true); if (pBlock->info.type == STREAM_REPROCESS) { - SArray *pUpWins = taosArrayInit(8, sizeof(STimeWindow)); - doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, - pInfo->primaryTsIndex, pOperator->numOfExprs, pBlock, pUpWins); + SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); + doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, pInfo->primaryTsIndex, pOperator->numOfExprs, + pBlock, pUpWins); if (isFinalInterval(pInfo)) { - int32_t childIndex = 0; //Todo(liuyao) get child id from SSDataBlock - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); + int32_t childIndex = 0; // Todo(liuyao) get child id from SSDataBlock + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SIntervalAggOperatorInfo* pChildInfo = pChildOp->info; - doClearWindows(&pChildInfo->aggSup, &pChildInfo->binfo, &pChildInfo->interval, - pChildInfo->primaryTsIndex, pChildOp->numOfExprs, pBlock, NULL); - rebuildIntervalWindow(pInfo, pUpWins, pInfo->binfo.pRes->info.groupId, - pOperator->numOfExprs, pOperator->pTaskInfo); + doClearWindows(&pChildInfo->aggSup, &pChildInfo->binfo, &pChildInfo->interval, pChildInfo->primaryTsIndex, + pChildOp->numOfExprs, pBlock, NULL); + rebuildIntervalWindow(pInfo, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->numOfExprs, + pOperator->pTaskInfo); } taosArrayDestroy(pUpWins); continue; } if (isFinalInterval(pInfo)) { - int32_t chIndex = 1; //Todo(liuyao) get it from SSDataBlock + int32_t chIndex = 1; // Todo(liuyao) get it from SSDataBlock SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); doStreamIntervalAgg(pChildOp); } @@ -1971,7 +2060,7 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) { if (pInfo->pChildren != NULL) { int32_t size = taosArrayGetSize(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { - SOperatorInfo *pChild = taosArrayGetP(pInfo->pChildren, i); + SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i); SStreamSessionAggOperatorInfo* pChInfo = pChild->info; destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput); taosMemoryFreeClear(pChild); @@ -1980,8 +2069,8 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) { } } -int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, - int32_t numOfCols, SSDataBlock* pResultBlock, SDiskbasedBuf* pResultBuf) { +int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, + SDiskbasedBuf* pResultBuf) { pBasicInfo->pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pBasicInfo->rowCellInfoOffset); pBasicInfo->pRes = pResultBlock; for (int32_t i = 0; i < numOfCols; ++i) { @@ -1995,39 +2084,37 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num pDummy[i].functionId = pCtx[i].functionId; } } -void initDownStream(SOperatorInfo* downstream, SStreamSessionAggOperatorInfo* pInfo) { +void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark, + uint8_t type) { ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); SStreamBlockScanInfo* pScanInfo = downstream->info; - pScanInfo->sessionSup = - (SessionWindowSupporter){.pStreamAggSup = &pInfo->streamAggSup, .gap = pInfo->gap}; - pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 60000 * 60 * 6); -} - -SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, - SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, - int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { - int32_t code = TSDB_CODE_OUT_OF_MEMORY; - SStreamSessionAggOperatorInfo* pInfo = - taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type}; + pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark); +} + +SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, + STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { + int32_t code = TSDB_CODE_OUT_OF_MEMORY; + SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } initResultSizeInfo(pOperator, 4096); - code = initStreamAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo"); + code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo"); if (code != TSDB_CODE_SUCCESS) { goto _error; } - code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, - pInfo->streamAggSup.pResultBuf); + code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, pInfo->streamAggSup.pResultBuf); if (code != TSDB_CODE_SUCCESS) { goto _error; } pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols); - + pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx)); if (pInfo->pDummyCtx == NULL) { goto _error; @@ -2056,11 +2143,11 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, pOperator->pExpr = pExprInfo; pOperator->numOfExprs = numOfCols; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionWindowAgg, - NULL, NULL, destroyStreamSessionAggOperatorInfo, aggEncodeResultRow, - aggDecodeResultRow, NULL); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); pOperator->pTaskInfo = pTaskInfo; - initDownStream(downstream, pInfo); + initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -2076,41 +2163,37 @@ _error: } int64_t getSessionWindowEndkey(void* data, int32_t index) { - SArray* pWinInfos = (SArray*) data; + SArray* pWinInfos = (SArray*)data; SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index); return pWin->win.ekey; } static bool isInWindow(SResultWindowInfo* pWin, TSKEY ts, int64_t gap) { int64_t sGap = ts - pWin->win.skey; int64_t eGap = pWin->win.ekey - ts; - if ( (sGap < 0 && sGap >= -gap) || (eGap < 0 && eGap >= -gap) || (sGap >= 0 && eGap >= 0) ) { + if ((sGap < 0 && sGap >= -gap) || (eGap < 0 && eGap >= -gap) || (sGap >= 0 && eGap >= 0)) { return true; } return false; } -static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, - int32_t index) { - SResultWindowInfo win = - {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; +static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) { + SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; return taosArrayInsert(pWinInfos, index, &win); } static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY ts) { - SResultWindowInfo win = - {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; + SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; return taosArrayPush(pWinInfos, &win); } -SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap, - int32_t* pIndex) { +SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap, int32_t* pIndex) { int32_t size = taosArrayGetSize(pWinInfos); if (size == 0) { + *pIndex = 0; return addNewSessionWindow(pWinInfos, ts); } // find the first position which is smaller than the key - int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, - getSessionWindowEndkey); + int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getSessionWindowEndkey); SResultWindowInfo* pWin = NULL; if (index >= 0) { pWin = taosArrayGet(pWinInfos, index); @@ -2132,12 +2215,12 @@ SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap *pIndex = taosArrayGetSize(pWinInfos); return addNewSessionWindow(pWinInfos, ts); } - *pIndex = index; - return insertNewSessionWindow(pWinInfos, ts, index); + *pIndex = index + 1; + return insertNewSessionWindow(pWinInfos, ts, index + 1); } -int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, - int32_t start, int64_t gap, SHashObj* pStDeleted) { +int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, int32_t start, int64_t gap, + SHashObj* pStDeleted) { for (int32_t i = start; i < rows; ++i) { if (!isInWindow(pWinInfo, pTs[i], gap)) { return i - start; @@ -2154,18 +2237,18 @@ int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t return rows - start; } -static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, - SqlFunctionCtx* pCtx, int32_t groupId, int32_t numOfOutput, - int32_t* rowCellInfoOffset, SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) { +static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx, + int32_t groupId, int32_t numOfOutput, int32_t* rowCellInfoOffset, + SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) { assert(pWinInfo->win.skey <= pWinInfo->win.ekey); // too many time window in query int32_t size = taosArrayGetSize(pAggSup->pResultRows); if (size > MAX_INTERVAL_TIME_WINDOW) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } - + if (pWinInfo->pos.pageId == -1) { - *pResult = getNewResultRow_rv(pAggSup->pResultBuf, groupId, pAggSup->resultRowSize); + *pResult = getNewResultRow(pAggSup->pResultBuf, groupId, pAggSup->resultRowSize); if (*pResult == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -2188,34 +2271,47 @@ static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pRes return TSDB_CODE_SUCCESS; } -static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, - SSDataBlock* pSDataBlock, SResultWindowInfo* pCurWin, SResultRow** pResult, - int32_t startIndex, int32_t winRows, int32_t numOutput, SExecTaskInfo* pTaskInfo ) { - SColumnInfoData* pColDataInfo = - taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - TSKEY* tsCols = (int64_t*)pColDataInfo->pData; - int32_t code = setWindowOutputBuf(pCurWin, pResult, pInfo->binfo.pCtx, pSDataBlock->info.groupId, - numOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->streamAggSup, pTaskInfo); +static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStreamAggSupporter* pAggSup, + SColumnInfoData* pTimeWindowData, SSDataBlock* pSDataBlock, + SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows, + int32_t numOutput, SExecTaskInfo* pTaskInfo) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, tsColId); + TSKEY* tsCols = (int64_t*)pColDataInfo->pData; + int32_t code = setWindowOutputBuf(pCurWin, pResult, pBinfo->pCtx, pSDataBlock->info.groupId, numOutput, + pBinfo->rowCellInfoOffset, pAggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->win, true); - doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &pCurWin->win, - &pInfo->twAggSup.timeWindowData, startIndex, winRows, tsCols, pSDataBlock->info.rows, - numOutput, TSDB_ORDER_ASC); + updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, true); + doApplyFunctions(pTaskInfo, pBinfo->pCtx, &pCurWin->win, pTimeWindowData, startIndex, winRows, tsCols, + pSDataBlock->info.rows, numOutput, TSDB_ORDER_ASC); return TSDB_CODE_SUCCESS; } -int32_t copyWinInfoToDataBlock(SSDataBlock* pBlock, SStreamAggSupporter* pAggSup, - int32_t start, int32_t num, int32_t numOfExprs, SOptrBasicInfo* pBinfo) { +static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock, + SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows, + int32_t numOutput, SExecTaskInfo* pTaskInfo) { + return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData, + pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pTaskInfo); +} + +static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock, + SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, + int32_t winRows, int32_t numOutput, SExecTaskInfo* pTaskInfo) { + return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData, + pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pTaskInfo); +} + +int32_t copyWinInfoToDataBlock(SSDataBlock* pBlock, SStreamAggSupporter* pAggSup, int32_t start, int32_t num, + int32_t numOfExprs, SOptrBasicInfo* pBinfo) { for (int32_t i = start; i < num; i += 1) { SResultWindowInfo* pWinInfo = taosArrayGet(pAggSup->pResultRows, start); - SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pWinInfo->pos.pageId); - SResultRow* pRow = (SResultRow*)((char*)bufPage + pWinInfo->pos.offset); + SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pWinInfo->pos.pageId); + SResultRow* pRow = (SResultRow*)((char*)bufPage + pWinInfo->pos.offset); for (int32_t j = 0; j < numOfExprs; ++j) { SResultRowEntryInfo* pResultInfo = getResultCell(pRow, j, pBinfo->rowCellInfoOffset); - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j); - char* in = GET_ROWCELL_INTERBUF(pBinfo->pCtx[j].resultInfo); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j); + char* in = GET_ROWCELL_INTERBUF(pBinfo->pCtx[j].resultInfo); colDataAppend(pColInfoData, pBlock->info.rows, in, pResultInfo->isNullRes); } pBlock->info.rows += pRow->numOfRows; @@ -2227,7 +2323,7 @@ int32_t copyWinInfoToDataBlock(SSDataBlock* pBlock, SStreamAggSupporter* pAggSup int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) { SResultWindowInfo* pCurWin = taosArrayGet(pWinInfos, startIndex); - int32_t size = taosArrayGetSize(pWinInfos); + int32_t size = taosArrayGetSize(pWinInfos); // Just look for the window behind StartIndex for (int32_t i = startIndex + 1; i < size; i++) { SResultWindowInfo* pWinInfo = taosArrayGet(pWinInfos, i); @@ -2239,20 +2335,20 @@ int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) return size - startIndex - 1; } -void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, - int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SHashObj* pStUpdated, SHashObj* pStDeleted) { +void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, int32_t groupId, + int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SHashObj* pStUpdated, SHashObj* pStDeleted) { SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pResultRows, startIndex); - SResultRow* pCurResult = NULL; - setWindowOutputBuf(pCurWin, &pCurResult, pInfo->binfo.pCtx, groupId, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->streamAggSup, pTaskInfo); + SResultRow* pCurResult = NULL; + setWindowOutputBuf(pCurWin, &pCurResult, pInfo->binfo.pCtx, groupId, numOfOutput, pInfo->binfo.rowCellInfoOffset, + &pInfo->streamAggSup, pTaskInfo); num += startIndex + 1; ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pResultRows)); // Just look for the window behind StartIndex for (int32_t i = startIndex + 1; i < num; i++) { SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pResultRows, i); - SResultRow* pWinResult = NULL; - setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->streamAggSup, pTaskInfo); + SResultRow* pWinResult = NULL; + setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pInfo->binfo.rowCellInfoOffset, + &pInfo->streamAggSup, pTaskInfo); pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey); compactFunctions(pInfo->binfo.pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo); taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition)); @@ -2264,43 +2360,41 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, } } -static void doStreamSessionWindowAggImpl(SOperatorInfo* pOperator, - SSDataBlock* pSDataBlock, SHashObj* pStUpdated, SHashObj* pStDeleted) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; +static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated, + SHashObj* pStDeleted) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - bool masterScan = true; - int32_t numOfOutput = pOperator->numOfExprs; - int64_t groupId = pSDataBlock->info.groupId; - int64_t gap = pInfo->gap; - int64_t code = TSDB_CODE_SUCCESS; - - int32_t step = 1; - bool ascScan = true; - TSKEY* tsCols = NULL; + bool masterScan = true; + int32_t numOfOutput = pOperator->numOfExprs; + int64_t groupId = pSDataBlock->info.groupId; + int64_t gap = pInfo->gap; + int64_t code = TSDB_CODE_SUCCESS; + + int32_t step = 1; + bool ascScan = true; + TSKEY* tsCols = NULL; SResultRow* pResult = NULL; - int32_t winRows = 0; + int32_t winRows = 0; if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = - taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; } else { - return ; + return; } - + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - for(int32_t i = 0; i < pSDataBlock->info.rows; ) { - int32_t winIndex = 0; - SResultWindowInfo* pCurWin = - getSessionTimeWindow(pAggSup->pResultRows, tsCols[i], gap, &winIndex); - winRows = - updateSessionWindowInfo(pCurWin, tsCols, pSDataBlock->info.rows, i, pInfo->gap, pStDeleted); + for (int32_t i = 0; i < pSDataBlock->info.rows;) { + int32_t winIndex = 0; + SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup->pResultRows, tsCols[i], gap, &winIndex); + winRows = updateSessionWindowInfo(pCurWin, tsCols, pSDataBlock->info.rows, i, pInfo->gap, pStDeleted); code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } // window start(end) key interpolation - // doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, + // doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, + // forwardRows, // pInfo->order, false); int32_t winNum = getNumCompactWindow(pAggSup->pResultRows, winIndex, gap); if (winNum > 0) { @@ -2318,15 +2412,14 @@ static void doStreamSessionWindowAggImpl(SOperatorInfo* pOperator, } } -static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SOptrBasicInfo* pBinfo, - SSDataBlock* pBlock, int32_t tsIndex, int32_t numOfOutput, int64_t gap, SArray* result) { +static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SOptrBasicInfo* pBinfo, SSDataBlock* pBlock, + int32_t tsIndex, int32_t numOfOutput, int64_t gap, SArray* result) { SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); - TSKEY *tsCols = (TSKEY*)pColDataInfo->pData; - int32_t step = 0; + TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; + int32_t step = 0; for (int32_t i = 0; i < pBlock->info.rows; i += step) { - int32_t winIndex = 0; - SResultWindowInfo* pCurWin = - getSessionTimeWindow(pAggSup->pResultRows, tsCols[i], gap, &winIndex); + int32_t winIndex = 0; + SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup->pResultRows, tsCols[i], gap, &winIndex); step = updateSessionWindowInfo(pCurWin, tsCols, pBlock->info.rows, i, gap, NULL); ASSERT(isInWindow(pCurWin, tsCols[i], gap)); doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pBinfo, numOfOutput); @@ -2337,9 +2430,9 @@ static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SOptrBasicInfo* } static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated, int32_t groupId) { - void* pData = NULL; + void* pData = NULL; size_t keyLen = 0; - while((pData = taosHashIterate(pStUpdated, pData)) != NULL) { + while ((pData = taosHashIterate(pStUpdated, pData)) != NULL) { void* key = taosHashGetKey(pData, &keyLen); ASSERT(keyLen == sizeof(SResultRowPosition)); SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); @@ -2357,12 +2450,12 @@ static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated, int32_t void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) { blockDataCleanup(pBlock); size_t keyLen = 0; - while(( (*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) { + while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); - colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false); + colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false); for (int32_t i = 1; i < pBlock->info.numOfCols; i++) { pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - colDataAppendNULL(pColInfoData, pBlock->info.rows); + colDataAppendNULL(pColInfoData, pBlock->info.rows); } pBlock->info.rows += 1; if (pBlock->info.rows + 1 >= pBlock->info.capacity) { @@ -2374,29 +2467,28 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It } } -static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray *pWinArray, - int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { +static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId, + int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { int32_t size = taosArrayGetSize(pWinArray); ASSERT(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i); - SResultRow* pCurResult = NULL; - setWindowOutputBuf(pParentWin, &pCurResult, pInfo->binfo.pCtx, groupId, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->streamAggSup, pTaskInfo); + SResultRow* pCurResult = NULL; + setWindowOutputBuf(pParentWin, &pCurResult, pInfo->binfo.pCtx, groupId, numOfOutput, pInfo->binfo.rowCellInfoOffset, + &pInfo->streamAggSup, pTaskInfo); int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); for (int32_t j = 0; j < numOfChildren; j++) { - SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, j); + SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, j); SStreamSessionAggOperatorInfo* pChInfo = pChild->info; - SArray* pChWins = pChInfo->streamAggSup.pResultRows; - int32_t chWinSize = taosArrayGetSize(pChWins); - int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, - TSDB_ORDER_DESC, getSessionWindowEndkey); + SArray* pChWins = pChInfo->streamAggSup.pResultRows; + int32_t chWinSize = taosArrayGetSize(pChWins); + int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey); for (int32_t k = index; k > 0 && k < chWinSize; k++) { SResultWindowInfo* pcw = taosArrayGet(pChWins, k); if (pParentWin->win.skey <= pcw->win.skey && pcw->win.ekey <= pParentWin->win.ekey) { SResultRow* pChResult = NULL; - setWindowOutputBuf(pcw, &pChResult, pChInfo->binfo.pCtx, groupId, - numOfOutput, pChInfo->binfo.rowCellInfoOffset, &pChInfo->streamAggSup, pTaskInfo); + setWindowOutputBuf(pcw, &pChResult, pChInfo->binfo.pCtx, groupId, numOfOutput, + pChInfo->binfo.rowCellInfoOffset, &pChInfo->streamAggSup, pTaskInfo); compactFunctions(pInfo->binfo.pCtx, pChInfo->binfo.pCtx, numOfOutput, pTaskInfo); continue; } @@ -2406,16 +2498,19 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray *pWin } } -bool isFinalSession(SStreamSessionAggOperatorInfo* pInfo) { - return pInfo->pChildren != NULL; -} +bool isFinalSession(SStreamSessionAggOperatorInfo* pInfo) { return pInfo->pChildren != NULL; } -int32_t closeSessionWindow(SArray *pWins, STimeWindowAggSupp *pTwSup, SArray *pClosed, - int8_t calTrigger) { +typedef SResultWindowInfo* (*__get_win_info_)(void*); +SResultWindowInfo* getSessionWinInfo(void* pData) { return (SResultWindowInfo*)pData; } +SResultWindowInfo* getStateWinInfo(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; } + +int32_t closeSessionWindow(SArray* pWins, STimeWindowAggSupp* pTwSup, SArray* pClosed, int8_t calTrigger, + __get_win_info_ fn) { // Todo(liuyao) save window to tdb - int32_t size = taosArrayGetSize(pWins); + int32_t size = taosArrayGetSize(pWins); for (int32_t i = 0; i < size; i++) { - SResultWindowInfo *pSeWin = taosArrayGet(pWins, i); + void* pWin = taosArrayGet(pWins, i); + SResultWindowInfo* pSeWin = fn(pWin); if (pSeWin->win.ekey < pTwSup->maxTs - pTwSup->waterMark) { if (!pSeWin->isClosed) { SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); @@ -2441,29 +2536,27 @@ int32_t closeSessionWindow(SArray *pWins, STimeWindowAggSupp *pTwSup, SArray *pC return TSDB_CODE_SUCCESS; } -static SSDataBlock* doStreamSessionWindowAgg(SOperatorInfo* pOperator) { +static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pInfo->binfo; + SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, - pInfo->streamAggSup.pResultBuf); - if (pBInfo->pRes->info.rows == 0 || - !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { + doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + if (pBInfo->pRes->info.rows == 0 || !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + SHashObj* pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); @@ -2472,35 +2565,34 @@ static SSDataBlock* doStreamSessionWindowAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - if (pBlock->info.type == STREAM_REPROCESS) { - SArray *pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); - doClearSessionWindows(&pInfo->streamAggSup, &pInfo->binfo, pBlock, 0, - pOperator->numOfExprs, pInfo->gap, pWins); + if (pBlock->info.type == STREAM_REPROCESS) { + SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); + doClearSessionWindows(&pInfo->streamAggSup, &pInfo->binfo, pBlock, 0, pOperator->numOfExprs, pInfo->gap, pWins); if (isFinalSession(pInfo)) { - int32_t childIndex = 0; //Todo(liuyao) get child id from SSDataBlock - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); + int32_t childIndex = 0; // Todo(liuyao) get child id from SSDataBlock + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; - doClearSessionWindows(&pChildInfo->streamAggSup, &pChildInfo->binfo, pBlock, 0, - pChildOp->numOfExprs, pChildInfo->gap, NULL); + doClearSessionWindows(&pChildInfo->streamAggSup, &pChildInfo->binfo, pBlock, 0, pChildOp->numOfExprs, + pChildInfo->gap, NULL); rebuildTimeWindow(pInfo, pWins, pInfo->binfo.pRes->info.groupId, pOperator->numOfExprs, pOperator->pTaskInfo); } taosArrayDestroy(pWins); continue; } if (isFinalSession(pInfo)) { - int32_t childIndex = 0; //Todo(liuyao) get child id from SSDataBlock + int32_t childIndex = 0; // Todo(liuyao) get child id from SSDataBlock SOptrBasicInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); - doStreamSessionWindowAggImpl(pOperator, pBlock, NULL, NULL); + doStreamSessionAggImpl(pOperator, pBlock, NULL, NULL); } - doStreamSessionWindowAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted); + doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); } // restore the value pOperator->status = OP_RES_TO_RETURN; - + SArray* pClosed = taosArrayInit(16, POINTER_BYTES); - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pClosed, - pInfo->twAggSup.calTrigger); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pClosed, pInfo->twAggSup.calTrigger, + getSessionWinInfo); SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); copyUpdateResult(pStUpdated, pUpdated, pBInfo->pRes->info.groupId); taosHashCleanup(pStUpdated); @@ -2509,36 +2601,36 @@ static SSDataBlock* doStreamSessionWindowAgg(SOperatorInfo* pOperator) { } finalizeUpdatedResult(pOperator->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, - pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.rowCellInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, - pInfo->streamAggSup.pResultBuf); + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } -SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, - SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, - int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { - int32_t code = TSDB_CODE_OUT_OF_MEMORY; +SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, + int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, + int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, + SExecTaskInfo* pTaskInfo) { + int32_t code = TSDB_CODE_OUT_OF_MEMORY; SStreamSessionAggOperatorInfo* pInfo = NULL; - SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pExprInfo, - numOfCols, pResBlock, gap, tsSlotId, pTwAggSupp, pTaskInfo); + SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pExprInfo, numOfCols, pResBlock, gap, + tsSlotId, pTwAggSupp, pTaskInfo); if (pOperator == NULL) { goto _error; } pOperator->name = "StreamFinalSessionWindowAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION_WINDOW; - int32_t numOfChild = 1; //Todo(liuyao) get it from phy plan + int32_t numOfChild = 1; // Todo(liuyao) get it from phy plan pInfo = pOperator->info; - pInfo->pChildren = taosArrayInit(8, sizeof(void *)); + pInfo->pChildren = taosArrayInit(8, sizeof(void*)); for (int32_t i = 0; i < numOfChild; i++) { - SOperatorInfo* pChild = createStreamSessionAggOperatorInfo(NULL, pExprInfo, - numOfCols, NULL, gap, tsSlotId, pTwAggSupp, pTaskInfo); + SOperatorInfo* pChild = + createStreamSessionAggOperatorInfo(NULL, pExprInfo, numOfCols, NULL, gap, tsSlotId, pTwAggSupp, pTaskInfo); if (pChild == NULL) { goto _error; } @@ -2556,3 +2648,393 @@ _error: pTaskInfo->code = code; return NULL; } + +void destroyStreamStateOperatorInfo(void* param, int32_t numOfOutput) { + SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + destroyStreamAggSupporter(&pInfo->streamAggSup); + cleanupGroupResInfo(&pInfo->groupResInfo); + if (pInfo->pChildren != NULL) { + int32_t size = taosArrayGetSize(pInfo->pChildren); + for (int32_t i = 0; i < size; i++) { + SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i); + SStreamSessionAggOperatorInfo* pChInfo = pChild->info; + destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput); + taosMemoryFreeClear(pChild); + taosMemoryFreeClear(pChInfo); + } + } +} + +int64_t getStateWinTsKey(void* data, int32_t index) { + SStateWindowInfo* pStateWin = taosArrayGet(data, index); + return pStateWin->winInfo.win.ekey; +} + +SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol) { + SStateWindowInfo win = { + .stateKey.bytes = pCol->bytes, + .stateKey.type = pCol->type, + .stateKey.pData = taosMemoryCalloc(1, pCol->bytes), + .winInfo.pos.offset = -1, + .winInfo.pos.pageId = -1, + .winInfo.win.skey = ts, + .winInfo.win.ekey = ts, + .winInfo.isOutput = false, + .winInfo.isClosed = false, + }; + if (IS_VAR_DATA_TYPE(win.stateKey.type)) { + varDataCopy(win.stateKey.pData, pKeyData); + } else { + memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes); + } + return taosArrayPush(pWinInfos, &win); +} + +SStateWindowInfo* insertNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, int32_t index, SColumn* pCol) { + SStateWindowInfo win = { + .stateKey.bytes = pCol->bytes, + .stateKey.type = pCol->type, + .stateKey.pData = taosMemoryCalloc(1, pCol->bytes), + .winInfo.pos.offset = -1, + .winInfo.pos.pageId = -1, + .winInfo.win.skey = ts, + .winInfo.win.ekey = ts, + .winInfo.isOutput = false, + .winInfo.isClosed = false, + }; + if (IS_VAR_DATA_TYPE(win.stateKey.type)) { + varDataCopy(win.stateKey.pData, pKeyData); + } else { + memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes); + } + return taosArrayInsert(pWinInfos, index, &win); +} + +bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) { + if (pWin->winInfo.win.skey <= ts && ts <= pWin->winInfo.win.ekey) { + return true; + } + return false; +} + +bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) { + return pKeyData && compareVal(pKeyData, &pWin->stateKey); +} + +SStateWindowInfo* getStateWindowByTs(SArray* pWinInfos, TSKEY ts, int32_t* pIndex) { + int32_t size = taosArrayGetSize(pWinInfos); + int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey); + SStateWindowInfo* pWin = NULL; + if (index >= 0) { + pWin = taosArrayGet(pWinInfos, index); + if (isTsInWindow(pWin, ts)) { + *pIndex = index; + return pWin; + } + } + + if (index + 1 < size) { + pWin = taosArrayGet(pWinInfos, index + 1); + if (isTsInWindow(pWin, ts)) { + *pIndex = index + 1; + return pWin; + } + } + *pIndex = 0; + return NULL; +} + +SStateWindowInfo* getStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol, int32_t* pIndex) { + int32_t size = taosArrayGetSize(pWinInfos); + if (size == 0) { + *pIndex = 0; + return addNewStateWindow(pWinInfos, ts, pKeyData, pCol); + } + int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey); + SStateWindowInfo* pWin = NULL; + if (index >= 0) { + pWin = taosArrayGet(pWinInfos, index); + if (isTsInWindow(pWin, ts)) { + *pIndex = index; + return pWin; + } + } + + if (index + 1 < size) { + pWin = taosArrayGet(pWinInfos, index + 1); + if (isTsInWindow(pWin, ts) || isEqualStateKey(pWin, pKeyData)) { + *pIndex = index + 1; + return pWin; + } + } + + if (index >= 0) { + pWin = taosArrayGet(pWinInfos, index); + if (isEqualStateKey(pWin, pKeyData)) { + *pIndex = index; + return pWin; + } + } + + if (index == size - 1) { + *pIndex = taosArrayGetSize(pWinInfos); + return addNewStateWindow(pWinInfos, ts, pKeyData, pCol); + } + *pIndex = index + 1; + return insertNewStateWindow(pWinInfos, ts, pKeyData, index + 1, pCol); +} + +int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, SColumnInfoData* pKeyCol, int32_t rows, + int32_t start, bool* allEqual, SHashObj* pSeDelete) { + *allEqual = true; + SStateWindowInfo* pWinInfo = taosArrayGet(pWinInfos, winIndex); + for (int32_t i = start; i < rows; ++i) { + char* pKeyData = colDataGetData(pKeyCol, i); + if (!isTsInWindow(pWinInfo, pTs[i])) { + if (isEqualStateKey(pWinInfo, pKeyData)) { + int32_t size = taosArrayGetSize(pWinInfos); + if (winIndex + 1 < size) { + SStateWindowInfo* pNextWin = taosArrayGet(pWinInfos, winIndex + 1); + // ts belongs to the next window + if (pTs[i] >= pNextWin->winInfo.win.skey) { + return i - start; + } + } + } else { + return i - start; + } + } + if (pWinInfo->winInfo.win.skey > pTs[i]) { + if (pSeDelete && pWinInfo->winInfo.isOutput) { + taosHashPut(pSeDelete, &pWinInfo->winInfo.pos, sizeof(SResultRowPosition), &pWinInfo->winInfo.win.skey, + sizeof(TSKEY)); + pWinInfo->winInfo.isOutput = false; + } + pWinInfo->winInfo.win.skey = pTs[i]; + } + pWinInfo->winInfo.win.ekey = TMAX(pWinInfo->winInfo.win.ekey, pTs[i]); + if (!isEqualStateKey(pWinInfo, pKeyData)) { + *allEqual = false; + } + } + return rows - start; +} + +void deleteWindow(SArray* pWinInfos, int32_t index) { + ASSERT(index >= 0 && index < taosArrayGetSize(pWinInfos)); + taosArrayRemove(pWinInfos, index); +} + +static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int32_t tsIndex, SColumn* pCol, + int32_t keyIndex, SHashObj* pSeUpdated, SHashObj* pSeDeleted) { + SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); + SColumnInfoData* pKeyColInfo = taosArrayGet(pBlock->pDataBlock, keyIndex); + TSKEY* tsCol = (TSKEY*)pTsColInfo->pData; + bool allEqual = false; + int32_t step = 1; + for (int32_t i = 0; i < pBlock->info.rows; i += step) { + char* pKeyData = colDataGetData(pKeyColInfo, i); + int32_t winIndex = 0; + SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup->pResultRows, tsCol[i], &winIndex); + if (!pCurWin) { + continue; + } + step = updateStateWindowInfo(pAggSup->pResultRows, winIndex, tsCol, pKeyColInfo, pBlock->info.rows, i, &allEqual, + pSeDeleted); + ASSERT(isTsInWindow(pCurWin, tsCol[i]) || isEqualStateKey(pCurWin, pKeyData)); + taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win); + taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); + deleteWindow(pAggSup->pResultRows, winIndex); + } +} + +static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pSeUpdated, + SHashObj* pStDeleted) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStreamStateAggOperatorInfo* pInfo = pOperator->info; + bool masterScan = true; + int32_t numOfOutput = pOperator->numOfExprs; + int64_t groupId = pSDataBlock->info.groupId; + int64_t code = TSDB_CODE_SUCCESS; + int32_t step = 1; + bool ascScan = true; + TSKEY* tsCols = NULL; + SResultRow* pResult = NULL; + int32_t winRows = 0; + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + } else { + return; + } + + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); + for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) { + char* pKeyData = colDataGetData(pKeyColInfo, i); + int32_t winIndex = 0; + bool allEqual = true; + SStateWindowInfo* pCurWin = getStateWindow(pAggSup->pResultRows, tsCols[i], pKeyData, &pInfo->stateCol, &winIndex); + winRows = updateStateWindowInfo(pAggSup->pResultRows, winIndex, tsCols, pKeyColInfo, pSDataBlock->info.rows, i, + &allEqual, pInfo->pSeDeleted); + if (!allEqual) { + taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win); + taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); + deleteWindow(pAggSup->pResultRows, winIndex); + continue; + } + code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, numOfOutput, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + pCurWin->winInfo.isClosed = false; + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { + code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), &(pCurWin->winInfo.win.skey), + sizeof(TSKEY)); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + pCurWin->winInfo.isOutput = true; + } + } +} + +static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SStreamStateAggOperatorInfo* pInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + if (pOperator->status == OP_RES_TO_RETURN) { + doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); + if (pInfo->pDelRes->info.rows > 0) { + return pInfo->pDelRes; + } + doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + if (pBInfo->pRes->info.rows == 0 || !hashRemainDataInGroupInfo(&pInfo->groupResInfo)) { + doSetOperatorCompleted(pOperator); + } + return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; + } + + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + SHashObj* pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + SOperatorInfo* downstream = pOperator->pDownstream[0]; + while (1) { + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + break; + } + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); + if (pBlock->info.type == STREAM_REPROCESS) { + doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId, + pSeUpdated, pInfo->pSeDeleted); + continue; + } + doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); + } + // restore the value + pOperator->status = OP_RES_TO_RETURN; + + SArray* pClosed = taosArrayInit(16, POINTER_BYTES); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pClosed, pInfo->twAggSup.calTrigger, + getStateWinInfo); + SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); + copyUpdateResult(pSeUpdated, pUpdated, pBInfo->pRes->info.groupId); + taosHashCleanup(pSeUpdated); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + taosArrayAddAll(pUpdated, pClosed); + } + + finalizeUpdatedResult(pOperator->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, + pInfo->binfo.rowCellInfoOffset); + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); + if (pInfo->pDelRes->info.rows > 0) { + return pInfo->pDelRes; + } + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; +} + +SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo) { + SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode; + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; + SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; + + SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols); + + pInfo->stateCol = extractColumnFromColumnNode(pColNode); + initResultSizeInfo(pOperator, 4096); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); + pInfo->twAggSup = (STimeWindowAggSupp){ + .waterMark = pStateNode->window.watermark, + .calTrigger = pStateNode->window.triggerType, + .maxTs = INT64_MIN, + .winMap = NULL, + }; + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + int32_t code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo"); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, pInfo->streamAggSup.pResultBuf); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols); + pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx)); + if (pInfo->pDummyCtx == NULL) { + goto _error; + } + + initDummyFunction(pInfo->pDummyCtx, pInfo->binfo.pCtx, numOfCols); + pInfo->primaryTsIndex = tsSlotId; + pInfo->order = TSDB_ORDER_ASC; + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + pInfo->pDelIterator = NULL; + pInfo->pDelRes = createOneDataBlock(pResBlock, false); + blockDataEnsureCapacity(pInfo->pDelRes, 64); + pInfo->pChildren = NULL; + + pOperator->name = "StreamStateAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->numOfExprs = numOfCols; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL, + destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType); + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + return pOperator; + +_error: + destroyStreamStateOperatorInfo(pInfo, numOfCols); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index cac86be91752575feba4dcb016eddd84564389d1..68b83f4a1955c72e119dcadd5d409ce10639e5e1 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -140,6 +140,10 @@ bool uniqueFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) int32_t uniqueFunction(SqlFunctionCtx *pCtx); //int32_t uniqueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool twaFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +int32_t twaFunction(SqlFunctionCtx *pCtx); +int32_t twaFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index d779cf50f4ce019ddcea41b71720347d54a34e96..c3d61d426d889cecda0723b48c6c26eae16316ff 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -52,13 +52,6 @@ typedef struct SInterpInfoDetail { int8_t primaryCol; } SInterpInfoDetail; -typedef struct STwaInfo { - int8_t hasResult; // flag to denote has value - double dOutput; - SPoint1 p; - STimeWindow win; -} STwaInfo; - bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval); /** diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 5560c9c1d5ec8887dacd1ea56eec2dba5673bde5..c9c63169c931cc03fdf58d16f551f6abbdc8ba85 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -18,7 +18,6 @@ #include "querynodes.h" #include "scalar.h" #include "taoserror.h" -#include "tdatablock.h" static int32_t buildFuncErrMsg(char* pErrBuf, int32_t len, int32_t errCode, const char* pFormat, ...) { va_list vArgList; @@ -362,7 +361,7 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len pValue->notReserved = true; - uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; + paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; if (!IS_INTEGER_TYPE(paraType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -456,7 +455,7 @@ static int32_t translateHLL(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { "The input parameter of HYPERLOGLOG function can only be column"); } - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_UBIGINT].bytes, .type = TSDB_DATA_TYPE_UBIGINT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } @@ -464,12 +463,9 @@ static bool validateStateOper(const SValueNode* pVal) { if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) { return false; } - return (0 == strcasecmp(varDataVal(pVal->datum.p), "GT") || - 0 == strcasecmp(varDataVal(pVal->datum.p), "GE") || - 0 == strcasecmp(varDataVal(pVal->datum.p), "LT") || - 0 == strcasecmp(varDataVal(pVal->datum.p), "LE") || - 0 == strcasecmp(varDataVal(pVal->datum.p), "EQ") || - 0 == strcasecmp(varDataVal(pVal->datum.p), "NE")); + return (0 == strcasecmp(varDataVal(pVal->datum.p), "GT") || 0 == strcasecmp(varDataVal(pVal->datum.p), "GE") || + 0 == strcasecmp(varDataVal(pVal->datum.p), "LT") || 0 == strcasecmp(varDataVal(pVal->datum.p), "LE") || + 0 == strcasecmp(varDataVal(pVal->datum.p), "EQ") || 0 == strcasecmp(varDataVal(pVal->datum.p), "NE")); } static int32_t translateStateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { @@ -553,7 +549,6 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32 "STATEDURATION function time unit parameter should be greater than db precision"); } - pValue->notReserved = true; } @@ -838,7 +833,7 @@ static int32_t translateConcatImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t int32_t resultBytes = 0; int32_t sepBytes = 0; - //concat_ws separator should be constant string + // concat_ws separator should be constant string if (hasSep) { SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); if (nodeType(pPara) != QUERY_NODE_VALUE) { @@ -851,7 +846,7 @@ static int32_t translateConcatImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t for (int32_t i = 0; i < numOfParams; ++i) { SNode* pPara = nodesListGetNode(pFunc->pParameterList, i); uint8_t paraType = ((SExprNode*)pPara)->resType.type; - if (!IS_VAR_DATA_TYPE(paraType) && TSDB_DATA_TYPE_NULL != paraType) { + if (!IS_VAR_DATA_TYPE(paraType) && !IS_NULL_TYPE(paraType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } if (TSDB_DATA_TYPE_NCHAR == paraType) { @@ -864,6 +859,12 @@ static int32_t translateConcatImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t uint8_t paraType = ((SExprNode*)pPara)->resType.type; int32_t paraBytes = ((SExprNode*)pPara)->resType.bytes; int32_t factor = 1; + if (IS_NULL_TYPE(paraType)) { + resultType = TSDB_DATA_TYPE_VARCHAR; + resultBytes = 0; + sepBytes = 0; + break; + } if (TSDB_DATA_TYPE_NCHAR == resultType && TSDB_DATA_TYPE_VARCHAR == paraType) { factor *= TSDB_NCHAR_SIZE; } @@ -964,7 +965,7 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { return false; } - char *tz = varDataVal(pVal->datum.p); + char* tz = varDataVal(pVal->datum.p); int32_t len = varDataLen(pVal->datum.p); if (len == 0) { @@ -1008,20 +1009,20 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { } void static addTimezoneParam(SNodeList* pList) { - char buf[6] = {0}; - time_t t = taosTime(NULL); - struct tm *tmInfo = taosLocalTime(&t, NULL); + char buf[6] = {0}; + time_t t = taosTime(NULL); + struct tm* tmInfo = taosLocalTime(&t, NULL); strftime(buf, sizeof(buf), "%z", tmInfo); int32_t len = (int32_t)strlen(buf); SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); pVal->literal = strndup(buf, len); - pVal->isDuration =false; + pVal->isDuration = false; pVal->translate = true; pVal->node.resType.type = TSDB_DATA_TYPE_BINARY; pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE; pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI; - pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE +1); + pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1); varDataSetLen(pVal->datum.p, len); strncpy(varDataVal(pVal->datum.p), pVal->literal, len); @@ -1034,25 +1035,24 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - //param0 + // param0 uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; if (!IS_INTEGER_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - //param1 + // param1 if (numOfParams == 2) { SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); if (!validateTimezoneFormat(pValue)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "Invalid timzone format"); + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "Invalid timzone format"); } - } else { //add default client timezone + } else { // add default client timezone addTimezoneParam(pFunc->pParameterList); } - //set result type + // set result type pFunc->node.resType = (SDataType){.bytes = 64, .type = TSDB_DATA_TYPE_BINARY}; return TSDB_CODE_SUCCESS; } @@ -1312,6 +1312,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = firstLastFinalize, .combineFunc = lastCombine, }, + { + .name = "twa", + .type = FUNCTION_TYPE_TWA, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = getTwaFuncEnv, + .initFunc = twaFunctionSetup, + .processFunc = twaFunction, + .finalizeFunc = twaFinalize + }, { .name = "histogram", .type = FUNCTION_TYPE_HISTOGRAM, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index a7e93246b765199369eda8e7aba0da94a49fce3b..4c7984c602f3c789183e96428a97374f4019829d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -14,6 +14,7 @@ */ #include "builtinsimpl.h" +#include "tglobal.h" #include "cJSON.h" #include "function.h" #include "querynodes.h" @@ -300,7 +301,7 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - //pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; + pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; char* in = GET_ROWCELL_INTERBUF(pResInfo); colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); @@ -357,7 +358,7 @@ bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -static FORCE_INLINE int32_t getNumofElem(SqlFunctionCtx* pCtx) { +static FORCE_INLINE int32_t getNumOfElems(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; /* @@ -392,11 +393,12 @@ static FORCE_INLINE int32_t getNumofElem(SqlFunctionCtx* pCtx) { * count function does not use the pCtx->interResBuf to keep the intermediate buffer */ int32_t countFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElem = getNumofElem(pCtx); - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + int32_t numOfElem = getNumOfElems(pCtx); + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; - int32_t type = pInput->pData[0]->info.type; + + int32_t type = pInput->pData[0]->info.type; char* buf = GET_ROWCELL_INTERBUF(pResInfo); if (IS_NULL_TYPE(type)) { @@ -407,12 +409,17 @@ int32_t countFunction(SqlFunctionCtx* pCtx) { *((int64_t*)buf) += numOfElem; } - SET_VAL(pResInfo, numOfElem, 1); + if (tsCountAlwaysReturnValue) { + pResInfo->numOfRes = 1; + } else { + SET_VAL(pResInfo, 1, 1); + } + return TSDB_CODE_SUCCESS; } int32_t countInvertFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElem = getNumofElem(pCtx); + int32_t numOfElem = getNumOfElems(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); char* buf = GET_ROWCELL_INTERBUF(pResInfo); @@ -603,8 +610,7 @@ int32_t sumCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { } else if (type == TSDB_DATA_TYPE_DOUBLE || type == TSDB_DATA_TYPE_FLOAT) { pDBuf->dsum += pSBuf->dsum; } - - SET_VAL(pDResInfo, *((int64_t*)pDBuf), 1); + pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); return TSDB_CODE_SUCCESS; } @@ -829,8 +835,10 @@ int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SInputColumnInfoData* pInput = &pCtx->input; - int32_t type = pInput->pData[0]->info.type; - SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + int32_t type = pInput->pData[0]->info.type; + SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + if (IS_INTEGER_TYPE(type)) { pAvgRes->result = pAvgRes->sum.isum / ((double)pAvgRes->count); } else { @@ -1315,13 +1323,11 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SMinmaxResInfo* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); - int32_t type = pCtx->input.pData[0]->info.type; int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + int32_t currentRow = pBlock->info.rows; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); - - // todo assign the tag value - int32_t currentRow = pBlock->info.rows; + pEntryInfo->isNullRes = (pEntryInfo->numOfRes == 0); if (pCol->info.type == TSDB_DATA_TYPE_FLOAT) { float v = *(double*) &pRes->v; @@ -1330,7 +1336,10 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { colDataAppend(pCol, currentRow, (const char*)&pRes->v, pEntryInfo->isNullRes); } - setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); + if (pEntryInfo->numOfRes > 0) { + setSelectivityValue(pCtx, pBlock, &pRes->tuplePos, currentRow); + } + return pEntryInfo->numOfRes; } @@ -1384,7 +1393,7 @@ int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 pDBuf->v = pSBuf->v; } } - SET_VAL(pDResInfo, *((int64_t*)pDBuf), 1); + pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); return TSDB_CODE_SUCCESS; } @@ -1629,6 +1638,7 @@ int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { pDBuf->quadraticDSum += pSBuf->quadraticDSum; } pDBuf->count += pSBuf->count; + pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); return TSDB_CODE_SUCCESS; } @@ -1832,7 +1842,7 @@ bool percentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultI } int32_t percentileFunction(SqlFunctionCtx* pCtx) { - int32_t notNullElems = 0; + int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; @@ -1910,11 +1920,11 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { } char* data = colDataGetData(pCol, i); - notNullElems += 1; + numOfElems += 1; tMemBucketPut(pInfo->pMemBucket, data, 1); } - SET_VAL(pResInfo, notNullElems, 1); + SET_VAL(pResInfo, numOfElems, 1); } return TSDB_CODE_SUCCESS; @@ -1988,7 +1998,7 @@ bool apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResult } int32_t apercentileFunction(SqlFunctionCtx* pCtx) { - int32_t notNullElems = 0; + int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; @@ -2005,7 +2015,7 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { if (colDataIsNull_f(pCol->nullbitmap, i)) { continue; } - notNullElems += 1; + numOfElems += 1; char* data = colDataGetData(pCol, i); double v = 0; // value @@ -2018,7 +2028,7 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { if (colDataIsNull_f(pCol->nullbitmap, i)) { continue; } - notNullElems += 1; + numOfElems += 1; char* data = colDataGetData(pCol, i); double v = 0; @@ -2027,7 +2037,7 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { } } - SET_VAL(pResInfo, notNullElems, 1); + SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } @@ -3240,13 +3250,13 @@ static uint64_t hllCountCnt(uint8_t *buckets) { z += buckethisto[j]; z *= 0.5; } + z += m * hllSigma(buckethisto[0]/(double)m); double E = (double)llroundl(HLL_ALPHA_INF*m*m/z); return (uint64_t) E; } - int32_t hllFunction(SqlFunctionCtx *pCtx) { SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -3279,7 +3289,6 @@ int32_t hllFunction(SqlFunctionCtx *pCtx) { if (count > oldcount) { pInfo->buckets[index] = count; } - } SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); @@ -3287,9 +3296,13 @@ int32_t hllFunction(SqlFunctionCtx *pCtx) { } int32_t hllFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { - SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + SResultRowEntryInfo *pInfo = GET_RES_INFO(pCtx); - pInfo->result = hllCountCnt(pInfo->buckets); + SHLLInfo* pHllInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + pHllInfo->result = hllCountCnt(pHllInfo->buckets); + if (tsCountAlwaysReturnValue && pHllInfo->result == 0) { + pInfo->numOfRes = 1; + } return functionFinalize(pCtx, pBlock); } @@ -3695,7 +3708,6 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { TSKEY* tsList = (int64_t*)pInput->pPTS->pData; SColumnInfoData* pInputCol = pInput->pData[0]; - SColumnInfoData* pTsOutput = pCtx->pTsOutput; SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; int32_t startOffset = pCtx->offset; @@ -3718,24 +3730,6 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { return pInfo->numSampled; } -//int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { -// SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); -// SSampleInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); -// int32_t slotId = pCtx->pExpr->base.resSchema.slotId; -// SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); -// -// //int32_t currentRow = pBlock->info.rows; -// pResInfo->numOfRes = pInfo->numSampled; -// -// for (int32_t i = 0; i < pInfo->numSampled; ++i) { -// colDataAppend(pCol, i, pInfo->data + i * pInfo->colBytes, false); -// //TODO: handle ts output -// } -// -// return pResInfo->numOfRes; -//} - - bool getTailFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0); SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); @@ -3812,7 +3806,6 @@ int32_t tailFunction(SqlFunctionCtx* pCtx) { TSKEY* tsList = (int64_t*)pInput->pPTS->pData; SColumnInfoData* pInputCol = pInput->pData[0]; - SColumnInfoData* pTsOutput = pCtx->pTsOutput; SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; int32_t startOffset = pCtx->offset; @@ -3879,7 +3872,6 @@ bool uniqueFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { pInfo->numOfPoints = 0; pInfo->colType = pCtx->resDataInfo.type; pInfo->colBytes = pCtx->resDataInfo.bytes; - pInfo->hasNull = false; if (pInfo->pHash != NULL) { taosHashClear(pInfo->pHash); } else { @@ -3917,8 +3909,6 @@ static void doUniqueAdd(SUniqueInfo* pInfo, char *data, TSKEY ts, bool isNull) { } else if (pHashItem->timestamp > ts) { pHashItem->timestamp = ts; } - - return; } int32_t uniqueFunction(SqlFunctionCtx* pCtx) { @@ -3960,7 +3950,7 @@ int32_t uniqueFunction(SqlFunctionCtx* pCtx) { int32_t uniqueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - SUniqueInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + SUniqueInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); @@ -3973,3 +3963,260 @@ int32_t uniqueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } +typedef struct STwaInfo { + double dOutput; + SPoint1 p; + STimeWindow win; +} STwaInfo; + +bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(STwaInfo); + return true; +} + +bool twaFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + STwaInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + pInfo->p.key = INT64_MIN; + pInfo->win = TSWINDOW_INITIALIZER; + return true; +} + +static double twa_get_area(SPoint1 s, SPoint1 e) { + if ((s.val >= 0 && e.val >= 0)|| (s.val <=0 && e.val <= 0)) { + return (s.val + e.val) * (e.key - s.key) / 2; + } + + double x = (s.key * e.val - e.key * s.val)/(e.val - s.val); + double val = (s.val * (x - s.key) + e.val * (e.key - x)) / 2; + return val; +} + +#define INIT_INTP_POINT(_p, _k, _v) \ + do { \ + (_p).key = (_k); \ + (_p).val = (_v); \ + } while (0) + +int32_t twaFunction(SqlFunctionCtx* pCtx) { + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pInputCol = pInput->pData[0]; + + TSKEY* tsList = (int64_t*)pInput->pPTS->pData; + + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + + STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + SPoint1 *last = &pInfo->p; + int32_t numOfElems = 0; + + int32_t i = pInput->startRowIndex; + if (pCtx->start.key != INT64_MIN) { + ASSERT((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || + (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); + + ASSERT(last->key == INT64_MIN); + last->key = tsList[i]; + + GET_TYPED_DATA(last->val, double, pInputCol->info.type, colDataGetData(pInputCol, i)); + + pInfo->dOutput += twa_get_area(pCtx->start, *last); + pInfo->win.skey = pCtx->start.key; + numOfElems++; + i += 1; + } else if (pInfo->p.key == INT64_MIN) { + last->key = tsList[i]; + GET_TYPED_DATA(last->val, double, pInputCol->info.type, colDataGetData(pInputCol, i)); + + pInfo->win.skey = last->key; + numOfElems++; + i += 1; + } + + SPoint1 st = {0}; + + // calculate the value of + switch(pInputCol->info.type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t *val = (int8_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + + INIT_INTP_POINT(st, tsList[i], val[i]); + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + + case TSDB_DATA_TYPE_SMALLINT: { + int16_t *val = (int16_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + + INIT_INTP_POINT(st, tsList[i], val[i]); + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_INT: { + int32_t *val = (int32_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + + INIT_INTP_POINT(st, tsList[i], val[i]); + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_BIGINT: { + int64_t *val = (int64_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + + INIT_INTP_POINT(st, tsList[i], val[i]); + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + float *val = (float*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + + INIT_INTP_POINT(st, tsList[i], val[i]); + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + double *val = (double*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + + INIT_INTP_POINT(st, tsList[i], val[i]); + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t *val = (uint8_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + + INIT_INTP_POINT(st, tsList[i], val[i]); + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t *val = (uint16_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + + INIT_INTP_POINT(st, tsList[i], val[i]); + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_UINT: { + uint32_t *val = (uint32_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + + INIT_INTP_POINT(st, tsList[i], val[i]); + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + uint64_t *val = (uint64_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + + INIT_INTP_POINT(st, tsList[i], val[i]); + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + + default: ASSERT(0); + } + + // the last interpolated time window value + if (pCtx->end.key != INT64_MIN) { + pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end); + pInfo->p = pCtx->end; + } + + pInfo->win.ekey = pInfo->p.key; + + SET_VAL(pResInfo, numOfElems, 1); + return TSDB_CODE_SUCCESS; +} + +/* + * To copy the input to interResBuf to avoid the input buffer space be over writen + * by next input data. The TWA function only applies to each table, so no merge procedure + * is required, we simply copy to the resut ot interResBuffer. + */ +//void twa_function_copy(SQLFunctionCtx *pCtx) { +// assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); +// SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); +// +// memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); +// pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult; +//} + +int32_t twaFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + + STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); + if (pResInfo->numOfRes == 0) { + pResInfo->isNullRes = 1; + } else { + // assert(pInfo->win.ekey == pInfo->p.key && pInfo->hasResult == pResInfo->hasResult); + if (pInfo->win.ekey == pInfo->win.skey) { + pInfo->dOutput = pInfo->p.val; + } else { + pInfo->dOutput = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey); + } + + pResInfo->numOfRes = 1; + } + + return functionFinalize(pCtx, pBlock); +} + diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 611ae8d81fdc681c28936456b5b46c0a7e09d4c0..67efdc5c0b91266500e3146b3c2d76350f28087d 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -226,6 +226,9 @@ static SColumnNode* createColumnByFunc(const SFunctionNode* pFunc) { } bool fmIsDistExecFunc(int32_t funcId) { + if (fmIsUserDefinedFunc(funcId)) { + return false; + } if (!fmIsVectorFunc(funcId)) { return true; } diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 950655e480b2b3413f26bc56d4771461b0dc4277..e683a38cbd1fd97ac7ba081a65f2af8ac18b8fee 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -236,7 +236,7 @@ bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry) { bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry) { return pEntry->initialized; } - +#if 0 int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength, bool isSuperTable/*, SUdfInfo* pUdfInfo*/) { if (!isValidDataType(dataType)) { @@ -470,6 +470,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } +#endif static bool function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (pResultInfo->initialized) { diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index b91af2d1577fc994ccaa6b11b8e9044ffb88b594..703b19ced7e1abeee312a414aafe6b34b936c271 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -36,12 +36,7 @@ void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) { if (pNode->nodeType == TEXPR_BINARYEXPR_NODE || pNode->nodeType == TEXPR_UNARYEXPR_NODE) { doExprTreeDestroy(&pNode, fp); - } else if (pNode->nodeType == TEXPR_VALUE_NODE) { - taosVariantDestroy(pNode->pVal); - } else if (pNode->nodeType == TEXPR_COL_NODE) { - taosMemoryFreeClear(pNode->pSchema); } - taosMemoryFree(pNode); } @@ -49,15 +44,6 @@ static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { if (*pExpr == NULL) { return; } - - int32_t type = (*pExpr)->nodeType; - if (type == TEXPR_VALUE_NODE) { - taosVariantDestroy((*pExpr)->pVal); - taosMemoryFree((*pExpr)->pVal); - } else if (type == TEXPR_COL_NODE) { - taosMemoryFree((*pExpr)->pSchema); - } - taosMemoryFree(*pExpr); *pExpr = NULL; } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 441648e52b2ef78326d73d1944bcfbfd0009abc6..472d67260730ca10522ee0d07fc1d608b132688e 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -66,7 +66,7 @@ void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) { } static int32_t udfSpawnUdfd(SUdfdData* pData) { - fnInfo("dnode start spawning udfd"); + fnInfo("start to init udfd"); uv_process_options_t options = {0}; char path[PATH_MAX] = {0}; @@ -140,6 +140,8 @@ static int32_t udfSpawnUdfd(SUdfdData* pData) { if (err != 0) { fnError("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err)); + } else { + fnInfo("udfd is initialized"); } return err; } diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 83dcb6d7f010936f46e1f9d375be517d8d06432b..02c485fa837bf4b487c4fdbd099171ead448c7f0 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -401,9 +401,17 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { udf->bufSize = pFuncInfo->bufSize; char path[PATH_MAX] = {0}; + #ifdef WINDOWS + snprintf(path, sizeof(path), "%s%s.dll", TD_TMP_DIR_PATH, pFuncInfo->name); + #else snprintf(path, sizeof(path), "%s/lib%s.so", TD_TMP_DIR_PATH, pFuncInfo->name); + #endif TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); + if (file == NULL) { + fnError("udfd write udf shared library: %s failed, error: %d %s", path, errno, strerror(errno)); + msgInfo->code = TSDB_CODE_FILE_CORRUPTED; + } int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); if (count != pFuncInfo->codeSize) { fnError("udfd write udf shared library failed"); diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c index e58c9cc00a2c8b8b85d741dca1ce9a19beafa043..9443d5cb945d6dfdbea37b8a090e5f5ebf045a01 100644 --- a/source/libs/function/test/udf1.c +++ b/source/libs/function/test/udf1.c @@ -9,15 +9,15 @@ #undef free #define free free -int32_t udf1_init() { +DLL_EXPORT int32_t udf1_init() { return 0; } -int32_t udf1_destroy() { +DLL_EXPORT int32_t udf1_destroy() { return 0; } -int32_t udf1(SUdfDataBlock* block, SUdfColumn *resultCol) { +DLL_EXPORT int32_t udf1(SUdfDataBlock* block, SUdfColumn *resultCol) { SUdfColumnMeta *meta = &resultCol->colMeta; meta->bytes = 4; meta->type = TSDB_DATA_TYPE_INT; diff --git a/source/libs/function/test/udf2.c b/source/libs/function/test/udf2.c index 49d681f5eb72b5e1f2ecca3baa45139f3a6a3116..1c270f5cf45880f7d56975617932e45f318331f2 100644 --- a/source/libs/function/test/udf2.c +++ b/source/libs/function/test/udf2.c @@ -9,22 +9,22 @@ #undef free #define free free -int32_t udf2_init() { +DLL_EXPORT int32_t udf2_init() { return 0; } -int32_t udf2_destroy() { +DLL_EXPORT int32_t udf2_destroy() { return 0; } -int32_t udf2_start(SUdfInterBuf *buf) { +DLL_EXPORT int32_t udf2_start(SUdfInterBuf *buf) { *(int64_t*)(buf->buf) = 0; buf->bufLen = sizeof(double); buf->numOfResult = 0; return 0; } -int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { +DLL_EXPORT int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { double sumSquares = *(double*)interBuf->buf; int8_t numNotNull = 0; for (int32_t i = 0; i < block->numOfCols; ++i) { @@ -71,7 +71,7 @@ int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInte return 0; } -int32_t udf2_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) { +DLL_EXPORT int32_t udf2_finish(SUdfInterBuf* buf, SUdfInterBuf *resultData) { if (buf->numOfResult == 0) { resultData->numOfResult = 0; return 0; diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index ba3aea969f6c8a5214a3999a7d4ca2c68ec503ac..058f3c1915ab67c48b179c6c0b32ef56244c24e8 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -202,7 +202,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { char buf[128] = {0}; ICacheKey key = {.suid = p->suid, .colName = p->colName, .nColName = strlen(p->colName), .colType = p->colType}; int32_t sz = indexSerialCacheKey(&key, buf); - indexDebug("suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); + indexDebug("w suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); IndexCache** cache = taosHashGet(index->colObj, buf, sz); assert(*cache != NULL); @@ -330,7 +330,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result char buf[128] = {0}; ICacheKey key = { .suid = term->suid, .colName = term->colName, .nColName = strlen(term->colName), .colType = term->colType}; - indexDebug("suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); + indexDebug("r suid: %" PRIu64 ", colName: %s, colType: %d", key.suid, key.colName, key.colType); int32_t sz = indexSerialCacheKey(&key, buf); taosThreadMutexLock(&sIdx->mtx); diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 5310e1c3451dee18bd3a31922b2ce14f752ebc1d..74b1773e877058a26bfa84afdf87cb2f73fa6aee 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -402,16 +402,16 @@ int32_t indexConvertDataToStr(void* src, int8_t type, void** dst) { break; } case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY - tlen = taosEncodeBinary(NULL, src, strlen(src)); + tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeBinary(dst, src, strlen(src)); + tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); *dst = (char*)*dst - tlen; break; } case TSDB_DATA_TYPE_VARBINARY: - tlen = taosEncodeBinary(NULL, src, strlen(src)); + tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); - tlen = taosEncodeBinary(dst, src, strlen(src)); + tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); *dst = (char*)*dst - tlen; break; default: diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 766746dd2a695076d3ab524076bfa143ceba1c54..2e30ca704a7c3d58e06a1371124cad051167ae2a 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -38,7 +38,7 @@ typedef struct SIFParam { col_id_t colId; int64_t suid; // add later char dbName[TSDB_DB_NAME_LEN]; - char colName[TSDB_COL_NAME_LEN]; + char colName[TSDB_COL_NAME_LEN * 2 + 4]; SIndexMetaArg arg; } SIFParam; @@ -140,7 +140,7 @@ static int32_t sifGetValueFromNode(SNode *node, char **value) { dataLen = 0; } else if (*pData == TSDB_DATA_TYPE_NCHAR) { dataLen = varDataTLen(pData + CHAR_BYTES); - } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + } else if (*pData == TSDB_DATA_TYPE_DOUBLE) { dataLen = LONG_BYTES; } else if (*pData == TSDB_DATA_TYPE_BOOL) { dataLen = CHAR_BYTES; @@ -162,12 +162,28 @@ static int32_t sifGetValueFromNode(SNode *node, char **value) { return TSDB_CODE_SUCCESS; } +static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { + SOperatorNode *nd = (SOperatorNode *)node; + assert(nodeType(node) == QUERY_NODE_OPERATOR); + SColumnNode *l = (SColumnNode *)nd->pLeft; + SValueNode * r = (SValueNode *)nd->pRight; + + param->colId = l->colId; + param->colValType = l->node.resType.type; + memcpy(param->dbName, l->dbName, sizeof(l->dbName)); + + sprintf(param->colName, "%s_%s", l->colName, r->literal); + param->colValType = r->typeData; + return 0; + // memcpy(param->colName, l->colName, sizeof(l->colName)); +} static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { switch (nodeType(node)) { case QUERY_NODE_VALUE: { SValueNode *vn = (SValueNode *)node; SIF_ERR_RET(sifGetValueFromNode(node, ¶m->condValue)); param->colId = -1; + param->colValType = (uint8_t)(vn->node.resType.type); break; } case QUERY_NODE_COLUMN: { @@ -219,17 +235,31 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx indexError("invalid operation node, left: %p, rigth: %p", node->pLeft, node->pRight); SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) { + return code; + } SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam)); if (NULL == paramList) { SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SIF_ERR_JRET(sifInitParam(node->pLeft, ¶mList[0], ctx)); - if (nParam > 1) { - SIF_ERR_JRET(sifInitParam(node->pRight, ¶mList[1], ctx)); + if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR) { + SNode *interNode = (node->pLeft); + SIF_ERR_JRET(sifInitJsonParam(interNode, ¶mList[0], ctx)); + if (nParam > 1) { + SIF_ERR_JRET(sifInitParam(node->pRight, ¶mList[1], ctx)); + } + paramList[0].colValType = TSDB_DATA_TYPE_JSON; + *params = paramList; + return TSDB_CODE_SUCCESS; + } else { + SIF_ERR_JRET(sifInitParam(node->pLeft, ¶mList[0], ctx)); + if (nParam > 1) { + SIF_ERR_JRET(sifInitParam(node->pRight, ¶mList[1], ctx)); + } + *params = paramList; + return TSDB_CODE_SUCCESS; } - *params = paramList; - return TSDB_CODE_SUCCESS; _return: taosMemoryFree(paramList); SIF_RET(code); @@ -265,23 +295,24 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu typedef int (*Filter)(void *a, void *b, int16_t dtype); int sifGreaterThan(void *a, void *b, int16_t dtype) { - __compar_fn_t func = indexGetCompar(dtype); + __compar_fn_t func = getComparFunc(dtype, 0); return tDoCompare(func, QUERY_GREATER_THAN, a, b); } int sifGreaterEqual(void *a, void *b, int16_t dtype) { - __compar_fn_t func = indexGetCompar(dtype); + __compar_fn_t func = getComparFunc(dtype, 0); return tDoCompare(func, QUERY_GREATER_EQUAL, a, b); } int sifLessEqual(void *a, void *b, int16_t dtype) { - __compar_fn_t func = indexGetCompar(dtype); + __compar_fn_t func = getComparFunc(dtype, 0); return tDoCompare(func, QUERY_LESS_EQUAL, a, b); } int sifLessThan(void *a, void *b, int16_t dtype) { - __compar_fn_t func = indexGetCompar(dtype); + __compar_fn_t func = getComparFunc(dtype, 0); return (int)tDoCompare(func, QUERY_LESS_THAN, a, b); } int sifEqual(void *a, void *b, int16_t dtype) { - __compar_fn_t func = indexGetCompar(dtype); + __compar_fn_t func = getComparFunc(dtype, 0); + //__compar_fn_t func = indexGetCompar(dtype); return (int)tDoCompare(func, QUERY_TERM, a, b); } static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { @@ -305,39 +336,34 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { } static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { SIndexMetaArg *arg = &output->arg; -#ifdef USE_INVERTED_INDEX - SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, left->colValType, left->colName, strlen(left->colName), - right->condValue, strlen(right->condValue)); - if (tm == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } + int ret = 0; EIndexQueryType qtype = 0; SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); + if (left->colValType == TSDB_DATA_TYPE_JSON) { + SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, right->colValType, left->colName, strlen(left->colName), + right->condValue, strlen(right->condValue)); + if (tm == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } - SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); - indexMultiTermQueryAdd(mtm, tm, qtype); - int ret = indexSearch(arg->metaHandle, mtm, output->result); - indexDebug("index filter data size: %d", (int)taosArrayGetSize(output->result)); - indexMultiTermQueryDestroy(mtm); - return ret; -#else - EIndexQueryType qtype = 0; - SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); - bool reverse; - Filter filterFunc = sifGetFilterFunc(qtype, &reverse); + SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); + indexMultiTermQueryAdd(mtm, tm, qtype); + ret = tIndexJsonSearch(arg->ivtIdx, mtm, output->result); + } else { + bool reverse; + Filter filterFunc = sifGetFilterFunc(qtype, &reverse); - SMetaFltParam param = {.suid = arg->suid, - .cid = left->colId, - .type = left->colValType, - .val = right->condValue, - .reverse = reverse, - .filterFunc = filterFunc}; + SMetaFltParam param = {.suid = arg->suid, + .cid = left->colId, + .type = left->colValType, + .val = right->condValue, + .reverse = reverse, + .filterFunc = filterFunc}; - int ret = metaFilteTableIds(arg->metaEx, ¶m, output->result); + ret = metaFilteTableIds(arg->metaEx, ¶m, output->result); + } return ret; -#endif - return 0; } static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { @@ -391,71 +417,106 @@ static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output int id = OP_TYPE_NMATCH; return sifDoIndex(left, right, id, output); } +static int32_t sifJsonContains(SIFParam *left, SIFParam *right, SIFParam *output) { + // return 0 + return 0; +} +static int32_t sifJsonGetValue(SIFParam *left, SIFParam *rigth, SIFParam *output) { + // return 0 + return 0; +} static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) { // add more except return TSDB_CODE_QRY_INVALID_INPUT; } -static sif_func_t sifGetOperFn(int32_t funcId) { +static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *status) { // impl later + *status = SFLT_ACCURATE_INDEX; switch (funcId) { case OP_TYPE_GREATER_THAN: - return sifGreaterThanFunc; + *func = sifGreaterThanFunc; + return 0; case OP_TYPE_GREATER_EQUAL: - return sifGreaterEqualFunc; + *func = sifGreaterEqualFunc; + return 0; case OP_TYPE_LOWER_THAN: - return sifLessThanFunc; + *func = sifLessThanFunc; + return 0; case OP_TYPE_LOWER_EQUAL: - return sifLessEqualFunc; + *func = sifLessEqualFunc; + return 0; case OP_TYPE_EQUAL: - return sifEqualFunc; + *func = sifEqualFunc; + return 0; case OP_TYPE_NOT_EQUAL: - return sifNotEqualFunc; + *status = SFLT_NOT_INDEX; + *func = sifNotEqualFunc; + return 0; case OP_TYPE_IN: - return sifInFunc; + *status = SFLT_NOT_INDEX; + *func = sifInFunc; + return 0; case OP_TYPE_NOT_IN: - return sifNotInFunc; + *status = SFLT_NOT_INDEX; + *func = sifNotInFunc; + return 0; case OP_TYPE_LIKE: - return sifLikeFunc; + *status = SFLT_NOT_INDEX; + *func = sifLikeFunc; + return 0; case OP_TYPE_NOT_LIKE: - return sifNotLikeFunc; + *status = SFLT_NOT_INDEX; + *func = sifNotLikeFunc; + return 0; case OP_TYPE_MATCH: - return sifMatchFunc; + *status = SFLT_NOT_INDEX; + *func = sifMatchFunc; + return 0; case OP_TYPE_NMATCH: - return sifNotMatchFunc; + *status = SFLT_NOT_INDEX; + *func = sifNotMatchFunc; + return 0; + case OP_TYPE_JSON_CONTAINS: + *status = SFLT_ACCURATE_INDEX; + *func = sifJsonContains; + return 0; + case OP_TYPE_JSON_GET_VALUE: + *status = SFLT_ACCURATE_INDEX; + *func = sifJsonGetValue; + return 0; default: - return sifNullFunc; + *status = SFLT_NOT_INDEX; + *func = sifNullFunc; + return 0; } - return sifNullFunc; + return 0; } +// typedef struct filterFuncDict { + static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { int32_t code = 0; int32_t nParam = sifGetOperParamNum(node->opType); if (nParam <= 1) { - SIF_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) { + return code; } - SIFParam *params = NULL; - SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); + SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); // ugly code, refactor later output->arg = ctx->arg; - sif_func_t operFn = sifGetOperFn(node->opType); - if (ctx->noExec && operFn == NULL) { - output->status = SFLT_NOT_INDEX; - } else { - output->status = SFLT_ACCURATE_INDEX; - } - - if (ctx->noExec) { - SIF_RET(code); + sif_func_t operFn = sifNullFunc; + code = sifGetOperFn(node->opType, &operFn, &output->status); + if (!ctx->noExec) { + code = operFn(¶ms[0], nParam > 1 ? ¶ms[1] : NULL, output); } - return operFn(¶ms[0], nParam > 1 ? ¶ms[1] : NULL, output); -_return: taosMemoryFree(params); - SIF_RET(code); + return code; } static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *output) { @@ -465,7 +526,7 @@ static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *ou return TSDB_CODE_QRY_INVALID_INPUT; } - int32_t code; + int32_t code = TSDB_CODE_SUCCESS; SIFParam *params = NULL; SIF_ERR_RET(sifInitParamList(¶ms, node->pParameterList, ctx)); @@ -477,7 +538,7 @@ static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *ou } else if (node->condType == LOGIC_COND_TYPE_OR) { taosArrayAddAll(output->result, params[m].result); } else if (node->condType == LOGIC_COND_TYPE_NOT) { - taosArrayAddAll(output->result, params[m].result); + // taosArrayAddAll(output->result, params[m].result); } } } else { @@ -552,7 +613,9 @@ EDealRes sifCalcWalker(SNode *node, void *context) { if (QUERY_NODE_LOGIC_CONDITION == nodeType(node)) { return sifWalkLogic(node, ctx); } + if (QUERY_NODE_OPERATOR == nodeType(node)) { + indexInfo("node type for index filter, type: %d", nodeType(node)); return sifWalkOper(node, ctx); } diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index 2835084a81b87e358916c20ce0e6c70cf6884021..040460ae5c9662404182b93780075cc88cb93c57 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -92,16 +92,14 @@ target_link_libraries (idxJsonUT index ) -if(NOT TD_WINDOWS) - add_test( - NAME idxtest - COMMAND idxTest - ) - add_test( - NAME idxJsonUT - COMMAND idxJsonUT - ) -endif(NOT TD_WINDOWS) +add_test( + NAME idxtest + COMMAND idxTest +) +add_test( + NAME idxJsonUT + COMMAND idxJsonUT +) add_test( NAME idxUtilUT COMMAND idxUtilUT diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 74a30c3387ea3c3133e4e4f82ffd3dd8dc38f540..327a4ae219322a9d4a773001cde7f13397d75e4d 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -498,10 +498,15 @@ TEST_F(IndexTFileEnv, test_tfile_write) { } taosArrayDestroy(data); - std::string colName("voltage"); - std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + std::string colName("voltage"); + std::string colVal("ab"); + + char buf[256] = {0}; + int16_t sz = colVal.size(); + memcpy(buf, (uint16_t*)&sz, 2); + memcpy(buf + 2, colVal.c_str(), colVal.size()); + SIndexTerm* term = + indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), buf, sizeof(buf)); SIndexTermQuery query = {term, QUERY_TERM}; SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); @@ -564,6 +569,18 @@ class IndexCacheEnv : public ::testing::Test { CacheObj* coj; }; +SIndexTerm* indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, + int32_t nColName, const char* colVal, int32_t nColVal) { + char buf[256] = {0}; + int16_t sz = nColVal; + memcpy(buf, (uint16_t*)&sz, 2); + memcpy(buf + 2, colVal, nColVal); + if (colType == TSDB_DATA_TYPE_BINARY) { + return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf)); + } else { + return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal); + } +} #define MAX_TERM_KEY_LEN 128 TEST_F(IndexCacheEnv, cache_test) { int version = 0; @@ -574,37 +591,37 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colName("voltage"); { std::string colVal("v1"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); // indexTermDestry(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } @@ -612,15 +629,15 @@ TEST_F(IndexCacheEnv, cache_test) { std::cout << "--------first----------" << std::endl; { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); } { std::string colVal("v4"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); indexTermDestroy(term); } @@ -630,8 +647,8 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colVal("v4"); for (size_t i = 0; i < 10; i++) { colVal[colVal.size() - 1] = 'a' + i; - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); indexTermDestroy(term); } @@ -640,8 +657,8 @@ TEST_F(IndexCacheEnv, cache_test) { // begin query { std::string colVal("v3"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -655,8 +672,8 @@ TEST_F(IndexCacheEnv, cache_test) { } { std::string colVal("v2"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexTermQuery query = {term, QUERY_TERM}; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -690,8 +707,8 @@ class IndexObj { return ret; } void Del(const std::string& colName, const std::string& colVal, uint64_t uid) { - SIndexTerm* term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); Put(terms, uid); @@ -699,8 +716,8 @@ class IndexObj { } int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world", size_t numOfTable = 100 * 10000) { - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = 0; i < numOfTable; i++) { @@ -721,8 +738,8 @@ class IndexObj { // opt tColVal[taosRand() % colValSize] = 'a' + k % 26; } - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - tColVal.c_str(), tColVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + tColVal.c_str(), tColVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t j = 0; j < skip; j++) { @@ -757,8 +774,8 @@ class IndexObj { int SearchOne(const std::string& colName, const std::string& colVal) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); @@ -779,8 +796,8 @@ class IndexObj { } int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); @@ -804,16 +821,16 @@ class IndexObj { void PutOne(const std::string& colName, const std::string& colVal) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, 10); indexMultiTermDestroy(terms); } void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) { SIndexMultiTerm* terms = indexMultiTermCreate(); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); indexMultiTermAdd(terms, term); Put(terms, val); indexMultiTermDestroy(terms); @@ -858,8 +875,8 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = 0; i < targetSize; i++) { @@ -873,8 +890,8 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = 0; i < size; i++) { @@ -888,8 +905,8 @@ TEST_F(IndexEnv2, testIndexOpen) { size_t size = 200; std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = size * 3; i < size * 4; i++) { @@ -903,8 +920,8 @@ TEST_F(IndexEnv2, testIndexOpen) { { std::string colName("tag1"), colVal("Hello"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); indexMultiTermQueryAdd(mq, term, QUERY_TERM); SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); @@ -926,8 +943,8 @@ TEST_F(IndexEnv2, testEmptyIndexOpen) { { std::string colName("tag1"), colVal("Hello"); - SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); for (size_t i = 0; i < targetSize; i++) { diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 48ce8839c459bb2c523d710f1804346f2bede33a..f795fbad01df5aa076b83d2770bed936efcd446f 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -19,6 +19,18 @@ static std::string dir = TD_TMP_DIR_PATH "json"; static std::string logDir = TD_TMP_DIR_PATH "log"; +SIndexTerm* indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, + int32_t nColName, const char* colVal, int32_t nColVal) { + char buf[256] = {0}; + int16_t sz = nColVal; + memcpy(buf, (uint16_t*)&sz, 2); + memcpy(buf + 2, colVal, nColVal); + if (colType == TSDB_DATA_TYPE_BINARY) { + return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf)); + } else { + return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal); + } +} static void initLog() { const char* defaultLogFileNamePrefix = "taoslog"; const int32_t maxLogFileNum = 10; @@ -59,8 +71,8 @@ class JsonEnv : public ::testing::Test { static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId, int8_t operType = ADD_VALUE) { - SIndexTerm* term = - indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen); + SIndexTerm* term = indexTermCreateT(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), + (const char*)data, dlen); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); tIndexJsonPut(index, terms, (int64_t)tableId); @@ -70,8 +82,8 @@ static void WriteData(SIndexJson* index, const std::string& colName, int8_t dtyp static void delData(SIndexJson* index, const std::string& colName, int8_t dtype, void* data, int dlen, int tableId, int8_t operType = DEL_VALUE) { - SIndexTerm* term = - indexTermCreate(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), (const char*)data, dlen); + SIndexTerm* term = indexTermCreateT(1, (SIndexOperOnColumn)operType, dtype, colName.c_str(), colName.size(), + (const char*)data, dlen); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); tIndexJsonPut(index, terms, (int64_t)tableId); @@ -83,7 +95,7 @@ static void Search(SIndexJson* index, const std::string& colNam, int8_t dtype, v std::string colName(colNam); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, dtype, colName.c_str(), colName.size(), (const char*)data, dlen); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, dtype, colName.c_str(), colName.size(), (const char*)data, dlen); SArray* res = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, (EIndexQueryType)filterType); @@ -95,8 +107,8 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -108,8 +120,8 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("ab1"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -121,8 +133,8 @@ TEST_F(JsonEnv, testWrite) { { std::string colName("voltage"); std::string colVal("123"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -136,8 +148,8 @@ TEST_F(JsonEnv, testWrite) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_TERM); @@ -150,8 +162,8 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("test"); std::string colVal("ab"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -165,8 +177,8 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("abxxxxxxxxxxxx"); for (int i = 0; i < 10; i++) { colVal[i % colVal.size()] = '0' + i % 128; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -179,8 +191,8 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -194,8 +206,8 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_TERM); @@ -209,8 +221,8 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -224,8 +236,8 @@ TEST_F(JsonEnv, testWriteMillonData) { std::string colVal("ab"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL); @@ -241,8 +253,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { std::string colName("test"); // std::string colVal("10"); int val = 10; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -254,8 +266,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { { std::string colName("test2"); int val = 20; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -267,8 +279,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { { std::string colName("test"); int val = 15; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -280,8 +292,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { { std::string colName("test2"); const char* val = "test"; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - (const char*)val, strlen(val)); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + (const char*)val, strlen(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -294,8 +306,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { std::string colName("test"); int val = 15; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_TERM); @@ -308,8 +320,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { int val = 15; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -323,8 +335,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { ; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(int)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(int)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL); @@ -338,8 +350,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { // std::string colVal("10"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN); @@ -353,8 +365,8 @@ TEST_F(JsonEnv, testWriteJsonNumberData) { // std::string colVal("10"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL); @@ -368,8 +380,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { { std::string colName("test1"); int val = 10; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -381,8 +393,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { { std::string colName("test"); std::string colVal("xxxxxxxxxxxxxxxxxxx"); - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), - colVal.c_str(), colVal.size()); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -396,8 +408,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_TERM); @@ -410,8 +422,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(int)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(int)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -425,8 +437,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL); @@ -439,8 +451,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_THAN); @@ -453,8 +465,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { int val = 10; SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_EQUAL); @@ -466,8 +478,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { std::string colName("other_column"); int val = 100; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -482,8 +494,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { // std::string colVal("10"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_LESS_THAN); @@ -494,8 +506,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { { std::string colName("test1"); int val = 15; - SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* term = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term); @@ -510,8 +522,8 @@ TEST_F(JsonEnv, testWriteJsonTfileAndCache_INT) { // std::string colVal("10"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); - SIndexTerm* q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), - (const char*)&val, sizeof(val)); + SIndexTerm* q = indexTermCreateT(1, ADD_VALUE, TSDB_DATA_TYPE_INT, colName.c_str(), colName.size(), + (const char*)&val, sizeof(val)); SArray* result = taosArrayInit(1, sizeof(uint64_t)); indexMultiTermQueryAdd(mq, q, QUERY_GREATER_EQUAL); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 94661388cb90e8ac41a7cca5e24871db8b8d7d98..f277d30eb1e79ce885c7a8baba81b947ac23671f 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -98,6 +98,7 @@ static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { COPY_SCALAR_FIELD(tableType); COPY_SCALAR_FIELD(colId); COPY_SCALAR_FIELD(colType); + COPY_SCALAR_FIELD(hasIndex); COPY_CHAR_ARRAY_FIELD(dbName); COPY_CHAR_ARRAY_FIELD(tableName); COPY_CHAR_ARRAY_FIELD(tableAlias); @@ -142,7 +143,7 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_VARBINARY:{ + case TSDB_DATA_TYPE_VARBINARY: { int32_t len = varDataTLen(pSrc->datum.p) + 1; pDst->datum.p = taosMemoryCalloc(1, len); if (NULL == pDst->datum.p) { @@ -315,7 +316,8 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pScanCols); CLONE_NODE_LIST_FIELD(pScanPseudoCols); - CLONE_OBJECT_FIELD(pMeta, tableMetaClone); + COPY_SCALAR_FIELD(tableType); + COPY_SCALAR_FIELD(tableId); CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone); COPY_SCALAR_FIELD(scanType); COPY_OBJECT_FIELD(scanSeq[0], sizeof(uint8_t) * 2); @@ -364,9 +366,15 @@ static SNode* logicProjectCopy(const SProjectLogicNode* pSrc, SProjectLogicNode* return (SNode*)pDst; } -static SNode* logicVnodeModifCopy(const SVnodeModifLogicNode* pSrc, SVnodeModifLogicNode* pDst) { +static SNode* logicVnodeModifCopy(const SVnodeModifyLogicNode* pSrc, SVnodeModifyLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + COPY_SCALAR_FIELD(modifyType); COPY_SCALAR_FIELD(msgType); + CLONE_NODE_FIELD(pModifyRows); + COPY_SCALAR_FIELD(tableId); + COPY_SCALAR_FIELD(tableType); + COPY_CHAR_ARRAY_FIELD(tableFName); + COPY_OBJECT_FIELD(deleteTimeRange, sizeof(STimeWindow)); return (SNode*)pDst; } @@ -399,6 +407,7 @@ static SNode* logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* pD COPY_SCALAR_FIELD(triggerType); COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(filesFactor); + COPY_SCALAR_FIELD(intervalAlgo); return (SNode*)pDst; } @@ -540,8 +549,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return logicAggCopy((const SAggLogicNode*)pNode, (SAggLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_PROJECT: return logicProjectCopy((const SProjectLogicNode*)pNode, (SProjectLogicNode*)pDst); - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: - return logicVnodeModifCopy((const SVnodeModifLogicNode*)pNode, (SVnodeModifLogicNode*)pDst); + case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: + return logicVnodeModifCopy((const SVnodeModifyLogicNode*)pNode, (SVnodeModifyLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return logicExchangeCopy((const SExchangeLogicNode*)pNode, (SExchangeLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_MERGE: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 4bce6381cde1cda187d11f5691bdb35b4e3dfca6..c54289bc568a2c3492d68ae0bce5f929e174fd23 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -178,6 +178,8 @@ const char* nodesNodeName(ENodeType type) { return "ShowQueriesStmt"; case QUERY_NODE_SHOW_VNODES_STMT: return "ShowVnodeStmt"; + case QUERY_NODE_DELETE_STMT: + return "DeleteStmt"; case QUERY_NODE_LOGIC_PLAN_SCAN: return "LogicScan"; case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -186,8 +188,8 @@ const char* nodesNodeName(ENodeType type) { return "LogicAgg"; case QUERY_NODE_LOGIC_PLAN_PROJECT: return "LogicProject"; - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: - return "LogicVnodeModif"; + case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: + return "LogicVnodeModify"; case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return "LogicExchange"; case QUERY_NODE_LOGIC_PLAN_MERGE: @@ -226,10 +228,14 @@ const char* nodesNodeName(ENodeType type) { return "PhysiMerge"; case QUERY_NODE_PHYSICAL_PLAN_SORT: return "PhysiSort"; - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: - return "PhysiInterval"; + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: + return "PhysiHashInterval"; case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: return "PhysiStreamInterval"; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: + return "PhysiStreamFinalInterval"; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: + return "PhysiStreamSemiInterval"; case QUERY_NODE_PHYSICAL_PLAN_FILL: return "PhysiFill"; case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: @@ -238,12 +244,16 @@ const char* nodesNodeName(ENodeType type) { return "PhysiStreamSessionWindow"; case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: return "PhysiStateWindow"; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW: + return "PhysiStreamStateWindow"; case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return "PhysiPartition"; case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return "PhysiDispatch"; case QUERY_NODE_PHYSICAL_PLAN_INSERT: return "PhysiInsert"; + case QUERY_NODE_PHYSICAL_PLAN_DELETE: + return "PhysiDelete"; case QUERY_NODE_PHYSICAL_SUBPLAN: return "PhysiSubplan"; case QUERY_NODE_PHYSICAL_PLAN: @@ -496,8 +506,8 @@ static int32_t jsonToLogicPlanNode(const SJson* pJson, void* pObj) { static const char* jkScanLogicPlanScanCols = "ScanCols"; static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols"; -static const char* jkScanLogicPlanTableMetaSize = "TableMetaSize"; -static const char* jkScanLogicPlanTableMeta = "TableMeta"; +static const char* jkScanLogicPlanTableId = "TableId"; +static const char* jkScanLogicPlanTableType = "TableType"; static const char* jkScanLogicPlanTagCond = "TagCond"; static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { @@ -511,10 +521,10 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { code = nodeListToJson(pJson, jkScanLogicPlanScanPseudoCols, pNode->pScanPseudoCols); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, TABLE_META_SIZE(pNode->pMeta)); + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableId, pNode->tableId); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkScanLogicPlanTableMeta, tableMetaToJson, pNode->pMeta); + code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableType, pNode->tableType); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkScanLogicPlanTagCond, nodeToJson, pNode->pTagCond); @@ -535,10 +545,10 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) { code = jsonToNodeList(pJson, jkScanLogicPlanScanPseudoCols, &pNode->pScanPseudoCols); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkScanLogicPlanTableMetaSize, &objSize); + code = tjsonGetUBigIntValue(pJson, jkScanLogicPlanTableId, &pNode->tableId); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonMakeObject(pJson, jkScanLogicPlanTableMeta, jsonToTableMeta, (void**)&pNode->pMeta, objSize); + code = tjsonGetTinyIntValue(pJson, jkScanLogicPlanTableType, &pNode->tableType); } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkScanLogicPlanTagCond, &pNode->pTagCond); @@ -570,7 +580,7 @@ static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanSlimit, pNode->slimit); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, pNode->soffset); + code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanSoffset, pNode->soffset); } return code; @@ -593,7 +603,45 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) { code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanSlimit, &pNode->slimit); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBigIntValue(pJson, jkScanLogicPlanTableMetaSize, &pNode->soffset); + code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanSoffset, &pNode->soffset); + } + + return code; +} + +static const char* jkVnodeModifyLogicPlanModifyType = "ModifyType"; +static const char* jkVnodeModifyLogicPlanMsgType = "MsgType"; +static const char* jkVnodeModifyLogicPlanModifyRows = "ModifyRows"; + +static int32_t logicVnodeModifyNodeToJson(const void* pObj, SJson* pJson) { + const SVnodeModifyLogicNode* pNode = (const SVnodeModifyLogicNode*)pObj; + + int32_t code = logicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVnodeModifyLogicPlanModifyType, pNode->modifyType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVnodeModifyLogicPlanMsgType, pNode->msgType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkVnodeModifyLogicPlanModifyRows, nodeToJson, pNode->pModifyRows); + } + + return code; +} + +static int32_t jsonToLogicVnodeModifyNode(const SJson* pJson, void* pObj) { + SVnodeModifyLogicNode* pNode = (SVnodeModifyLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkVnodeModifyLogicPlanModifyType, pNode->modifyType, code); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkVnodeModifyLogicPlanMsgType, &pNode->msgType); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkVnodeModifyLogicPlanModifyRows, &pNode->pModifyRows); } return code; @@ -1947,6 +1995,58 @@ static int32_t physiDispatchNodeToJson(const void* pObj, SJson* pJson) { return static int32_t jsonToPhysiDispatchNode(const SJson* pJson, void* pObj) { return jsonToPhysicDataSinkNode(pJson, pObj); } +static const char* jkDeletePhysiPlanTableId = "TableId"; +static const char* jkDeletePhysiPlanTableType = "TableType"; +static const char* jkDeletePhysiPlanTableFName = "TableFName"; +static const char* jkDeletePhysiPlanDeleteTimeRangeStartKey = "DeleteTimeRangeStartKey"; +static const char* jkDeletePhysiPlanDeleteTimeRangeEndKey = "DeleteTimeRangeEndKey"; + +static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) { + const SDataDeleterNode* pNode = (const SDataDeleterNode*)pObj; + + int32_t code = physicDataSinkNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanTableId, pNode->tableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanTableType, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDeletePhysiPlanTableFName, pNode->tableFName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanDeleteTimeRangeStartKey, pNode->deleteTimeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanDeleteTimeRangeEndKey, pNode->deleteTimeRange.ekey); + } + + return code; +} + +static int32_t jsonToPhysiDeleteNode(const SJson* pJson, void* pObj) { + SDataDeleterNode* pNode = (SDataDeleterNode*)pObj; + + int32_t code = jsonToPhysicDataSinkNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUBigIntValue(pJson, jkDeletePhysiPlanTableId, &pNode->tableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkDeletePhysiPlanTableType, &pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDeletePhysiPlanTableFName, pNode->tableFName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkDeletePhysiPlanDeleteTimeRangeStartKey, &pNode->deleteTimeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkDeletePhysiPlanDeleteTimeRangeEndKey, &pNode->deleteTimeRange.ekey); + } + + return code; +} + static const char* jkQueryNodeAddrId = "Id"; static const char* jkQueryNodeAddrInUse = "InUse"; static const char* jkQueryNodeAddrNumOfEps = "NumOfEps"; @@ -2738,6 +2838,28 @@ static int32_t jsonToOrderByExprNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkStateWindowCol = "StateWindowCol"; +static const char* jkStateWindowExpr = "StateWindowExpr"; + +static int32_t stateWindowNodeToJson(const void* pObj, SJson* pJson) { + const SStateWindowNode* pNode = (const SStateWindowNode*)pObj; + int32_t code = tjsonAddObject(pJson, jkStateWindowCol, nodeToJson, pNode->pCol); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkStateWindowExpr, nodeToJson, pNode->pExpr); + } + return code; +} + +static int32_t jsonToStateWindowNode(const SJson* pJson, void* pObj) { + SStateWindowNode* pNode = (SStateWindowNode*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkStateWindowCol, (SNode**)&pNode->pCol); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkStateWindowExpr, (SNode**)&pNode->pExpr); + } + return code; +} + static const char* jkSessionWindowTsPrimaryKey = "TsPrimaryKey"; static const char* jkSessionWindowGap = "Gap"; @@ -3492,6 +3614,73 @@ static int32_t jsonToCreateTopicStmt(const SJson* pJson, void* pObj) { return code; } +static const char* jkDeleteStmtFromTable = "FromTable"; +static const char* jkDeleteStmtWhere = "Where"; +static const char* jkDeleteStmtCountFunc = "CountFunc"; +static const char* jkDeleteStmtTagIndexCond = "TagIndexCond"; +static const char* jkDeleteStmtTimeRangeStartKey = "TimeRangeStartKey"; +static const char* jkDeleteStmtTimeRangeEndKey = "TimeRangeEndKey"; +static const char* jkDeleteStmtPrecision = "Precision"; +static const char* jkDeleteStmtDeleteZeroRows = "DeleteZeroRows"; + +static int32_t deleteStmtToJson(const void* pObj, SJson* pJson) { + const SDeleteStmt* pNode = (const SDeleteStmt*)pObj; + + int32_t code = tjsonAddObject(pJson, jkDeleteStmtFromTable, nodeToJson, pNode->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDeleteStmtWhere, nodeToJson, pNode->pWhere); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDeleteStmtCountFunc, nodeToJson, pNode->pCountFunc); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDeleteStmtTagIndexCond, nodeToJson, pNode->pTagIndexCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDeleteStmtTimeRangeStartKey, pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDeleteStmtTimeRangeEndKey, pNode->timeRange.ekey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDeleteStmtPrecision, pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDeleteStmtDeleteZeroRows, pNode->deleteZeroRows); + } + + return code; +} + +static int32_t jsonToDeleteStmt(const SJson* pJson, void* pObj) { + SDeleteStmt* pNode = (SDeleteStmt*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkDeleteStmtFromTable, &pNode->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDeleteStmtWhere, &pNode->pWhere); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDeleteStmtCountFunc, &pNode->pCountFunc); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDeleteStmtTagIndexCond, &pNode->pTagIndexCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkDeleteStmtTimeRangeStartKey, &pNode->timeRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkDeleteStmtTimeRangeEndKey, &pNode->timeRange.ekey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUTinyIntValue(pJson, jkDeleteStmtPrecision, &pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDeleteStmtDeleteZeroRows, &pNode->deleteZeroRows); + } + + return code; +} + static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { switch (nodeType(pObj)) { case QUERY_NODE_COLUMN: @@ -3515,8 +3704,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_ORDER_BY_EXPR: return orderByExprNodeToJson(pObj, pJson); case QUERY_NODE_LIMIT: - case QUERY_NODE_STATE_WINDOW: break; + case QUERY_NODE_STATE_WINDOW: + return stateWindowNodeToJson(pObj, pJson); case QUERY_NODE_SESSION_WINDOW: return sessionWindowNodeToJson(pObj, pJson); case QUERY_NODE_INTERVAL_WINDOW: @@ -3563,6 +3753,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { break; case QUERY_NODE_CREATE_TOPIC_STMT: return createTopicStmtToJson(pObj, pJson); + case QUERY_NODE_DELETE_STMT: + return deleteStmtToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_SCAN: return logicScanNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -3571,8 +3763,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return logicAggNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_PROJECT: return logicProjectNodeToJson(pObj, pJson); - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: - break; + case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: + return logicVnodeModifyNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return logicExchangeNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_MERGE: @@ -3609,8 +3801,10 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiMergeNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SORT: return physiSortNodeToJson(pObj, pJson); - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: return physiIntervalNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_FILL: return physiFillNodeToJson(pObj, pJson); @@ -3618,6 +3812,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION_WINDOW: return physiSessionWindowNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW: return physiStateWindowNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return physiPartitionNodeToJson(pObj, pJson); @@ -3625,6 +3820,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiDispatchNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INSERT: break; + case QUERY_NODE_PHYSICAL_PLAN_DELETE: + return physiDeleteNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_SUBPLAN: return subplanToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN: @@ -3654,6 +3851,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToTempTableNode(pJson, pObj); case QUERY_NODE_ORDER_BY_EXPR: return jsonToOrderByExprNode(pJson, pObj); + case QUERY_NODE_STATE_WINDOW: + return jsonToStateWindowNode(pJson, pObj); case QUERY_NODE_SESSION_WINDOW: return jsonToSessionWindowNode(pJson, pObj); case QUERY_NODE_INTERVAL_WINDOW: @@ -3686,10 +3885,14 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToAlterDnodeStmt(pJson, pObj); case QUERY_NODE_CREATE_TOPIC_STMT: return jsonToCreateTopicStmt(pJson, pObj); + case QUERY_NODE_DELETE_STMT: + return jsonToDeleteStmt(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_SCAN: return jsonToLogicScanNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_PROJECT: return jsonToLogicProjectNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: + return jsonToLogicVnodeModifyNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return jsonToLogicExchangeNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_MERGE: @@ -3726,8 +3929,10 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiMergeNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_SORT: return jsonToPhysiSortNode(pJson, pObj); - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: return jsonToPhysiIntervalNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_FILL: return jsonToPhysiFillNode(pJson, pObj); @@ -3735,11 +3940,14 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION_WINDOW: return jsonToPhysiSessionWindowNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW: return jsonToPhysiStateWindowNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return jsonToPhysiPartitionNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return jsonToPhysiDispatchNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_DELETE: + return jsonToPhysiDeleteNode(pJson, pObj); case QUERY_NODE_PHYSICAL_SUBPLAN: return jsonToSubplan(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN: diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index ae1ff5744bcc48eeaec661137e01eeaf01684636..d9b61bb0028f4e2447849f8aaad12278b65f44ab 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -512,7 +512,7 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk } break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext); break; @@ -520,7 +520,8 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION_WINDOW: res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext); break; - case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: { + case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW: { SStateWinodwPhysiNode* pState = (SStateWinodwPhysiNode*)pNode; res = walkWindowPhysi((SWinodwPhysiNode*)pNode, order, walker, pContext); if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 51111ad864568649f5757e9b20f5f14190271c1a..16ce4d7a48d6915a46e65400058b978c30945c9d 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -208,6 +208,8 @@ SNodeptr nodesMakeNode(ENodeType type) { case QUERY_NODE_KILL_QUERY_STMT: case QUERY_NODE_KILL_TRANSACTION_STMT: return makeNode(type, sizeof(SKillStmt)); + case QUERY_NODE_DELETE_STMT: + return makeNode(type, sizeof(SDeleteStmt)); case QUERY_NODE_LOGIC_PLAN_SCAN: return makeNode(type, sizeof(SScanLogicNode)); case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -216,8 +218,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SAggLogicNode)); case QUERY_NODE_LOGIC_PLAN_PROJECT: return makeNode(type, sizeof(SProjectLogicNode)); - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: - return makeNode(type, sizeof(SVnodeModifLogicNode)); + case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: + return makeNode(type, sizeof(SVnodeModifyLogicNode)); case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return makeNode(type, sizeof(SExchangeLogicNode)); case QUERY_NODE_LOGIC_PLAN_MERGE: @@ -256,10 +258,16 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SMergePhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SORT: return makeNode(type, sizeof(SSortPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL: + return makeNode(type, sizeof(SSortMergeIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: return makeNode(type, sizeof(SStreamIntervalPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: + return makeNode(type, sizeof(SStreamFinalIntervalPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: + return makeNode(type, sizeof(SStreamSemiIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_FILL: return makeNode(type, sizeof(SFillPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: @@ -268,12 +276,16 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SStreamSessionWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW: return makeNode(type, sizeof(SStateWinodwPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW: + return makeNode(type, sizeof(SStreamStateWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return makeNode(type, sizeof(SPartitionPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: return makeNode(type, sizeof(SDataDispatcherNode)); case QUERY_NODE_PHYSICAL_PLAN_INSERT: return makeNode(type, sizeof(SDataInserterNode)); + case QUERY_NODE_PHYSICAL_PLAN_DELETE: + return makeNode(type, sizeof(SDataDeleterNode)); case QUERY_NODE_PHYSICAL_SUBPLAN: return makeNode(type, sizeof(SSubplan)); case QUERY_NODE_PHYSICAL_PLAN: @@ -553,7 +565,6 @@ void nodesDestroyNode(SNodeptr pNode) { SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyList(pLogicNode->pScanCols); - taosMemoryFreeClear(pLogicNode->pMeta); taosMemoryFreeClear(pLogicNode->pVgroupList); break; } @@ -576,8 +587,8 @@ void nodesDestroyNode(SNodeptr pNode) { nodesDestroyList(pLogicNode->pProjections); break; } - case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: { - SVnodeModifLogicNode* pLogicNode = (SVnodeModifLogicNode*)pNode; + case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: { + SVnodeModifyLogicNode* pLogicNode = (SVnodeModifyLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); destroyVgDataBlockArray(pLogicNode->pDataBlocks); // pVgDataBlocks is weak reference @@ -665,7 +676,7 @@ void nodesDestroyNode(SNodeptr pNode) { nodesDestroyNode(pPhyNode->pSortKeys); break; } - case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); break; @@ -1058,7 +1069,7 @@ char* nodesGetStrValueFromNode(SValueNode* pNode) { bool nodesIsExprNode(const SNode* pNode) { ENodeType type = nodeType(pNode); return (QUERY_NODE_COLUMN == type || QUERY_NODE_VALUE == type || QUERY_NODE_OPERATOR == type || - QUERY_NODE_FUNCTION == type); + QUERY_NODE_FUNCTION == type || QUERY_NODE_LOGIC_CONDITION == type); } bool nodesIsUnaryOp(const SOperatorNode* pOp) { @@ -1085,7 +1096,7 @@ bool nodesIsArithmeticOp(const SOperatorNode* pOp) { case OP_TYPE_SUB: case OP_TYPE_MULTI: case OP_TYPE_DIV: - case OP_TYPE_MOD: + case OP_TYPE_REM: return true; default: break; @@ -1301,7 +1312,7 @@ int32_t nodesCollectSpecialNodes(SSelectStmt* pSelect, ESqlClause clause, ENodeT return TSDB_CODE_SUCCESS; } -char* getFillModeString(EFillMode mode) { +char* nodesGetFillModeString(EFillMode mode) { switch (mode) { case FILL_MODE_NONE: return "none"; @@ -1349,7 +1360,7 @@ int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots) { return num; } -void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal) { +void nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal) { pVal->nType = pNode->node.resType.type; pVal->nLen = pNode->node.resType.bytes; switch (pNode->node.resType.type) { @@ -1390,3 +1401,159 @@ void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal) { break; } } + +int32_t nodesMergeConds(SNode** pDst, SNodeList** pSrc) { + if (NULL == *pSrc) { + return TSDB_CODE_SUCCESS; + } + + if (1 == LIST_LENGTH(*pSrc)) { + *pDst = nodesListGetNode(*pSrc, 0); + nodesClearList(*pSrc); + } else { + SLogicConditionNode* pLogicCond = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + if (NULL == pLogicCond) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL; + pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + pLogicCond->condType = LOGIC_COND_TYPE_AND; + pLogicCond->pParameterList = *pSrc; + *pDst = (SNode*)pLogicCond; + } + *pSrc = NULL; + + return TSDB_CODE_SUCCESS; +} + +typedef struct SClassifyConditionCxt { + bool hasPrimaryKey; + bool hasTagIndexCol; + bool hasOtherCol; +} SClassifyConditionCxt; + +static EDealRes classifyConditionImpl(SNode* pNode, void* pContext) { + SClassifyConditionCxt* pCxt = (SClassifyConditionCxt*)pContext; + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + SColumnNode* pCol = (SColumnNode*)pNode; + if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) { + pCxt->hasPrimaryKey = true; + } else if (pCol->hasIndex) { + pCxt->hasTagIndexCol = true; + } else { + pCxt->hasOtherCol = true; + } + return *((bool*)pContext) ? DEAL_RES_CONTINUE : DEAL_RES_END; + } + return DEAL_RES_CONTINUE; +} + +typedef enum EConditionType { COND_TYPE_PRIMARY_KEY = 1, COND_TYPE_TAG_INDEX, COND_TYPE_NORMAL } EConditionType; + +static EConditionType classifyCondition(SNode* pNode) { + SClassifyConditionCxt cxt = {.hasPrimaryKey = false, .hasTagIndexCol = false, .hasOtherCol = false}; + nodesWalkExpr(pNode, classifyConditionImpl, &cxt); + return cxt.hasOtherCol ? COND_TYPE_NORMAL + : (cxt.hasPrimaryKey && cxt.hasTagIndexCol + ? COND_TYPE_NORMAL + : (cxt.hasPrimaryKey ? COND_TYPE_PRIMARY_KEY : COND_TYPE_TAG_INDEX)); +} + +static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagCond, SNode** pOtherCond) { + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(*pCondition); + + int32_t code = TSDB_CODE_SUCCESS; + + SNodeList* pPrimaryKeyConds = NULL; + SNodeList* pTagConds = NULL; + SNodeList* pOtherConds = NULL; + SNode* pCond = NULL; + FOREACH(pCond, pLogicCond->pParameterList) { + switch (classifyCondition(pCond)) { + case COND_TYPE_PRIMARY_KEY: + if (NULL != pPrimaryKeyCond) { + code = nodesListMakeAppend(&pPrimaryKeyConds, nodesCloneNode(pCond)); + } + break; + case COND_TYPE_TAG_INDEX: + if (NULL != pTagCond) { + code = nodesListMakeAppend(&pTagConds, nodesCloneNode(pCond)); + } + break; + case COND_TYPE_NORMAL: + default: + if (NULL != pOtherCond) { + code = nodesListMakeAppend(&pOtherConds, nodesCloneNode(pCond)); + } + break; + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + + SNode* pTempPrimaryKeyCond = NULL; + SNode* pTempTagCond = NULL; + SNode* pTempOtherCond = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = nodesMergeConds(&pTempPrimaryKeyCond, &pPrimaryKeyConds); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesMergeConds(&pTempTagCond, &pTagConds); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesMergeConds(&pTempOtherCond, &pOtherConds); + } + + if (TSDB_CODE_SUCCESS == code) { + if (NULL != pPrimaryKeyCond) { + *pPrimaryKeyCond = pTempPrimaryKeyCond; + } + if (NULL != pTagCond) { + *pTagCond = pTempTagCond; + } + if (NULL != pOtherCond) { + *pOtherCond = pTempOtherCond; + } + nodesDestroyNode(*pCondition); + *pCondition = NULL; + } else { + nodesDestroyList(pPrimaryKeyConds); + nodesDestroyList(pTagConds); + nodesDestroyList(pOtherConds); + nodesDestroyNode(pTempPrimaryKeyCond); + nodesDestroyNode(pTempTagCond); + nodesDestroyNode(pTempOtherCond); + } + + return code; +} + +int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagCond, SNode** pOtherCond) { + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pCondition) && + LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)*pCondition)->condType) { + return partitionLogicCond(pCondition, pPrimaryKeyCond, pTagCond, pOtherCond); + } + + switch (classifyCondition(*pCondition)) { + case COND_TYPE_PRIMARY_KEY: + if (NULL != pPrimaryKeyCond) { + *pPrimaryKeyCond = *pCondition; + } + break; + case COND_TYPE_TAG_INDEX: + if (NULL != pTagCond) { + *pTagCond = *pCondition; + } + break; + case COND_TYPE_NORMAL: + default: + if (NULL != pOtherCond) { + *pOtherCond = *pCondition; + } + break; + } + *pCondition = NULL; + + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 7dd0ef2616bf3fda27192fa7099906348753c163..ca7fa7a4c671e039f4858f422737d62b8cf49342 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -193,6 +193,7 @@ SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId); SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName); SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName); SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName); +SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 8ec20cde5a07b54a5609a6097316e4ae5e538a83..3efe6700d2339b2234d33f868c0b42fa993d1b64 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -32,7 +32,6 @@ int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery); int32_t translate(SParseContext* pParseCxt, SQuery* pQuery); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); -int32_t isNotSchemalessDb(SParseContext* pContext, char *dbName); #ifdef __cplusplus } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 6c090a07901c1d96a567961b8a2ec3daabaaedf8..d63ce74fa79f90cc2c3a96df12e4c1811c64b7af 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -481,6 +481,9 @@ dnode_list(A) ::= dnode_list(B) DNODE NK_INTEGER(C). /************************************************ syncdb **************************************************************/ cmd ::= SYNCDB db_name(A) REPLICA. { pCxt->pRootNode = createSyncdbStmt(pCxt, &A); } +/************************************************ syncdb **************************************************************/ +cmd ::= DELETE FROM full_table_name(A) where_clause_opt(B). { pCxt->pRootNode = createDeleteStmt(pCxt, A, B); } + /************************************************ select **************************************************************/ cmd ::= query_expression(A). { pCxt->pRootNode = A; } @@ -608,7 +611,7 @@ expression(A) ::= expression(B) NK_SLASH expression(C). expression(A) ::= expression(B) NK_REM expression(C). { SToken s = getTokenFromRawExprNode(pCxt, B); SToken e = getTokenFromRawExprNode(pCxt, C); - A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); } expression(A) ::= column_reference(B) NK_ARROW NK_STRING(C). { SToken s = getTokenFromRawExprNode(pCxt, B); diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 72a88548d2270d6d4776e2614bf05fc8c2b7ebf6..24db0671aca56b0cbd2a215a2d67f4281c8e5ce0 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1488,3 +1488,28 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDb strncpy(pStmt->userName, pUserName->z, pUserName->n); return (SNode*)pStmt; } + +SNode* createCountFuncForDelete(SAstCreateContext* pCxt) { + SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); + CHECK_OUT_OF_MEM(pFunc); + strcpy(pFunc->functionName, "count"); + if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createPrimaryKeyCol(pCxt))) { + nodesDestroyNode(pFunc); + CHECK_OUT_OF_MEM(NULL); + } + return (SNode*)pFunc; +} + +SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere) { + CHECK_PARSER_STATUS(pCxt); + SDeleteStmt* pStmt = nodesMakeNode(QUERY_NODE_DELETE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->pFromTable = pTable; + pStmt->pWhere = pWhere; + pStmt->pCountFunc = createCountFuncForDelete(pCxt); + if (NULL == pStmt->pCountFunc) { + nodesDestroyNode(pStmt); + CHECK_OUT_OF_MEM(NULL); + } + return (SNode*)pStmt; +} diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 68c9684c97ac8eba986a339b7618e51bc02d7d79..6c44ce4c3c47660705724fe67e682140977fbb3c 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -113,17 +113,26 @@ static EDealRes collectMetaKeyFromFunction(SCollectMetaKeyFromExprCxt* pCxt, SFu return reserveUdfInCache(pFunc->functionName, pCxt->pComCxt->pMetaCache); } -static EDealRes collectMetaKeyFromRealTable(SCollectMetaKeyFromExprCxt* pCxt, SRealTableNode* pRealTable) { - pCxt->errCode = reserveTableMetaInCache(pCxt->pComCxt->pParseCxt->acctId, pRealTable->table.dbName, - pRealTable->table.tableName, pCxt->pComCxt->pMetaCache); - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pCxt->errCode = reserveTableVgroupInCache(pCxt->pComCxt->pParseCxt->acctId, pRealTable->table.dbName, - pRealTable->table.tableName, pCxt->pComCxt->pMetaCache); +static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, SRealTableNode* pRealTable, + AUTH_TYPE authType) { + int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, + pCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, + pCxt->pMetaCache); } - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pCxt->errCode = reserveUserAuthInCache(pCxt->pComCxt->pParseCxt->acctId, pCxt->pComCxt->pParseCxt->pUser, - pRealTable->table.dbName, AUTH_TYPE_READ, pCxt->pComCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pRealTable->table.dbName, authType, + pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code) { + code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pCxt->pMetaCache); + } + return code; +} + +static EDealRes collectMetaKeyFromRealTable(SCollectMetaKeyFromExprCxt* pCxt, SRealTableNode* pRealTable) { + pCxt->errCode = collectMetaKeyFromRealTableImpl(pCxt->pComCxt, pRealTable, AUTH_TYPE_READ); return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } @@ -171,11 +180,11 @@ static int32_t collectMetaKeyFromSelect(SCollectMetaKeyCxt* pCxt, SSelectStmt* p } static int32_t collectMetaKeyFromCreateTable(SCollectMetaKeyCxt* pCxt, SCreateTableStmt* pStmt) { - if (NULL == pStmt->pTags) { - return reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); - } else { - return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); + int32_t code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code && NULL == pStmt->pTags) { + code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); } + return code; } static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCreateMultiTableStmt* pStmt) { @@ -183,8 +192,27 @@ static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCre SNode* pNode = NULL; FOREACH(pNode, pStmt->pSubTables) { SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode; - code = - reserveTableMetaInCache(pCxt->pParseCxt->acctId, pClause->useDbName, pClause->useTableName, pCxt->pMetaCache); + code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pClause->dbName, pCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + code = + reserveTableMetaInCache(pCxt->pParseCxt->acctId, pClause->useDbName, pClause->useTableName, pCxt->pMetaCache); + } + if (TSDB_CODE_SUCCESS == code) { + code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache); + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; +} + +static int32_t collectMetaKeyFromDropTable(SCollectMetaKeyCxt* pCxt, SDropTableStmt* pStmt) { + int32_t code = TSDB_CODE_SUCCESS; + SNode* pNode = NULL; + FOREACH(pNode, pStmt->pTables) { + SDropTableClause* pClause = (SDropTableClause*)pNode; + code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache); if (TSDB_CODE_SUCCESS == code) { code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache); } @@ -327,6 +355,10 @@ static int32_t collectMetaKeyFromShowTransactions(SCollectMetaKeyCxt* pCxt, SSho pCxt->pMetaCache); } +static int32_t collectMetaKeyFromDelete(SCollectMetaKeyCxt* pCxt, SDeleteStmt* pStmt) { + return collectMetaKeyFromRealTableImpl(pCxt, (SRealTableNode*)pStmt->pFromTable, AUTH_TYPE_WRITE); +} + static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { switch (nodeType(pStmt)) { case QUERY_NODE_SET_OPERATOR: @@ -337,6 +369,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromCreateTable(pCxt, (SCreateTableStmt*)pStmt); case QUERY_NODE_CREATE_MULTI_TABLE_STMT: return collectMetaKeyFromCreateMultiTable(pCxt, (SCreateMultiTableStmt*)pStmt); + case QUERY_NODE_DROP_TABLE_STMT: + return collectMetaKeyFromDropTable(pCxt, (SDropTableStmt*)pStmt); case QUERY_NODE_ALTER_TABLE_STMT: return collectMetaKeyFromAlterTable(pCxt, (SAlterTableStmt*)pStmt); case QUERY_NODE_USE_DATABASE_STMT: @@ -383,6 +417,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromShowTopics(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_TRANSACTIONS_STMT: return collectMetaKeyFromShowTransactions(pCxt, (SShowStmt*)pStmt); + case QUERY_NODE_DELETE_STMT: + return collectMetaKeyFromDelete(pCxt, (SDeleteStmt*)pStmt); default: break; } diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 2670e5710b9f5418c401e9799678c68d82c8f29d..2ee23dedf2d9ebf6041347b7b2631748cf737348 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -80,6 +80,10 @@ static int32_t authDropUser(SAuthCxt* pCxt, SDropUserStmt* pStmt) { return TSDB_CODE_SUCCESS; } +static int32_t authDelete(SAuthCxt* pCxt, SDeleteStmt* pDelete) { + return checkAuth(pCxt, ((SRealTableNode*)pDelete->pFromTable)->table.dbName, AUTH_TYPE_WRITE); +} + static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { switch (nodeType(pStmt)) { case QUERY_NODE_SET_OPERATOR: @@ -88,6 +92,8 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { return authSelect(pCxt, (SSelectStmt*)pStmt); case QUERY_NODE_DROP_USER_STMT: return authDropUser(pCxt, (SDropUserStmt*)pStmt); + case QUERY_NODE_DELETE_STMT: + return authDelete(pCxt, (SDeleteStmt*)pStmt); default: break; } diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 42b001c1318058be96871918bea5aee0f084c82a..b2cb67b1c3fc13a66922f51f250c2e39621798b3 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -135,11 +135,11 @@ static int32_t rewriteConditionForFromTable(SCalcConstContext* pCxt, SNode* pTab return code; } -static int32_t calcConstFromTable(SCalcConstContext* pCxt, SSelectStmt* pSelect) { - return rewriteConditionForFromTable(pCxt, pSelect->pFromTable); +static int32_t calcConstFromTable(SCalcConstContext* pCxt, SNode* pTable) { + return rewriteConditionForFromTable(pCxt, pTable); } -static void rewriteConstCondition(SSelectStmt* pSelect, SNode** pCond) { +static void rewriteConstCondition(SNode** pCond, bool* pAlwaysFalse) { if (QUERY_NODE_VALUE != nodeType(*pCond)) { return; } @@ -147,11 +147,11 @@ static void rewriteConstCondition(SSelectStmt* pSelect, SNode** pCond) { nodesDestroyNode(*pCond); *pCond = NULL; } else { - pSelect->isEmptyResult = true; + *pAlwaysFalse = true; } } -static int32_t calcConstSelectCondition(SCalcConstContext* pCxt, SSelectStmt* pSelect, SNode** pCond) { +static int32_t calcConstStmtCondition(SCalcConstContext* pCxt, SNode** pCond, bool* pAlwaysFalse) { if (NULL == *pCond) { return TSDB_CODE_SUCCESS; } @@ -161,7 +161,7 @@ static int32_t calcConstSelectCondition(SCalcConstContext* pCxt, SSelectStmt* pS code = calcConstNode(pCond); } if (TSDB_CODE_SUCCESS == code) { - rewriteConstCondition(pSelect, pCond); + rewriteConstCondition(pCond, pAlwaysFalse); } return code; } @@ -233,12 +233,12 @@ static int32_t calcConstGroupBy(SCalcConstContext* pCxt, SSelectStmt* pSelect) { } static int32_t calcConstSelect(SCalcConstContext* pCxt, SSelectStmt* pSelect, bool subquery) { - int32_t code = calcConstFromTable(pCxt, pSelect); + int32_t code = calcConstFromTable(pCxt, pSelect->pFromTable); if (TSDB_CODE_SUCCESS == code) { code = calcConstProjections(pCxt, pSelect, subquery); } if (TSDB_CODE_SUCCESS == code) { - code = calcConstSelectCondition(pCxt, pSelect, &pSelect->pWhere); + code = calcConstStmtCondition(pCxt, &pSelect->pWhere, &pSelect->isEmptyResult); } if (TSDB_CODE_SUCCESS == code) { code = calcConstList(pSelect->pPartitionByList); @@ -250,7 +250,7 @@ static int32_t calcConstSelect(SCalcConstContext* pCxt, SSelectStmt* pSelect, bo code = calcConstGroupBy(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { - code = calcConstSelectCondition(pCxt, pSelect, &pSelect->pHaving); + code = calcConstStmtCondition(pCxt, &pSelect->pHaving, &pSelect->isEmptyResult); } if (TSDB_CODE_SUCCESS == code) { code = calcConstList(pSelect->pOrderByList); @@ -258,6 +258,14 @@ static int32_t calcConstSelect(SCalcConstContext* pCxt, SSelectStmt* pSelect, bo return code; } +static int32_t calcConstDelete(SCalcConstContext* pCxt, SDeleteStmt* pDelete) { + int32_t code = calcConstFromTable(pCxt, pDelete->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + code = calcConstStmtCondition(pCxt, &pDelete->pWhere, &pDelete->deleteZeroRows); + } + return code; +} + static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt, bool subquery) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pStmt)) { @@ -275,6 +283,9 @@ static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt, bool subque } break; } + case QUERY_NODE_DELETE_STMT: + code = calcConstDelete(pCxt, (SDeleteStmt*)pStmt); + break; default: break; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 422c48039743185070ca5ea5f704627d26217253..efe0bd75b2fe66efc0d84932e877f873c4ae0319 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -254,7 +254,7 @@ static int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, con static int32_t checkAuth(SInsertParseContext* pCxt, char* pDbFname, bool* pPass) { SParseContext* pBasicCtx = pCxt->pComCxt; - if (NULL != pCxt->pMetaCache) { + if (pBasicCtx->async) { return getUserAuthFromCache(pCxt->pMetaCache, pBasicCtx->pUser, pDbFname, AUTH_TYPE_WRITE, pPass); } return catalogChkAuth(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pBasicCtx->pUser, pDbFname, @@ -263,7 +263,7 @@ static int32_t checkAuth(SInsertParseContext* pCxt, char* pDbFname, bool* pPass) static int32_t getTableSchema(SInsertParseContext* pCxt, SName* pTbName, bool isStb, STableMeta** pTableMeta) { SParseContext* pBasicCtx = pCxt->pComCxt; - if (NULL != pCxt->pMetaCache) { + if (pBasicCtx->async) { return getTableMetaFromCache(pCxt->pMetaCache, pTbName, pTableMeta); } if (isStb) { @@ -275,7 +275,7 @@ static int32_t getTableSchema(SInsertParseContext* pCxt, SName* pTbName, bool is static int32_t getTableVgroup(SInsertParseContext* pCxt, SName* pTbName, SVgroupInfo* pVg) { SParseContext* pBasicCtx = pCxt->pComCxt; - if (NULL != pCxt->pMetaCache) { + if (pBasicCtx->async) { return getTableVgroupFromCache(pCxt->pMetaCache, pTbName, pVg); } return catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pTbName, pVg); @@ -305,6 +305,16 @@ static int32_t getSTableMeta(SInsertParseContext* pCxt, SName* name, char* dbFna return getTableMetaImpl(pCxt, name, dbFname, true); } +static int32_t getDBCfg(SInsertParseContext* pCxt, const char* pDbFName, SDbCfgInfo* pInfo) { + SParseContext* pBasicCtx = pCxt->pComCxt; + if (pBasicCtx->async) { + CHECK_CODE(getDbCfgFromCache(pCxt->pMetaCache, pDbFName, pInfo)); + } else { + CHECK_CODE(catalogGetDBCfg(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pDbFName, pInfo)); + } + return TSDB_CODE_SUCCESS; +} + static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) { while (start < end) { if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { @@ -1090,10 +1100,10 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SName* name, char* tb SName sname; createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); - char stbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(&sname, stbFName); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(&sname, dbFName); - CHECK_CODE(getSTableMeta(pCxt, &sname, stbFName)); + CHECK_CODE(getSTableMeta(pCxt, &sname, dbFName)); if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); } @@ -1282,6 +1292,14 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { destroyBlockArrayList(pCxt->pVgDataBlocks); } +static int32_t checkSchemalessDb(SInsertParseContext* pCxt, char* pDbName) { + SDbCfgInfo pInfo = {0}; + char fullName[TSDB_TABLE_FNAME_LEN]; + snprintf(fullName, sizeof(fullName), "%d.%s", pCxt->pComCxt->acctId, pDbName); + CHECK_CODE(getDBCfg(pCxt, fullName, &pInfo)); + return pInfo.schemaless ? TSDB_CODE_SML_INVALID_DB_CONF : TSDB_CODE_SUCCESS; +} + // tb_name // [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] // [(field1_name, ...)] @@ -1335,7 +1353,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { SName name; CHECK_CODE(createSName(&name, &tbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); - CHECK_CODE(isNotSchemalessDb(pCxt->pComCxt, name.dbname)); + CHECK_CODE(checkSchemalessDb(pCxt, name.dbname)); tNameExtractFullName(&name, tbFName); CHECK_CODE(taosHashPut(pCxt->pTableNameHashObj, tbFName, strlen(tbFName), &name, sizeof(SName))); @@ -1413,23 +1431,6 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildOutput(pCxt); } -int32_t isNotSchemalessDb(SParseContext* pContext, char *dbName){ - SName name; - tNameSetDbName(&name, pContext->acctId, dbName, strlen(dbName)); - char dbFname[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&name, dbFname); - SDbCfgInfo pInfo = {0}; - int32_t code = catalogGetDBCfg(pContext->pCatalog, pContext->pTransporter, &pContext->mgmtEpSet, dbFname, &pInfo); - if (code != TSDB_CODE_SUCCESS) { - parserError("catalogGetDBCfg error, code:%s, dbFName:%s", tstrerror(code), dbFname); - return code; - } - if (pInfo.schemaless){ - parserError("can not insert into schemaless db:%s", dbFname); - return TSDB_CODE_SML_INVALID_DB_CONF; - } - return TSDB_CODE_SUCCESS; -} // INSERT INTO // tb_name // [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] @@ -1472,6 +1473,8 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { if (NULL == *pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } + } else { + context.pMetaCache = (*pQuery)->pMetaCache; } (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE; (*pQuery)->haveResultSet = false; @@ -1586,12 +1589,20 @@ static int32_t skipUsingClause(SInsertParseSyntaxCxt* pCxt) { static int32_t collectTableMetaKey(SInsertParseSyntaxCxt* pCxt, SToken* pTbToken) { SName name; CHECK_CODE(createSName(&name, pTbToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); + CHECK_CODE(reserveDbCfgInCache(pCxt->pComCxt->acctId, name.dbname, pCxt->pMetaCache)); CHECK_CODE(reserveUserAuthInCacheExt(pCxt->pComCxt->pUser, &name, AUTH_TYPE_WRITE, pCxt->pMetaCache)); CHECK_CODE(reserveTableMetaInCacheExt(&name, pCxt->pMetaCache)); CHECK_CODE(reserveTableVgroupInCacheExt(&name, pCxt->pMetaCache)); return TSDB_CODE_SUCCESS; } +static int32_t collectAutoCreateTableMetaKey(SInsertParseSyntaxCxt* pCxt, SToken* pTbToken) { + SName name; + CHECK_CODE(createSName(&name, pTbToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); + CHECK_CODE(reserveTableVgroupInCacheExt(&name, pCxt->pMetaCache)); + return TSDB_CODE_SUCCESS; +} + static int32_t parseInsertBodySyntax(SInsertParseSyntaxCxt* pCxt) { bool hasData = false; // for each table @@ -1620,6 +1631,7 @@ static int32_t parseInsertBodySyntax(SInsertParseSyntaxCxt* pCxt) { // USING clause if (TK_USING == sToken.type) { + CHECK_CODE(collectAutoCreateTableMetaKey(pCxt, &tbnameToken)); NEXT_TOKEN(pCxt->pSql, sToken); CHECK_CODE(collectTableMetaKey(pCxt, &sToken)); CHECK_CODE(skipUsingClause(pCxt)); @@ -2172,8 +2184,8 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p val.nData = kv->length; } else if (pTagSchema->type == TSDB_DATA_TYPE_NCHAR) { int32_t output = 0; - void *p = taosMemoryCalloc(1, kv->length * TSDB_NCHAR_SIZE); - if(p == NULL){ + void* p = taosMemoryCalloc(1, kv->length * TSDB_NCHAR_SIZE); + if (p == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto end; } diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index e9539073583c6d21a100efa5b33516eb9db18393..685c299c01af44d479d4e03cf42711cab30bfc73 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -68,6 +68,7 @@ static SKeyword keywordTable[] = { {"DATABASES", TK_DATABASES}, {"DAYS", TK_DAYS}, {"DBS", TK_DBS}, + {"DELETE", TK_DELETE}, {"DESC", TK_DESC}, {"DESCRIBE", TK_DESCRIBE}, {"DISTINCT", TK_DISTINCT}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f0cba6ddc9031387db7528fc1f47706f23ee2863..f1892d0040293c6596dbb5862d3245fa370f53af 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -35,7 +35,7 @@ typedef struct STranslateContext { SArray* pNsLevel; // element is SArray*, the element of this subarray is STableNode* int32_t currLevel; ESqlClause currClause; - SSelectStmt* pCurrStmt; + SSelectStmt* pCurrSelectStmt; SCmdMsgInfo* pCmdMsg; SHashObj* pDbs; SHashObj* pTables; @@ -335,7 +335,7 @@ static bool isIndefiniteRowsFunc(const SNode* pNode) { } static bool isDistinctOrderBy(STranslateContext* pCxt) { - return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrStmt->isDistinct); + return (SQL_CLAUSE_ORDER_BY == pCxt->currClause && pCxt->pCurrSelectStmt->isDistinct); } static bool belongTable(const char* currentDb, const SColumnNode* pCol, const STableNode* pTable) { @@ -360,7 +360,7 @@ static SNodeList* getProjectList(const SNode* pNode) { return NULL; } -static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* pColSchema, bool isTag, +static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* pColSchema, int32_t tagFlag, SColumnNode* pCol) { strcpy(pCol->dbName, pTable->table.dbName); strcpy(pCol->tableAlias, pTable->table.tableAlias); @@ -372,7 +372,8 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p pCol->tableId = pTable->pMeta->uid; pCol->tableType = pTable->pMeta->tableType; pCol->colId = pColSchema->colId; - pCol->colType = isTag ? COLUMN_TYPE_TAG : COLUMN_TYPE_COLUMN; + pCol->colType = (tagFlag >= 0 ? COLUMN_TYPE_TAG : COLUMN_TYPE_COLUMN); + pCol->hasIndex = (0 == tagFlag); pCol->node.resType.type = pColSchema->type; pCol->node.resType.bytes = pColSchema->bytes; if (TSDB_DATA_TYPE_TIMESTAMP == pCol->node.resType.type) { @@ -414,7 +415,7 @@ static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* p if (NULL == pCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY); } - setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i >= pMeta->tableInfo.numOfColumns), pCol); + setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i - pMeta->tableInfo.numOfColumns), pCol); nodesListAppend(pList, (SNode*)pCol); } } else { @@ -472,14 +473,14 @@ static int32_t findAndSetColumn(STranslateContext* pCxt, SColumnNode** pColRef, if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; if (isInternalPrimaryKey(pCol)) { - setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema, false, pCol); + setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema, -1, pCol); *pFound = true; return TSDB_CODE_SUCCESS; } int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; for (int32_t i = 0; i < nums; ++i) { if (0 == strcmp(pCol->colName, pMeta->schema[i].name)) { - setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i >= pMeta->tableInfo.numOfColumns), pCol); + setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i - pMeta->tableInfo.numOfColumns), pCol); *pFound = true; break; } @@ -551,7 +552,7 @@ static EDealRes translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNod } if (!found) { if (isInternalPk) { - if (NULL != pCxt->pCurrStmt->pWindow) { + if (NULL != pCxt->pCurrSelectStmt && NULL != pCxt->pCurrSelectStmt->pWindow) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY); } return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_INTERNAL_PK); @@ -563,7 +564,7 @@ static EDealRes translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNod } static bool translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** pCol) { - SNodeList* pProjectionList = pCxt->pCurrStmt->pProjectionList; + SNodeList* pProjectionList = pCxt->pCurrSelectStmt->pProjectionList; SNode* pNode; FOREACH(pNode, pProjectionList) { SExprNode* pExpr = (SExprNode*)pNode; @@ -621,7 +622,7 @@ static int32_t parseTimeFromValueNode(STranslateContext* pCxt, SValueNode* pVal) } static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SDataType targetDt) { - uint8_t precision = (NULL != pCxt->pCurrStmt ? pCxt->pCurrStmt->precision : targetDt.precision); + uint8_t precision = (NULL != pCxt->pCurrSelectStmt ? pCxt->pCurrSelectStmt->precision : targetDt.precision); pVal->node.resType.precision = precision; if (pVal->placeholderNo > 0) { return DEAL_RES_CONTINUE; @@ -829,8 +830,9 @@ static EDealRes translateComparisonOperator(STranslateContext* pCxt, SOperatorNo if (!IS_VAR_DATA_TYPE(((SExprNode*)(pOp->pLeft))->resType.type)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pLeft))->aliasName); } - if (QUERY_NODE_VALUE != nodeType(pOp->pRight) || - ((!IS_STR_DATA_TYPE(((SExprNode*)(pOp->pRight))->resType.type)) && (((SExprNode*)(pOp->pRight))->resType.type != TSDB_DATA_TYPE_NULL))) { + if (QUERY_NODE_VALUE != nodeType(pOp->pRight) || + ((!IS_STR_DATA_TYPE(((SExprNode*)(pOp->pRight))->resType.type)) && + (((SExprNode*)(pOp->pRight))->resType.type != TSDB_DATA_TYPE_NULL))) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } } @@ -910,7 +912,7 @@ static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) STableNode* pTable = NULL; int32_t code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable); if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { - setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol); } return code; } @@ -936,7 +938,7 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (hasInvalidFuncNesting(pFunc->pParameterList)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AGG_FUNC_NESTING); } - if (pCxt->pCurrStmt->hasIndefiniteRowsFunc) { + if (NULL != pCxt->pCurrSelectStmt && pCxt->pCurrSelectStmt->hasIndefiniteRowsFunc) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } @@ -948,7 +950,7 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (0 == LIST_LENGTH(pFunc->pParameterList)) { - if (QUERY_NODE_REAL_TABLE != nodeType(pCxt->pCurrStmt->pFromTable)) { + if (QUERY_NODE_REAL_TABLE != nodeType(pCxt->pCurrSelectStmt->pFromTable)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TBNAME); } } else { @@ -963,7 +965,8 @@ static int32_t translateScanPseudoColumnFunc(STranslateContext* pCxt, SFunctionN } static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { - if (SQL_CLAUSE_SELECT != pCxt->currClause || pCxt->pCurrStmt->hasIndefiniteRowsFunc || pCxt->pCurrStmt->hasAggFuncs) { + if (SQL_CLAUSE_SELECT != pCxt->currClause || pCxt->pCurrSelectStmt->hasIndefiniteRowsFunc || + pCxt->pCurrSelectStmt->hasAggFuncs) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } if (hasInvalidFuncNesting(pFunc->pParameterList)) { @@ -973,9 +976,11 @@ static int32_t translateIndefiniteRowsFunc(STranslateContext* pCxt, SFunctionNod } static void setFuncClassification(SSelectStmt* pSelect, SFunctionNode* pFunc) { - pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); - pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId); - pSelect->hasIndefiniteRowsFunc = pSelect->hasIndefiniteRowsFunc ? true : fmIsIndefiniteRowsFunc(pFunc->funcId); + if (NULL != pSelect) { + pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); + pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId); + pSelect->hasIndefiniteRowsFunc = pSelect->hasIndefiniteRowsFunc ? true : fmIsIndefiniteRowsFunc(pFunc->funcId); + } } static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { @@ -997,7 +1002,7 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) pCxt->errCode = translateIndefiniteRowsFunc(pCxt, pFunc); } if (TSDB_CODE_SUCCESS == pCxt->errCode) { - setFuncClassification(pCxt->pCurrStmt, pFunc); + setFuncClassification(pCxt->pCurrSelectStmt, pFunc); } return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } @@ -1045,9 +1050,9 @@ static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) { static SNodeList* getGroupByList(STranslateContext* pCxt) { if (isDistinctOrderBy(pCxt)) { - return pCxt->pCurrStmt->pProjectionList; + return pCxt->pCurrSelectStmt->pProjectionList; } - return pCxt->pCurrStmt->pGroupByList; + return pCxt->pCurrSelectStmt->pGroupByList; } static SNode* getGroupByNode(SNode* pNode) { @@ -1085,7 +1090,7 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode } if (TSDB_CODE_SUCCESS == pCxt->errCode) { *pNode = (SNode*)pFunc; - pCxt->pCurrStmt->hasSelectValFunc = true; + pCxt->pCurrSelectStmt->hasSelectValFunc = true; } else { nodesDestroyNode(pFunc); } @@ -1308,6 +1313,8 @@ static int32_t setTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTa static uint8_t getStmtPrecision(SNode* pStmt) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { return ((SSelectStmt*)pStmt)->precision; + } else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) { + return ((SSetOperator*)pStmt)->precision; } return 0; } @@ -1319,10 +1326,10 @@ static bool stmtIsSingleTable(SNode* pStmt) { return false; } -static uint8_t getJoinTablePrecision(SJoinTableNode* pJoinTable) { - uint8_t lp = ((STableNode*)pJoinTable->pLeft)->precision; - uint8_t rp = ((STableNode*)pJoinTable->pRight)->precision; - return (lp > rp ? rp : lp); +static uint8_t calcPrecision(uint8_t lp, uint8_t rp) { return (lp > rp ? rp : lp); } + +static uint8_t calcJoinTablePrecision(SJoinTableNode* pJoinTable) { + return calcPrecision(((STableNode*)pJoinTable->pLeft)->precision, ((STableNode*)pJoinTable->pRight)->precision); } static bool joinTableIsSingleTable(SJoinTableNode* pJoinTable) { @@ -1378,7 +1385,7 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { code = translateTable(pCxt, pJoinTable->pRight); } if (TSDB_CODE_SUCCESS == code) { - pJoinTable->table.precision = getJoinTablePrecision(pJoinTable); + pJoinTable->table.precision = calcJoinTablePrecision(pJoinTable); pJoinTable->table.singleTable = joinTableIsSingleTable(pJoinTable); code = translateExpr(pCxt, &pJoinTable->pOnCond); } @@ -1678,71 +1685,40 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } -static EDealRes isPrimaryKeyCondImpl(SNode* pNode, void* pContext) { - if (QUERY_NODE_COLUMN == nodeType(pNode)) { - *((bool*)pContext) = ((PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) ? true : false); - return *((bool*)pContext) ? DEAL_RES_CONTINUE : DEAL_RES_END; - } - return DEAL_RES_CONTINUE; -} - -static bool isPrimaryKeyCond(SNode* pNode) { - bool isPrimaryKeyCond = false; - nodesWalkExpr(pNode, isPrimaryKeyCondImpl, &isPrimaryKeyCond); - return isPrimaryKeyCond; -} - -static int32_t getTimeRangeFromLogicCond(STranslateContext* pCxt, SLogicConditionNode* pLogicCond, - STimeWindow* pTimeRange) { - SNodeList* pPrimaryKeyConds = NULL; - SNode* pCond = NULL; - FOREACH(pCond, pLogicCond->pParameterList) { - if (isPrimaryKeyCond(pCond)) { - if (TSDB_CODE_SUCCESS != nodesListMakeAppend(&pPrimaryKeyConds, pCond)) { - nodesClearList(pPrimaryKeyConds); - return TSDB_CODE_OUT_OF_MEMORY; - } - } - } - - if (NULL == pPrimaryKeyConds) { - *pTimeRange = TSWINDOW_INITIALIZER; - return TSDB_CODE_SUCCESS; - } - - SLogicConditionNode* pPrimaryKeyLogicCond = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); - if (NULL == pPrimaryKeyLogicCond) { - nodesClearList(pPrimaryKeyConds); - return TSDB_CODE_OUT_OF_MEMORY; +static int32_t getTimeRange(SNode** pPrimaryKeyCond, STimeWindow* pTimeRange, bool* pIsStrict) { + SNode* pNew = NULL; + int32_t code = scalarCalculateConstants(*pPrimaryKeyCond, &pNew); + if (TSDB_CODE_SUCCESS == code) { + *pPrimaryKeyCond = pNew; + code = filterGetTimeRange(*pPrimaryKeyCond, pTimeRange, pIsStrict); } - pPrimaryKeyLogicCond->condType = LOGIC_COND_TYPE_AND; - pPrimaryKeyLogicCond->pParameterList = pPrimaryKeyConds; - bool isStrict = false; - int32_t code = filterGetTimeRange((SNode*)pPrimaryKeyLogicCond, pTimeRange, &isStrict); - nodesClearList(pPrimaryKeyConds); - pPrimaryKeyLogicCond->pParameterList = NULL; - nodesDestroyNode(pPrimaryKeyLogicCond); return code; } -static int32_t getTimeRange(STranslateContext* pCxt, SNode* pWhere, STimeWindow* pTimeRange) { +static int32_t getFillTimeRange(STranslateContext* pCxt, SNode* pWhere, STimeWindow* pTimeRange) { if (NULL == pWhere) { *pTimeRange = TSWINDOW_INITIALIZER; return TSDB_CODE_SUCCESS; } - if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere) && - LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)pWhere)->condType) { - return getTimeRangeFromLogicCond(pCxt, (SLogicConditionNode*)pWhere, pTimeRange); + SNode* pCond = nodesCloneNode(pWhere); + if (NULL == pCond) { + return TSDB_CODE_OUT_OF_MEMORY; } - if (isPrimaryKeyCond(pWhere)) { + SNode* pPrimaryKeyCond = NULL; + nodesPartitionCond(&pCond, &pPrimaryKeyCond, NULL, NULL); + + int32_t code = TSDB_CODE_SUCCESS; + if (NULL != pPrimaryKeyCond) { bool isStrict = false; - return filterGetTimeRange(pWhere, pTimeRange, &isStrict); + code = getTimeRange(&pPrimaryKeyCond, pTimeRange, &isStrict); } else { *pTimeRange = TSWINDOW_INITIALIZER; } - return TSDB_CODE_SUCCESS; + nodesDestroyNode(pCond); + nodesDestroyNode(pPrimaryKeyCond); + return code; } static int32_t checkFill(STranslateContext* pCxt, SIntervalWindowNode* pInterval) { @@ -1778,7 +1754,7 @@ static int32_t translateFill(STranslateContext* pCxt, SNode* pWhere, SIntervalWi return TSDB_CODE_SUCCESS; } - int32_t code = getTimeRange(pCxt, pWhere, &(((SFillNode*)pInterval->pFill)->timeRange)); + int32_t code = getFillTimeRange(pCxt, pWhere, &(((SFillNode*)pInterval->pFill)->timeRange)); if (TSDB_CODE_SUCCESS == code) { code = checkFill(pCxt, pInterval); } @@ -1924,13 +1900,9 @@ static int32_t translateWhere(STranslateContext* pCxt, SNode** pWhere) { return translateExpr(pCxt, pWhere); } -static int32_t translateFrom(STranslateContext* pCxt, SSelectStmt* pSelect) { +static int32_t translateFrom(STranslateContext* pCxt, SNode* pTable) { pCxt->currClause = SQL_CLAUSE_FROM; - int32_t code = translateTable(pCxt, pSelect->pFromTable); - if (TSDB_CODE_SUCCESS == code) { - pSelect->precision = ((STableNode*)pSelect->pFromTable)->precision; - } - return code; + return translateTable(pCxt, pTable); } static int32_t checkLimit(STranslateContext* pCxt, SSelectStmt* pSelect) { @@ -1991,9 +1963,10 @@ static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SSelectStmt* pSelect } static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { - pCxt->pCurrStmt = pSelect; - int32_t code = translateFrom(pCxt, pSelect); + pCxt->pCurrSelectStmt = pSelect; + int32_t code = translateFrom(pCxt, pSelect->pFromTable); if (TSDB_CODE_SUCCESS == code) { + pSelect->precision = ((STableNode*)pSelect->pFromTable)->precision; code = translateWhere(pCxt, &pSelect->pWhere); } if (TSDB_CODE_SUCCESS == code) { @@ -2094,6 +2067,10 @@ static int32_t translateSetOperatorImpl(STranslateContext* pCxt, SSetOperator* p return TSDB_CODE_SUCCESS; } +static uint8_t calcSetOperatorPrecision(SSetOperator* pSetOperator) { + return calcPrecision(getStmtPrecision(pSetOperator->pLeft), getStmtPrecision(pSetOperator->pRight)); +} + static int32_t translateSetOperator(STranslateContext* pCxt, SSetOperator* pSetOperator) { int32_t code = translateQuery(pCxt, pSetOperator->pLeft); if (TSDB_CODE_SUCCESS == code) { @@ -2103,11 +2080,56 @@ static int32_t translateSetOperator(STranslateContext* pCxt, SSetOperator* pSetO code = translateQuery(pCxt, pSetOperator->pRight); } if (TSDB_CODE_SUCCESS == code) { + pSetOperator->precision = calcSetOperatorPrecision(pSetOperator); code = translateSetOperatorImpl(pCxt, pSetOperator); } return code; } +static int32_t partitionDeleteWhere(STranslateContext* pCxt, SDeleteStmt* pDelete) { + if (NULL == pDelete->pWhere) { + pDelete->timeRange = TSWINDOW_INITIALIZER; + return TSDB_CODE_SUCCESS; + } + + SNode* pPrimaryKeyCond = NULL; + SNode* pOtherCond = NULL; + int32_t code = nodesPartitionCond(&pDelete->pWhere, &pPrimaryKeyCond, &pDelete->pTagIndexCond, &pOtherCond); + if (TSDB_CODE_SUCCESS == code && NULL != pOtherCond) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DELETE_WHERE); + } + if (TSDB_CODE_SUCCESS == code) { + bool isStrict = false; + code = getTimeRange(&pPrimaryKeyCond, &pDelete->timeRange, &isStrict); + if (TSDB_CODE_SUCCESS == code && !isStrict) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DELETE_WHERE); + } + } + nodesDestroyNode(pPrimaryKeyCond); + nodesDestroyNode(pOtherCond); + return code; +} + +static int32_t translateDeleteWhere(STranslateContext* pCxt, SDeleteStmt* pDelete) { + int32_t code = translateWhere(pCxt, &pDelete->pWhere); + if (TSDB_CODE_SUCCESS == code) { + code = partitionDeleteWhere(pCxt, pDelete); + } + return code; +} + +static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) { + int32_t code = translateFrom(pCxt, pDelete->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + code = translateDeleteWhere(pCxt, pDelete); + } + if (TSDB_CODE_SUCCESS == code) { + pCxt->currClause = SQL_CLAUSE_SELECT; + code = translateExpr(pCxt, &pDelete->pCountFunc); + } + return code; +} + static int64_t getUnitPerMinute(uint8_t precision) { switch (precision) { case TSDB_TIME_PRECISION_MILLI: @@ -2632,8 +2654,23 @@ static int32_t checkTableSchema(STranslateContext* pCxt, SCreateTableStmt* pStmt return code; } +static int32_t checkSchemalessDb(STranslateContext* pCxt, const char* pDbName) { + if (0 != pCxt->pParseCxt->schemalessType) { + return TSDB_CODE_SUCCESS; + } + SDbCfgInfo info = {0}; + int32_t code = getDBCfg(pCxt, pDbName, &info); + if (TSDB_CODE_SUCCESS == code) { + code = info.schemaless ? TSDB_CODE_SML_INVALID_DB_CONF : TSDB_CODE_SUCCESS; + } + return code; +} + static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { - int32_t code = checTableFactorOption(pCxt, pStmt->pOptions->filesFactor); + int32_t code = checkSchemalessDb(pCxt, pStmt->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = checTableFactorOption(pCxt, pStmt->pOptions->filesFactor); + } if (TSDB_CODE_SUCCESS == code) { code = checkTableRollupOption(pCxt, pStmt->pOptions->pRollupFuncs); } @@ -2646,11 +2683,6 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt if (TSDB_CODE_SUCCESS == code) { code = checkTableSchema(pCxt, pStmt); } - if (TSDB_CODE_SUCCESS == code) { - if(pCxt->pParseCxt->schemalessType == 0){ - code = isNotSchemalessDb(pCxt->pParseCxt, pStmt->dbName); - } - } return code; } @@ -2857,7 +2889,6 @@ static int32_t buildRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt, int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg); int32_t num = taosArrayGetSize(dbCfg.pRetensions); if (TSDB_CODE_SUCCESS != code || num < 2) { - taosArrayDestroy(dbCfg.pRetensions); return code; } for (int32_t i = 1; i < num; ++i) { @@ -2924,25 +2955,9 @@ static int32_t doTranslateDropSuperTable(STranslateContext* pCxt, const SName* p static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt) { SDropTableClause* pClause = nodesListGetNode(pStmt->pTables, 0); - - STableMeta* pTableMeta = NULL; - SName tableName; - int32_t code = getTableMetaImpl( - pCxt, toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName), &pTableMeta); - if ((TSDB_CODE_PAR_TABLE_NOT_EXIST == code || TSDB_CODE_VND_TB_NOT_EXIST == code) && pClause->ignoreNotExists) { - return TSDB_CODE_SUCCESS; - } - if (TSDB_CODE_SUCCESS == code) { - if (TSDB_SUPER_TABLE == pTableMeta->tableType) { - code = doTranslateDropSuperTable(pCxt, &tableName, pClause->ignoreNotExists); - } else { - // todo : drop normal table or child table - code = TSDB_CODE_FAILED; - } - taosMemoryFreeClear(pTableMeta); - } - - return code; + SName tableName; + return doTranslateDropSuperTable( + pCxt, toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &tableName), pClause->ignoreNotExists); } static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableStmt* pStmt) { @@ -3450,7 +3465,6 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt* SName name; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->streamName, strlen(pStmt->streamName)); tNameGetFullDbName(&name, pReq->name); - // tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->streamName, &name), pReq->name); if ('\0' != pStmt->targetTabName[0]) { strcpy(name.dbname, pStmt->targetDbName); @@ -3603,6 +3617,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_SET_OPERATOR: code = translateSetOperator(pCxt, (SSetOperator*)pNode); break; + case QUERY_NODE_DELETE_STMT: + code = translateDelete(pCxt, (SDeleteStmt*)pNode); + break; case QUERY_NODE_CREATE_DATABASE_STMT: code = translateCreateDatabase(pCxt, (SCreateDatabaseStmt*)pNode); break; @@ -3722,11 +3739,11 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { ++(pCxt->currLevel); ESqlClause currClause = pCxt->currClause; - SSelectStmt* pCurrStmt = pCxt->pCurrStmt; + SSelectStmt* pCurrStmt = pCxt->pCurrSelectStmt; int32_t code = translateQuery(pCxt, pNode); --(pCxt->currLevel); pCxt->currClause = currClause; - pCxt->pCurrStmt = pCurrStmt; + pCxt->pCurrSelectStmt = pCurrStmt; return code; } @@ -4239,7 +4256,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla int16_t nTags = 0, nBufPos = 0; SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta); SNode * pTag = NULL, *pNode = NULL; - bool isJson = false; + bool isJson = false; FORBOTH(pTag, pStmt->pSpecificTags, pNode, pStmt->pValsOfTags) { SColumnNode* pCol = (SColumnNode*)pTag; SSchema* pSchema = NULL; @@ -4272,11 +4289,11 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla isJson = true; code = parseJsontoTagData(pVal->literal, pTagArray, ppTag, &pCxt->msgBuf); - if(code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { goto end; } - }else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { - void* nodeVal = nodesGetValueFromNode(pVal); + } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { + void* nodeVal = nodesGetValueFromNode(pVal); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; if (IS_VAR_DATA_TYPE(pTagSchema->type)) { val.pData = varDataVal(nodeVal); @@ -4288,13 +4305,13 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } } - if(!isJson) code = tTagNew(pTagArray, 1, false, ppTag); + if (!isJson) code = tTagNew(pTagArray, 1, false, ppTag); end: - if(isJson){ + if (isJson) { for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { - STagVal *p = (STagVal *)taosArrayGet(pTagArray, i); - if(IS_VAR_DATA_TYPE(p->type)){ + STagVal* p = (STagVal*)taosArrayGet(pTagArray, i); + if (IS_VAR_DATA_TYPE(p->type)) { taosMemoryFree(p->pData); } } @@ -4339,11 +4356,11 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau isJson = true; code = parseJsontoTagData(pVal->literal, pTagArray, ppTag, &pCxt->msgBuf); - if(code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { goto end; } - }else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { - char* tmpVal = nodesGetValueFromNode(pVal); + } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { + char* tmpVal = nodesGetValueFromNode(pVal); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; if (IS_VAR_DATA_TYPE(pTagSchema->type)) { val.pData = varDataVal(tmpVal); @@ -4355,13 +4372,13 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau } ++index; } - if(!isJson) code = tTagNew(pTagArray, 1, false, ppTag); + if (!isJson) code = tTagNew(pTagArray, 1, false, ppTag); end: - if(isJson){ + if (isJson) { for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { - STagVal *p = (STagVal *)taosArrayGet(pTagArray, i); - if(IS_VAR_DATA_TYPE(p->type)){ + STagVal* p = (STagVal*)taosArrayGet(pTagArray, i); + if (IS_VAR_DATA_TYPE(p->type)) { taosMemoryFree(p->pData); } } @@ -4429,7 +4446,6 @@ static SArray* serializeVgroupsCreateTableBatch(int32_t acctId, SHashObj* pVgrou } static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) { - SCreateMultiTableStmt* pStmt = (SCreateMultiTableStmt*)pQuery->pRoot; SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -4440,11 +4456,11 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) int32_t code = TSDB_CODE_SUCCESS; SNode* pNode; FOREACH(pNode, pStmt->pSubTables) { - if(pCxt->pParseCxt->schemalessType == 0 && - (code = isNotSchemalessDb(pCxt->pParseCxt, ((SCreateSubTableClause*)pNode)->dbName)) != TSDB_CODE_SUCCESS){ - return code; + SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode; + code = checkSchemalessDb(pCxt, pClause->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteCreateSubTable(pCxt, pClause, pVgroupHashmap); } - code = rewriteCreateSubTable(pCxt, (SCreateSubTableClause*)pNode, pVgroupHashmap); if (TSDB_CODE_SUCCESS != code) { taosHashCleanup(pVgroupHashmap); return code; @@ -4634,27 +4650,27 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS strlen(pStmt->pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pStmt->pVal->literal); } - SArray *pTagVals = taosArrayInit(1, sizeof(STagVal)); + SArray* pTagVals = taosArrayInit(1, sizeof(STagVal)); int32_t code = TSDB_CODE_SUCCESS; - STag* pTag = NULL; - do{ + STag* pTag = NULL; + do { code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf); if (TSDB_CODE_SUCCESS != code) { break; } - }while(0); + } while (0); for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { - STagVal *p = (STagVal *)taosArrayGet(pTagVals, i); - if(IS_VAR_DATA_TYPE(p->type)){ + STagVal* p = (STagVal*)taosArrayGet(pTagVals, i); + if (IS_VAR_DATA_TYPE(p->type)) { taosMemoryFree(p->pData); } } taosArrayDestroy(pTagVals); - if (code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { return code; } pReq->nTagVal = pTag->len; - pReq->pTagVal = (uint8_t *)pTag; + pReq->pTagVal = (uint8_t*)pTag; pStmt->pVal->datum.p = (char*)pTag; // for free } else { pReq->nTagVal = pStmt->pVal->node.resType.bytes; @@ -4855,13 +4871,8 @@ static int32_t buildModifyVnodeArray(STranslateContext* pCxt, SAlterTableStmt* p static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { SAlterTableStmt* pStmt = (SAlterTableStmt*)pQuery->pRoot; - int32_t code = TSDB_CODE_SUCCESS; - if(pCxt->pParseCxt->schemalessType == 0 && - (code = isNotSchemalessDb(pCxt->pParseCxt, pStmt->dbName)) != TSDB_CODE_SUCCESS){ - return code; - } - - STableMeta* pTableMeta = NULL; + int32_t code = checkSchemalessDb(pCxt, pStmt->dbName); + STableMeta* pTableMeta = NULL; code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pTableMeta); if (TSDB_CODE_SUCCESS != code) { return code; @@ -5038,9 +5049,6 @@ int32_t translate(SParseContext* pParseCxt, SQuery* pQuery) { STranslateContext cxt = {0}; int32_t code = initTranslateContext(pParseCxt, pQuery->pMetaCache, &cxt); - if (TSDB_CODE_SUCCESS == code) { - code = fmFuncMgtInit(); - } if (TSDB_CODE_SUCCESS == code) { code = rewriteQuery(&cxt, pQuery); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 0a1915d6c2fc5c1c776f6e991a2e39ee6d8a9aa3..f30cc1111f15d5d238e4c9cc724c0325a30af579 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -178,6 +178,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Only tag can be json type"; case TSDB_CODE_PAR_VALUE_TOO_LONG: return "Value too long for column/tag: %s"; + case TSDB_CODE_PAR_INVALID_DELETE_WHERE: + return "The DELETE statement must have a definite time window range"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: @@ -322,11 +324,11 @@ static bool isValidateTag(char* input) { return true; } -int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMsgBuf* pMsgBuf) { +int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, SMsgBuf* pMsgBuf) { int32_t retCode = TSDB_CODE_SUCCESS; - cJSON* root = NULL; + cJSON* root = NULL; SHashObj* keyHash = NULL; - int32_t size = 0; + int32_t size = 0; // set json NULL data if (!json || strtrim((char*)json) == 0 || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0) { retCode = TSDB_CODE_SUCCESS; @@ -371,7 +373,8 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMs } STagVal val = {0}; val.pKey = jsonKey; - taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless + taosHashPut(keyHash, jsonKey, keyLen, &keyLen, + CHAR_BYTES); // add key to hash to remove dumplicate, value is useless if (item->type == cJSON_String) { // add json value format: type|data char* jsonValue = item->valuestring; @@ -382,8 +385,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMs goto end; } val.type = TSDB_DATA_TYPE_NCHAR; - if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, - (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { + if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno)); retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); @@ -413,7 +415,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMs end: taosHashCleanup(keyHash); - if(retCode == TSDB_CODE_SUCCESS){ + if (retCode == TSDB_CODE_SUCCESS) { tTagNew(pTagVals, 1, true, ppTag); } cJSON_Delete(root); @@ -679,6 +681,7 @@ int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, S tNameExtractFullName(pName, fullName); STableMeta** pRes = taosHashGet(pMetaCache->pTableMeta, fullName, strlen(fullName)); if (NULL == pRes || NULL == *pRes) { + parserError("getTableMetaFromCache error: %s", fullName); return TSDB_CODE_PAR_INTERNAL_ERROR; } *pMeta = tableMetaDup(*pRes); @@ -707,6 +710,7 @@ int32_t reserveDbVgInfoInCache(int32_t acctId, const char* pDb, SParseMetaCache* int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo) { SArray** pRes = taosHashGet(pMetaCache->pDbVgroup, pDbFName, strlen(pDbFName)); if (NULL == pRes) { + parserError("getDbVgInfoFromCache error: %s", pDbFName); return TSDB_CODE_PAR_INTERNAL_ERROR; } // *pRes is null, which is a legal value, indicating that the user DB has not been created @@ -734,6 +738,7 @@ int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, tNameExtractFullName(pName, fullName); SVgroupInfo** pRes = taosHashGet(pMetaCache->pTableVgroup, fullName, strlen(fullName)); if (NULL == pRes || NULL == *pRes) { + parserError("getTableVgroupFromCache error: %s", fullName); return TSDB_CODE_PAR_INTERNAL_ERROR; } memcpy(pVgroup, *pRes, sizeof(SVgroupInfo)); @@ -748,6 +753,7 @@ int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFNam int32_t* pTableNum) { SDbInfo** pRes = taosHashGet(pMetaCache->pDbCfg, pDbFName, strlen(pDbFName)); if (NULL == pRes || NULL == *pRes) { + parserError("getDbVgVersionFromCache error: %s", pDbFName); return TSDB_CODE_PAR_INTERNAL_ERROR; } *pVersion = (*pRes)->vgVer; @@ -763,6 +769,7 @@ int32_t reserveDbCfgInCache(int32_t acctId, const char* pDb, SParseMetaCache* pM int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo) { SDbCfgInfo** pRes = taosHashGet(pMetaCache->pDbCfg, pDbFName, strlen(pDbFName)); if (NULL == pRes || NULL == *pRes) { + parserError("getDbCfgFromCache error: %s", pDbFName); return TSDB_CODE_PAR_INTERNAL_ERROR; } memcpy(pInfo, *pRes, sizeof(SDbCfgInfo)); @@ -801,6 +808,7 @@ int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, con int32_t len = userAuthToStringExt(pUser, pDbFName, type, key); bool* pRes = taosHashGet(pMetaCache->pUserAuth, key, len); if (NULL == pRes) { + parserError("getUserAuthFromCache error: %s, %s, %d", pUser, pDbFName, type); return TSDB_CODE_PAR_INTERNAL_ERROR; } *pPass = *pRes; @@ -820,6 +828,7 @@ int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache) { int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFuncInfo* pInfo) { SFuncInfo** pRes = taosHashGet(pMetaCache->pUdf, pFunc, strlen(pFunc)); if (NULL == pRes || NULL == *pRes) { + parserError("getUdfInfoFromCache error: %s", pFunc); return TSDB_CODE_PAR_INTERNAL_ERROR; } memcpy(pInfo, *pRes, sizeof(SFuncInfo)); diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index ff4fe4032e9be6ab95696bf41d6e1f398983e7b1..ed837ae20f86b46dfc27a8a55d378b0af53bb53e 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -104,25 +104,25 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 357 +#define YYNOCODE 358 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - SAlterOption yy53; - ENullOrder yy109; - SToken yy113; - EJoinType yy120; - int64_t yy123; - bool yy131; - EOrder yy428; - SDataType yy490; - EFillMode yy522; - int32_t yy550; - EOperatorType yy632; - SNodeList* yy670; - SNode* yy686; + EOrder yy14; + ENullOrder yy17; + SNodeList* yy60; + SToken yy105; + int32_t yy140; + SNode* yy172; + EFillMode yy202; + SDataType yy248; + EOperatorType yy572; + int64_t yy593; + SAlterOption yy609; + bool yy617; + EJoinType yy636; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -138,18 +138,17 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 612 -#define YYNRULE 451 -#define YYNRULE_WITH_ACTION 451 -#define YYNTOKEN 237 -#define YY_MAX_SHIFT 611 -#define YY_MIN_SHIFTREDUCE 898 -#define YY_MAX_SHIFTREDUCE 1348 -#define YY_ERROR_ACTION 1349 -#define YY_ACCEPT_ACTION 1350 -#define YY_NO_ACTION 1351 -#define YY_MIN_REDUCE 1352 -#define YY_MAX_REDUCE 1802 +#define YYNSTATE 616 +#define YYNRULE 452 +#define YYNTOKEN 238 +#define YY_MAX_SHIFT 615 +#define YY_MIN_SHIFTREDUCE 902 +#define YY_MAX_SHIFTREDUCE 1353 +#define YY_ERROR_ACTION 1354 +#define YY_ACCEPT_ACTION 1355 +#define YY_NO_ACTION 1356 +#define YY_MIN_REDUCE 1357 +#define YY_MAX_REDUCE 1808 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -216,622 +215,606 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2125) +#define YY_ACTTAB_COUNT (2165) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 132, 1780, 345, 1636, 1440, 1636, 294, 385, 311, 386, - /* 10 */ 1384, 78, 35, 33, 1779, 1472, 24, 1649, 1777, 131, - /* 20 */ 303, 1364, 1162, 1633, 114, 1633, 36, 34, 32, 31, - /* 30 */ 30, 1780, 1475, 36, 34, 32, 31, 30, 1629, 1635, - /* 40 */ 1629, 1635, 1780, 525, 147, 1665, 928, 1160, 1777, 529, - /* 50 */ 525, 529, 1350, 489, 393, 146, 386, 1384, 14, 1777, - /* 60 */ 35, 33, 1289, 509, 1168, 56, 384, 1619, 303, 388, - /* 70 */ 1162, 36, 34, 32, 31, 30, 36, 34, 32, 31, - /* 80 */ 30, 1, 77, 1678, 932, 933, 82, 1650, 512, 1652, - /* 90 */ 1653, 508, 73, 529, 1375, 1160, 1718, 1414, 1780, 1296, - /* 100 */ 296, 1714, 142, 608, 39, 1186, 14, 1353, 35, 33, - /* 110 */ 319, 1778, 1168, 1161, 220, 1777, 303, 277, 1162, 462, - /* 120 */ 468, 1745, 36, 34, 32, 31, 30, 71, 95, 2, - /* 130 */ 1374, 94, 93, 92, 91, 90, 89, 88, 87, 86, - /* 140 */ 525, 1200, 55, 1160, 1619, 315, 1303, 307, 1476, 1251, - /* 150 */ 1780, 608, 1563, 1565, 14, 129, 1163, 438, 437, 1780, - /* 160 */ 1168, 1161, 436, 146, 1485, 110, 433, 1777, 277, 432, - /* 170 */ 431, 430, 146, 945, 497, 944, 1777, 2, 1166, 1167, - /* 180 */ 1619, 1213, 1214, 1216, 1217, 1218, 1219, 1220, 505, 527, - /* 190 */ 1228, 1229, 1230, 1231, 1232, 1233, 286, 1239, 1252, 608, - /* 200 */ 1251, 38, 946, 1186, 1163, 55, 62, 95, 149, 1161, - /* 210 */ 94, 93, 92, 91, 90, 89, 88, 87, 86, 1257, - /* 220 */ 1732, 36, 34, 32, 31, 30, 1166, 1167, 1479, 1213, - /* 230 */ 1214, 1216, 1217, 1218, 1219, 1220, 505, 527, 1228, 1229, - /* 240 */ 1230, 1231, 1232, 1233, 1729, 287, 1373, 285, 284, 1252, - /* 250 */ 426, 403, 1163, 1372, 428, 27, 301, 1246, 1247, 1248, - /* 260 */ 1249, 1250, 1254, 1255, 1256, 513, 1188, 1215, 306, 149, - /* 270 */ 1257, 1572, 28, 228, 1166, 1167, 427, 1213, 1214, 1216, - /* 280 */ 1217, 1218, 1219, 1220, 505, 527, 1228, 1229, 1230, 1231, - /* 290 */ 1232, 1233, 35, 33, 1352, 1313, 1619, 64, 292, 1461, - /* 300 */ 303, 192, 1162, 1619, 526, 351, 27, 301, 1246, 1247, - /* 310 */ 1248, 1249, 1250, 1254, 1255, 1256, 349, 1189, 104, 103, - /* 320 */ 102, 101, 100, 99, 98, 97, 96, 1160, 149, 452, - /* 330 */ 560, 1649, 149, 1483, 472, 1311, 1312, 1314, 1315, 275, - /* 340 */ 35, 33, 1234, 1162, 1168, 486, 313, 1665, 303, 559, - /* 350 */ 1162, 558, 557, 556, 129, 479, 403, 1345, 526, 1665, - /* 360 */ 498, 8, 1371, 1485, 1560, 1215, 526, 489, 1160, 1780, - /* 370 */ 350, 157, 1527, 392, 113, 1160, 388, 509, 360, 293, - /* 380 */ 1186, 1619, 146, 608, 1525, 1168, 1777, 1483, 35, 33, - /* 390 */ 478, 219, 1168, 1161, 1370, 1483, 303, 1678, 1162, 1459, - /* 400 */ 82, 1650, 512, 1652, 1653, 508, 55, 529, 26, 9, - /* 410 */ 1718, 111, 1619, 1288, 296, 1714, 142, 141, 36, 34, - /* 420 */ 32, 31, 30, 1160, 608, 488, 143, 1725, 1726, 1521, - /* 430 */ 1730, 608, 62, 1369, 1161, 1746, 1163, 1344, 438, 437, - /* 440 */ 1168, 1161, 204, 436, 1619, 109, 110, 433, 11, 10, - /* 450 */ 432, 431, 430, 480, 1478, 1368, 562, 9, 1166, 1167, - /* 460 */ 475, 1213, 1214, 1216, 1217, 1218, 1219, 1220, 505, 527, - /* 470 */ 1228, 1229, 1230, 1231, 1232, 1233, 1187, 1163, 316, 608, - /* 480 */ 344, 336, 343, 1619, 1163, 1460, 129, 997, 149, 1161, - /* 490 */ 36, 34, 32, 31, 30, 1485, 1527, 604, 603, 1166, - /* 500 */ 1167, 338, 334, 308, 999, 1619, 1166, 1167, 1525, 1213, - /* 510 */ 1214, 1216, 1217, 1218, 1219, 1220, 505, 527, 1228, 1229, - /* 520 */ 1230, 1231, 1232, 1233, 36, 34, 32, 31, 30, 1265, - /* 530 */ 481, 476, 1163, 149, 7, 1035, 552, 551, 550, 1039, - /* 540 */ 549, 1041, 1042, 548, 1044, 545, 1367, 1050, 542, 1052, - /* 550 */ 1053, 539, 536, 1366, 1166, 1167, 1649, 1213, 1214, 1216, - /* 560 */ 1217, 1218, 1219, 1220, 505, 527, 1228, 1229, 1230, 1231, - /* 570 */ 1232, 1233, 35, 33, 274, 375, 1184, 1411, 560, 526, - /* 580 */ 303, 526, 1162, 368, 1665, 1732, 380, 250, 155, 390, - /* 590 */ 1513, 105, 507, 361, 1200, 1184, 1619, 559, 424, 558, - /* 600 */ 557, 556, 509, 1619, 381, 1527, 1619, 1160, 1483, 1728, - /* 610 */ 1483, 1363, 314, 60, 1253, 513, 59, 1525, 1287, 159, - /* 620 */ 158, 1573, 1678, 128, 1168, 270, 1650, 512, 1652, 1653, - /* 630 */ 508, 506, 529, 503, 1690, 1258, 486, 584, 583, 582, - /* 640 */ 318, 2, 581, 580, 579, 115, 574, 573, 572, 571, - /* 650 */ 570, 569, 568, 567, 122, 563, 1362, 32, 31, 30, - /* 660 */ 1186, 1619, 1361, 608, 1360, 113, 435, 434, 562, 578, - /* 670 */ 576, 25, 1359, 1161, 379, 1458, 1358, 374, 373, 372, - /* 680 */ 371, 370, 367, 366, 365, 364, 363, 359, 358, 357, - /* 690 */ 356, 355, 354, 353, 352, 486, 1564, 1565, 526, 932, - /* 700 */ 933, 1732, 111, 1527, 1284, 198, 1619, 54, 1357, 1356, - /* 710 */ 402, 526, 1619, 1355, 1619, 1526, 1163, 144, 1725, 1726, - /* 720 */ 1185, 1730, 1619, 105, 113, 1727, 1619, 1483, 129, 1649, - /* 730 */ 429, 55, 566, 65, 1455, 1365, 1474, 1486, 1166, 1167, - /* 740 */ 1483, 1213, 1214, 1216, 1217, 1218, 1219, 1220, 505, 527, - /* 750 */ 1228, 1229, 1230, 1231, 1232, 1233, 1633, 1665, 1619, 1619, - /* 760 */ 1649, 111, 1608, 1619, 428, 510, 969, 944, 560, 1737, - /* 770 */ 1284, 1629, 1635, 1147, 1148, 509, 145, 1725, 1726, 1619, - /* 780 */ 1730, 555, 529, 970, 490, 526, 427, 559, 1665, 558, - /* 790 */ 557, 556, 422, 502, 577, 1678, 510, 1480, 81, 1650, - /* 800 */ 512, 1652, 1653, 508, 494, 529, 509, 326, 1718, 1468, - /* 810 */ 1619, 526, 276, 1714, 1483, 490, 183, 185, 1637, 181, - /* 820 */ 184, 1649, 1215, 1599, 1780, 187, 1678, 1470, 186, 81, - /* 830 */ 1650, 512, 1652, 1653, 508, 339, 529, 148, 1633, 1718, - /* 840 */ 1483, 1777, 1639, 276, 1714, 130, 310, 309, 526, 1665, - /* 850 */ 256, 565, 450, 1629, 1635, 1780, 1176, 510, 149, 1466, - /* 860 */ 460, 195, 254, 53, 529, 448, 52, 509, 146, 504, - /* 870 */ 526, 1619, 1777, 443, 526, 189, 119, 1483, 188, 1641, - /* 880 */ 46, 1169, 523, 160, 1649, 207, 524, 1678, 451, 526, - /* 890 */ 82, 1650, 512, 1652, 1653, 508, 1171, 529, 1168, 1483, - /* 900 */ 1718, 241, 191, 1483, 296, 1714, 1793, 1401, 55, 1396, - /* 910 */ 1394, 526, 1665, 1243, 446, 1752, 554, 464, 1483, 440, - /* 920 */ 510, 1310, 1441, 317, 190, 37, 209, 492, 1170, 439, - /* 930 */ 509, 441, 444, 37, 1619, 1347, 1348, 530, 46, 223, - /* 940 */ 1483, 37, 11, 10, 80, 230, 117, 1172, 459, 51, - /* 950 */ 1678, 473, 50, 82, 1650, 512, 1652, 1653, 508, 453, - /* 960 */ 529, 214, 1174, 1718, 1666, 421, 1259, 296, 1714, 1793, - /* 970 */ 1649, 1385, 118, 119, 1221, 58, 57, 348, 1775, 249, - /* 980 */ 154, 1522, 1120, 222, 1748, 342, 232, 518, 495, 534, - /* 990 */ 1177, 118, 119, 487, 1173, 225, 1184, 273, 1665, 3, - /* 1000 */ 332, 1649, 328, 324, 151, 321, 510, 227, 325, 120, - /* 1010 */ 282, 997, 1180, 238, 1028, 1131, 509, 246, 118, 283, - /* 1020 */ 1619, 362, 1562, 527, 1228, 1229, 156, 369, 377, 1665, - /* 1030 */ 1056, 376, 1060, 1066, 378, 149, 1678, 510, 382, 82, - /* 1040 */ 1650, 512, 1652, 1653, 508, 1190, 529, 509, 383, 1718, - /* 1050 */ 1064, 1619, 391, 296, 1714, 1793, 490, 1193, 486, 121, - /* 1060 */ 394, 163, 1649, 395, 1736, 165, 1192, 1678, 1194, 397, - /* 1070 */ 261, 1650, 512, 1652, 1653, 508, 396, 529, 168, 399, - /* 1080 */ 170, 400, 1191, 401, 173, 61, 425, 113, 404, 176, - /* 1090 */ 1665, 1473, 423, 180, 1168, 291, 1780, 1469, 510, 85, - /* 1100 */ 247, 454, 1603, 455, 182, 193, 490, 458, 509, 148, - /* 1110 */ 123, 124, 1619, 1777, 1471, 1467, 125, 490, 196, 126, - /* 1120 */ 461, 199, 202, 1189, 111, 1649, 1759, 466, 1678, 474, - /* 1130 */ 516, 261, 1650, 512, 1652, 1653, 508, 1758, 529, 217, - /* 1140 */ 1725, 485, 465, 484, 6, 483, 1780, 471, 463, 205, - /* 1150 */ 208, 470, 295, 1665, 477, 213, 1649, 1780, 1739, 148, - /* 1160 */ 1284, 510, 5, 1777, 1749, 1188, 112, 1733, 40, 136, - /* 1170 */ 146, 509, 499, 496, 1777, 1619, 215, 18, 1571, 1570, - /* 1180 */ 1796, 514, 519, 297, 1665, 515, 305, 520, 234, 216, - /* 1190 */ 521, 1678, 510, 236, 83, 1650, 512, 1652, 1653, 508, - /* 1200 */ 1699, 529, 509, 248, 1718, 70, 1619, 72, 1717, 1714, - /* 1210 */ 1649, 1484, 251, 607, 532, 1456, 1776, 221, 47, 1649, - /* 1220 */ 135, 493, 1678, 243, 224, 83, 1650, 512, 1652, 1653, - /* 1230 */ 508, 500, 529, 226, 262, 1718, 272, 263, 1665, 501, - /* 1240 */ 1714, 253, 255, 1613, 1612, 320, 510, 1665, 1609, 322, - /* 1250 */ 323, 1156, 1157, 152, 327, 510, 509, 1607, 329, 330, - /* 1260 */ 1619, 331, 1606, 333, 1605, 509, 335, 1604, 337, 1619, - /* 1270 */ 1589, 153, 340, 341, 1134, 1133, 1678, 346, 347, 133, - /* 1280 */ 1650, 512, 1652, 1653, 508, 1678, 529, 1583, 83, 1650, - /* 1290 */ 512, 1652, 1653, 508, 1582, 529, 611, 1649, 1718, 1103, - /* 1300 */ 1555, 1554, 1553, 1715, 1581, 1580, 1552, 1551, 1649, 1550, - /* 1310 */ 245, 1549, 1548, 1547, 1546, 1545, 1544, 1543, 1542, 1541, - /* 1320 */ 1540, 1539, 106, 491, 1794, 1665, 1538, 1537, 600, 596, - /* 1330 */ 592, 588, 244, 510, 1536, 116, 1665, 1535, 1534, 1533, - /* 1340 */ 1532, 1531, 1530, 509, 510, 1105, 1529, 1619, 161, 935, - /* 1350 */ 469, 1528, 1413, 1381, 509, 1380, 1597, 79, 1619, 107, - /* 1360 */ 239, 934, 108, 1678, 1591, 139, 271, 1650, 512, 1652, - /* 1370 */ 1653, 508, 387, 529, 1678, 389, 1649, 266, 1650, 512, - /* 1380 */ 1652, 1653, 508, 162, 529, 1579, 167, 169, 1578, 1568, - /* 1390 */ 1462, 172, 963, 522, 1412, 1410, 407, 405, 1408, 411, - /* 1400 */ 1406, 1404, 406, 415, 1665, 409, 410, 413, 414, 419, - /* 1410 */ 418, 1393, 510, 1392, 417, 482, 1070, 179, 467, 1379, - /* 1420 */ 1464, 200, 509, 1463, 1069, 1649, 1619, 996, 995, 994, - /* 1430 */ 993, 990, 575, 45, 577, 1402, 1649, 288, 1397, 1139, - /* 1440 */ 289, 194, 1678, 989, 988, 133, 1650, 512, 1652, 1653, - /* 1450 */ 508, 442, 529, 1665, 1395, 290, 445, 1378, 447, 1377, - /* 1460 */ 1596, 510, 449, 84, 1665, 201, 456, 1577, 1141, 1590, - /* 1470 */ 1576, 509, 507, 1575, 1567, 1619, 212, 49, 300, 41, - /* 1480 */ 66, 457, 509, 4, 15, 134, 1619, 1649, 37, 48, - /* 1490 */ 1795, 1678, 206, 43, 271, 1650, 512, 1652, 1653, 508, - /* 1500 */ 1639, 529, 1678, 211, 1309, 270, 1650, 512, 1652, 1653, - /* 1510 */ 508, 210, 529, 197, 1691, 1665, 203, 10, 22, 23, - /* 1520 */ 42, 1302, 67, 510, 178, 218, 1649, 1281, 1280, 127, - /* 1530 */ 137, 1338, 1327, 509, 17, 1333, 140, 1619, 19, 1649, - /* 1540 */ 302, 1332, 420, 416, 412, 408, 177, 298, 1337, 1336, - /* 1550 */ 299, 1244, 29, 1678, 1665, 138, 271, 1650, 512, 1652, - /* 1560 */ 1653, 508, 510, 529, 1223, 1222, 12, 1665, 20, 1208, - /* 1570 */ 150, 63, 509, 21, 175, 510, 1619, 229, 1307, 304, - /* 1580 */ 231, 1566, 16, 235, 1178, 509, 13, 511, 1649, 1619, - /* 1590 */ 233, 517, 1678, 68, 69, 271, 1650, 512, 1652, 1653, - /* 1600 */ 508, 237, 529, 1638, 240, 1678, 1225, 73, 257, 1650, - /* 1610 */ 512, 1652, 1653, 508, 1681, 529, 1665, 1649, 528, 44, - /* 1620 */ 531, 1057, 533, 312, 510, 535, 537, 1054, 1049, 538, - /* 1630 */ 540, 174, 1051, 166, 509, 171, 541, 398, 1619, 543, - /* 1640 */ 1045, 546, 544, 1034, 547, 1665, 1043, 1048, 1047, 553, - /* 1650 */ 74, 75, 1065, 510, 1678, 164, 1649, 265, 1650, 512, - /* 1660 */ 1652, 1653, 508, 509, 529, 76, 1063, 1619, 1062, 961, - /* 1670 */ 1046, 561, 985, 1003, 564, 242, 983, 982, 981, 980, - /* 1680 */ 979, 978, 977, 1678, 1665, 976, 267, 1650, 512, 1652, - /* 1690 */ 1653, 508, 510, 529, 998, 973, 972, 971, 968, 967, - /* 1700 */ 966, 1000, 509, 1409, 585, 1649, 1619, 586, 587, 1407, - /* 1710 */ 589, 590, 591, 1405, 593, 594, 1649, 595, 1403, 597, - /* 1720 */ 599, 598, 1678, 1391, 601, 258, 1650, 512, 1652, 1653, - /* 1730 */ 508, 1390, 529, 1665, 602, 1376, 605, 606, 1351, 1351, - /* 1740 */ 609, 510, 1164, 252, 1665, 610, 1351, 1351, 1351, 1351, - /* 1750 */ 1351, 509, 510, 1351, 1351, 1619, 1351, 1351, 1351, 1351, - /* 1760 */ 1351, 1351, 509, 1351, 1351, 1649, 1619, 1351, 1351, 1351, - /* 1770 */ 1351, 1678, 1351, 1351, 268, 1650, 512, 1652, 1653, 508, - /* 1780 */ 1649, 529, 1678, 1351, 1351, 259, 1650, 512, 1652, 1653, - /* 1790 */ 508, 1351, 529, 1665, 1351, 1351, 1351, 1649, 1351, 1351, - /* 1800 */ 1351, 510, 1351, 1351, 1351, 1351, 1351, 1351, 1665, 1351, - /* 1810 */ 1351, 509, 1649, 1351, 1351, 1619, 510, 1351, 1351, 1351, - /* 1820 */ 1351, 1351, 1351, 1351, 1351, 1665, 509, 1351, 1351, 1649, - /* 1830 */ 1619, 1678, 1351, 510, 269, 1650, 512, 1652, 1653, 508, - /* 1840 */ 1665, 529, 1351, 509, 1351, 1351, 1678, 1619, 510, 260, - /* 1850 */ 1650, 512, 1652, 1653, 508, 1351, 529, 1665, 509, 1351, - /* 1860 */ 1351, 1351, 1619, 1678, 1351, 510, 1661, 1650, 512, 1652, - /* 1870 */ 1653, 508, 1351, 529, 1351, 509, 1649, 1351, 1678, 1619, - /* 1880 */ 1351, 1660, 1650, 512, 1652, 1653, 508, 1351, 529, 1351, - /* 1890 */ 1351, 1351, 1351, 1351, 1351, 1678, 1351, 1351, 1659, 1650, - /* 1900 */ 512, 1652, 1653, 508, 1665, 529, 1351, 1351, 1649, 1351, - /* 1910 */ 1351, 1351, 510, 1351, 1351, 1351, 1351, 1351, 1351, 1351, - /* 1920 */ 1351, 1351, 509, 1351, 1351, 1649, 1619, 1351, 1351, 1351, - /* 1930 */ 1351, 1351, 1351, 1351, 1351, 1351, 1665, 1351, 1351, 1351, - /* 1940 */ 1351, 1351, 1678, 1351, 510, 280, 1650, 512, 1652, 1653, - /* 1950 */ 508, 1351, 529, 1665, 509, 1351, 1351, 1649, 1619, 1351, - /* 1960 */ 1351, 510, 1351, 1351, 1351, 1351, 1351, 1351, 1351, 1351, - /* 1970 */ 1351, 509, 1351, 1351, 1678, 1619, 1351, 279, 1650, 512, - /* 1980 */ 1652, 1653, 508, 1351, 529, 1665, 1351, 1351, 1351, 1351, - /* 1990 */ 1351, 1678, 1351, 510, 281, 1650, 512, 1652, 1653, 508, - /* 2000 */ 1351, 529, 1351, 509, 1351, 1351, 1649, 1619, 1351, 1351, - /* 2010 */ 1351, 1351, 1351, 1351, 1351, 1351, 1351, 1351, 1351, 486, - /* 2020 */ 1351, 1351, 1351, 1678, 1351, 1351, 278, 1650, 512, 1652, - /* 2030 */ 1653, 508, 1351, 529, 1665, 1351, 1351, 1351, 1351, 1351, - /* 2040 */ 1351, 1351, 510, 1351, 1351, 1351, 1351, 1351, 113, 1351, - /* 2050 */ 1351, 1351, 509, 1351, 1351, 1351, 1619, 1351, 1351, 1351, - /* 2060 */ 1351, 1351, 1351, 1351, 1351, 1351, 1351, 490, 1351, 1351, - /* 2070 */ 1351, 1351, 1678, 1351, 1351, 264, 1650, 512, 1652, 1653, - /* 2080 */ 508, 1351, 529, 1351, 1351, 111, 1351, 1351, 1351, 1351, - /* 2090 */ 1351, 1351, 1351, 1351, 1351, 1351, 1351, 1351, 1351, 1351, - /* 2100 */ 217, 1725, 485, 1351, 484, 1351, 1351, 1780, 1351, 1351, - /* 2110 */ 1351, 1351, 1351, 1351, 1351, 1351, 1351, 1351, 1351, 1351, - /* 2120 */ 146, 1351, 1351, 1351, 1777, + /* 0 */ 72, 355, 1655, 1642, 28, 230, 296, 1465, 1642, 608, + /* 10 */ 607, 313, 35, 33, 530, 36, 34, 32, 31, 30, + /* 20 */ 305, 1481, 1167, 1639, 530, 490, 106, 1477, 1639, 389, + /* 30 */ 1671, 390, 1389, 428, 490, 277, 152, 349, 514, 1635, + /* 40 */ 1641, 529, 133, 1488, 1635, 1641, 1445, 1165, 513, 132, + /* 50 */ 533, 1369, 1625, 1488, 114, 533, 529, 494, 14, 530, + /* 60 */ 35, 33, 1294, 114, 1173, 1416, 932, 63, 305, 1684, + /* 70 */ 1167, 353, 82, 1656, 516, 1658, 1659, 512, 1786, 533, + /* 80 */ 110, 1, 1724, 56, 1532, 1308, 278, 1720, 1488, 1483, + /* 90 */ 147, 295, 112, 517, 1783, 1165, 1530, 529, 1786, 1578, + /* 100 */ 564, 112, 39, 612, 936, 937, 14, 145, 1731, 1732, + /* 110 */ 149, 1736, 1173, 1166, 1783, 492, 144, 1731, 1732, 563, + /* 120 */ 1736, 562, 561, 560, 407, 588, 587, 586, 320, 2, + /* 130 */ 585, 584, 583, 116, 578, 577, 576, 575, 574, 573, + /* 140 */ 572, 571, 123, 567, 1358, 36, 34, 32, 31, 30, + /* 150 */ 397, 612, 390, 1389, 142, 317, 1168, 1191, 38, 1671, + /* 160 */ 55, 1166, 1568, 1570, 1466, 96, 1526, 483, 95, 94, + /* 170 */ 93, 92, 91, 90, 89, 88, 87, 479, 388, 1171, + /* 180 */ 1172, 392, 1218, 1219, 1221, 1222, 1223, 1224, 1225, 509, + /* 190 */ 531, 1233, 1234, 1235, 1236, 1237, 1238, 36, 34, 32, + /* 200 */ 31, 30, 517, 482, 1168, 308, 581, 96, 1577, 150, + /* 210 */ 95, 94, 93, 92, 91, 90, 89, 88, 87, 288, + /* 220 */ 130, 407, 36, 34, 32, 31, 30, 1171, 1172, 1491, + /* 230 */ 1218, 1219, 1221, 1222, 1223, 1224, 1225, 509, 531, 1233, + /* 240 */ 1234, 1235, 1236, 1237, 1238, 35, 33, 343, 485, 480, + /* 250 */ 1355, 442, 441, 305, 1655, 1167, 440, 200, 252, 111, + /* 260 */ 437, 1518, 379, 436, 435, 434, 35, 33, 289, 447, + /* 270 */ 287, 286, 1001, 430, 305, 1479, 1167, 432, 129, 309, + /* 280 */ 1165, 79, 1671, 55, 455, 66, 150, 130, 150, 1003, + /* 290 */ 514, 14, 1205, 484, 115, 1639, 1490, 1173, 193, 431, + /* 300 */ 513, 1165, 1480, 949, 1625, 948, 161, 160, 321, 494, + /* 310 */ 450, 1635, 1641, 394, 2, 444, 1192, 1643, 1173, 1189, + /* 320 */ 192, 1684, 533, 490, 82, 1656, 516, 1658, 1659, 512, + /* 330 */ 24, 533, 950, 1350, 1724, 8, 612, 1639, 278, 1720, + /* 340 */ 36, 34, 32, 31, 30, 51, 1166, 1464, 50, 1786, + /* 350 */ 1786, 566, 114, 1635, 1641, 65, 294, 612, 1380, 194, + /* 360 */ 559, 147, 147, 1463, 533, 1783, 1783, 1166, 1357, 1039, + /* 370 */ 556, 555, 554, 1043, 553, 1045, 1046, 552, 1048, 549, + /* 380 */ 1379, 1054, 546, 1056, 1057, 543, 540, 7, 55, 1168, + /* 390 */ 112, 1190, 105, 104, 103, 102, 101, 100, 99, 98, + /* 400 */ 97, 32, 31, 30, 566, 146, 1731, 1732, 1625, 1736, + /* 410 */ 1168, 150, 1171, 1172, 1349, 1218, 1219, 1221, 1222, 1223, + /* 420 */ 1224, 1225, 509, 531, 1233, 1234, 1235, 1236, 1237, 1238, + /* 430 */ 1625, 569, 157, 1171, 1172, 463, 1218, 1219, 1221, 1222, + /* 440 */ 1223, 1224, 1225, 509, 531, 1233, 1234, 1235, 1236, 1237, + /* 450 */ 1238, 35, 33, 1239, 26, 1378, 564, 61, 466, 305, + /* 460 */ 60, 1167, 564, 150, 36, 34, 32, 31, 30, 63, + /* 470 */ 1406, 396, 35, 33, 392, 563, 1655, 562, 561, 560, + /* 480 */ 305, 563, 1167, 562, 561, 560, 1165, 1193, 530, 1318, + /* 490 */ 530, 1484, 443, 36, 34, 32, 31, 30, 1786, 1786, + /* 500 */ 354, 1645, 106, 1173, 1671, 1625, 1473, 1165, 55, 433, + /* 510 */ 1785, 147, 493, 456, 1783, 1783, 150, 1488, 1377, 1488, + /* 520 */ 9, 570, 513, 1460, 1173, 1419, 1625, 454, 476, 1316, + /* 530 */ 1317, 1319, 1320, 36, 34, 32, 31, 30, 1647, 279, + /* 540 */ 452, 9, 612, 1684, 439, 438, 83, 1656, 516, 1658, + /* 550 */ 1659, 512, 1166, 533, 1786, 1475, 1724, 1532, 1289, 315, + /* 560 */ 298, 1720, 143, 612, 310, 1786, 147, 130, 1625, 1530, + /* 570 */ 1783, 1256, 1301, 1166, 222, 1786, 1490, 148, 1191, 472, + /* 580 */ 1751, 1783, 348, 1738, 347, 442, 441, 1784, 530, 340, + /* 590 */ 440, 1783, 1532, 111, 437, 1168, 1532, 436, 435, 434, + /* 600 */ 364, 582, 580, 316, 1531, 1569, 1570, 1735, 1530, 342, + /* 610 */ 338, 1293, 221, 1376, 936, 937, 1168, 1488, 1171, 1172, + /* 620 */ 1257, 1218, 1219, 1221, 1222, 1223, 1224, 1225, 509, 531, + /* 630 */ 1233, 1234, 1235, 1236, 1237, 1238, 150, 11, 10, 1171, + /* 640 */ 1172, 1262, 1218, 1219, 1221, 1222, 1223, 1224, 1225, 509, + /* 650 */ 531, 1233, 1234, 1235, 1236, 1237, 1238, 35, 33, 276, + /* 660 */ 1655, 1189, 530, 1625, 530, 305, 1375, 1167, 372, 54, + /* 670 */ 1374, 384, 1373, 1471, 365, 197, 406, 508, 27, 303, + /* 680 */ 1251, 1252, 1253, 1254, 1255, 1259, 1260, 1261, 1671, 385, + /* 690 */ 530, 1488, 1165, 1488, 1244, 530, 514, 1372, 1565, 530, + /* 700 */ 1191, 432, 1485, 1371, 318, 159, 513, 1604, 1191, 1173, + /* 710 */ 1625, 464, 130, 530, 1151, 1152, 1625, 1194, 1270, 1488, + /* 720 */ 1625, 1490, 1625, 431, 1488, 527, 2, 1684, 1488, 1258, + /* 730 */ 83, 1656, 516, 1658, 1659, 512, 948, 533, 1368, 1367, + /* 740 */ 1724, 1220, 1488, 498, 298, 1720, 1799, 1625, 612, 120, + /* 750 */ 1263, 1743, 1289, 1625, 1366, 1758, 1365, 1364, 1166, 383, + /* 760 */ 1363, 426, 378, 377, 376, 375, 374, 371, 370, 369, + /* 770 */ 368, 367, 363, 362, 361, 360, 359, 358, 357, 356, + /* 780 */ 530, 131, 530, 530, 1738, 1370, 258, 25, 1625, 1625, + /* 790 */ 468, 1738, 528, 1362, 243, 319, 1361, 501, 256, 53, + /* 800 */ 1613, 1168, 52, 1360, 1625, 506, 1625, 1625, 1734, 1488, + /* 810 */ 1625, 1488, 1488, 185, 1446, 1733, 183, 1292, 187, 162, + /* 820 */ 189, 186, 1176, 188, 1171, 1172, 46, 1218, 1219, 1221, + /* 830 */ 1222, 1223, 1224, 1225, 509, 531, 1233, 1234, 1235, 1236, + /* 840 */ 1237, 1238, 206, 1625, 55, 330, 1625, 36, 34, 32, + /* 850 */ 31, 30, 191, 1625, 1401, 190, 1167, 1399, 209, 11, + /* 860 */ 10, 1352, 1353, 1220, 46, 37, 37, 1315, 37, 232, + /* 870 */ 558, 1220, 457, 118, 119, 78, 445, 120, 1175, 448, + /* 880 */ 81, 1165, 538, 225, 973, 74, 119, 477, 1179, 120, + /* 890 */ 496, 216, 1672, 279, 121, 1390, 1655, 1527, 1173, 211, + /* 900 */ 425, 974, 119, 1754, 491, 251, 1264, 1226, 322, 1124, + /* 910 */ 234, 59, 58, 352, 522, 240, 156, 1205, 1032, 1189, + /* 920 */ 224, 346, 3, 1060, 1671, 1256, 1248, 1064, 499, 227, + /* 930 */ 1070, 229, 493, 275, 5, 1068, 336, 612, 332, 328, + /* 940 */ 153, 323, 513, 122, 1178, 329, 1625, 1166, 325, 284, + /* 950 */ 1001, 285, 312, 311, 248, 1135, 158, 366, 1567, 373, + /* 960 */ 381, 380, 1181, 1684, 382, 386, 83, 1656, 516, 1658, + /* 970 */ 1659, 512, 150, 533, 1257, 1655, 1724, 1195, 1198, 387, + /* 980 */ 298, 1720, 143, 395, 502, 398, 399, 1174, 1197, 400, + /* 990 */ 1168, 165, 167, 1199, 170, 1262, 172, 404, 401, 403, + /* 1000 */ 1752, 1196, 175, 1671, 1173, 62, 405, 408, 178, 427, + /* 1010 */ 1173, 514, 86, 1171, 1172, 429, 293, 1478, 1608, 182, + /* 1020 */ 1474, 513, 184, 124, 249, 1625, 125, 1476, 1655, 1472, + /* 1030 */ 126, 127, 27, 303, 1251, 1252, 1253, 1254, 1255, 1259, + /* 1040 */ 1260, 1261, 1684, 534, 195, 83, 1656, 516, 1658, 1659, + /* 1050 */ 512, 458, 533, 1177, 198, 1724, 1671, 462, 465, 298, + /* 1060 */ 1720, 1799, 201, 467, 514, 469, 459, 470, 478, 204, + /* 1070 */ 1781, 1194, 1765, 207, 513, 520, 1764, 475, 1625, 1655, + /* 1080 */ 297, 1755, 210, 481, 215, 6, 487, 1745, 474, 217, + /* 1090 */ 1655, 1289, 113, 1193, 40, 1684, 1182, 503, 83, 1656, + /* 1100 */ 516, 1658, 1659, 512, 1739, 533, 218, 1671, 1724, 137, + /* 1110 */ 500, 299, 298, 1720, 1799, 514, 18, 518, 1671, 1185, + /* 1120 */ 519, 1705, 1576, 1742, 1782, 513, 514, 223, 226, 1625, + /* 1130 */ 531, 1233, 1234, 228, 494, 1575, 513, 1802, 1655, 523, + /* 1140 */ 1625, 307, 236, 497, 524, 494, 1684, 238, 504, 263, + /* 1150 */ 1656, 516, 1658, 1659, 512, 525, 533, 1684, 250, 71, + /* 1160 */ 263, 1656, 516, 1658, 1659, 512, 1671, 533, 1489, 73, + /* 1170 */ 1461, 536, 253, 245, 514, 1786, 611, 47, 1655, 255, + /* 1180 */ 136, 264, 274, 265, 513, 1619, 1786, 149, 1625, 257, + /* 1190 */ 1618, 1783, 57, 1617, 324, 1614, 326, 327, 147, 331, + /* 1200 */ 1160, 1161, 1783, 154, 1612, 1684, 1671, 333, 84, 1656, + /* 1210 */ 516, 1658, 1659, 512, 514, 533, 334, 335, 1724, 1611, + /* 1220 */ 180, 337, 1723, 1720, 513, 1610, 339, 1609, 1625, 341, + /* 1230 */ 1594, 155, 141, 1138, 344, 345, 1137, 1655, 424, 420, + /* 1240 */ 416, 412, 179, 1588, 1587, 1684, 350, 351, 84, 1656, + /* 1250 */ 516, 1658, 1659, 512, 1586, 533, 1585, 1107, 1724, 1109, + /* 1260 */ 117, 1542, 505, 1720, 1560, 1671, 1559, 64, 1558, 1557, + /* 1270 */ 177, 1556, 1555, 511, 1554, 1553, 1552, 1551, 1550, 1549, + /* 1280 */ 1548, 1547, 1546, 513, 1545, 1544, 1543, 1625, 1541, 1655, + /* 1290 */ 1540, 1539, 1538, 1537, 1536, 1535, 1534, 1533, 1655, 1418, + /* 1300 */ 1386, 140, 163, 939, 1684, 1385, 1602, 272, 1656, 516, + /* 1310 */ 1658, 1659, 512, 510, 533, 507, 1696, 1671, 938, 1596, + /* 1320 */ 108, 1584, 171, 1583, 109, 514, 1671, 176, 1573, 168, + /* 1330 */ 1467, 173, 967, 402, 514, 513, 1417, 1415, 1413, 1625, + /* 1340 */ 164, 1411, 391, 1409, 513, 393, 169, 174, 1625, 411, + /* 1350 */ 409, 166, 410, 415, 1655, 413, 1684, 1398, 414, 134, + /* 1360 */ 1656, 516, 1658, 1659, 512, 1684, 533, 417, 84, 1656, + /* 1370 */ 516, 1658, 1659, 512, 418, 533, 419, 422, 1724, 615, + /* 1380 */ 421, 423, 1671, 1721, 1397, 1384, 1469, 1074, 1073, 1468, + /* 1390 */ 514, 579, 1000, 247, 999, 998, 997, 581, 994, 993, + /* 1400 */ 513, 992, 495, 1800, 1625, 107, 181, 473, 1655, 1407, + /* 1410 */ 290, 604, 600, 596, 592, 246, 45, 1655, 1402, 291, + /* 1420 */ 446, 1684, 1400, 292, 273, 1656, 516, 1658, 1659, 512, + /* 1430 */ 449, 533, 1383, 451, 1382, 453, 1671, 85, 1601, 1145, + /* 1440 */ 80, 1595, 460, 241, 514, 1671, 1582, 1581, 1580, 1572, + /* 1450 */ 67, 203, 15, 514, 513, 208, 4, 37, 1625, 49, + /* 1460 */ 461, 135, 48, 513, 199, 43, 213, 1625, 16, 1314, + /* 1470 */ 1655, 205, 212, 214, 22, 1684, 526, 1307, 268, 1656, + /* 1480 */ 516, 1658, 1659, 512, 1684, 533, 128, 134, 1656, 516, + /* 1490 */ 1658, 1659, 512, 68, 533, 1645, 23, 220, 1671, 42, + /* 1500 */ 1655, 471, 17, 1286, 202, 1285, 514, 138, 1343, 1338, + /* 1510 */ 1332, 1337, 300, 1342, 1341, 486, 513, 301, 10, 19, + /* 1520 */ 1625, 1228, 1143, 302, 196, 1249, 29, 139, 1671, 151, + /* 1530 */ 1213, 1801, 1227, 12, 515, 20, 511, 1684, 21, 521, + /* 1540 */ 273, 1656, 516, 1658, 1659, 512, 513, 533, 13, 41, + /* 1550 */ 1625, 231, 1655, 1312, 233, 235, 69, 1571, 70, 237, + /* 1560 */ 1687, 1644, 242, 74, 1230, 532, 1183, 1684, 1655, 44, + /* 1570 */ 272, 1656, 516, 1658, 1659, 512, 239, 533, 535, 1697, + /* 1580 */ 1671, 1061, 537, 314, 539, 541, 1058, 542, 514, 1055, + /* 1590 */ 544, 545, 547, 1049, 548, 550, 1671, 1047, 513, 1053, + /* 1600 */ 551, 1052, 1625, 1051, 514, 304, 1050, 1038, 1655, 75, + /* 1610 */ 76, 1069, 77, 557, 513, 1067, 1066, 965, 1625, 1684, + /* 1620 */ 1655, 306, 273, 1656, 516, 1658, 1659, 512, 565, 533, + /* 1630 */ 989, 1007, 244, 568, 987, 1684, 1671, 986, 273, 1656, + /* 1640 */ 516, 1658, 1659, 512, 514, 533, 985, 984, 1671, 983, + /* 1650 */ 982, 981, 980, 1004, 513, 1002, 514, 977, 1625, 976, + /* 1660 */ 975, 972, 971, 970, 1414, 589, 513, 590, 1655, 591, + /* 1670 */ 1625, 1412, 593, 595, 594, 1684, 1410, 597, 259, 1656, + /* 1680 */ 516, 1658, 1659, 512, 598, 533, 599, 1684, 1408, 601, + /* 1690 */ 267, 1656, 516, 1658, 1659, 512, 1671, 533, 602, 603, + /* 1700 */ 1655, 605, 606, 1395, 514, 1396, 1381, 609, 610, 1356, + /* 1710 */ 1356, 1169, 254, 613, 513, 614, 1356, 1356, 1625, 1356, + /* 1720 */ 1356, 1356, 1356, 1356, 1356, 1356, 1356, 1356, 1671, 1356, + /* 1730 */ 1655, 1356, 1356, 1356, 1356, 1684, 514, 1356, 269, 1656, + /* 1740 */ 516, 1658, 1659, 512, 1356, 533, 513, 1356, 1356, 1356, + /* 1750 */ 1625, 1356, 1356, 1356, 1356, 1356, 1356, 1356, 1671, 1356, + /* 1760 */ 1356, 1356, 1655, 1356, 1356, 1356, 514, 1684, 1356, 1356, + /* 1770 */ 260, 1656, 516, 1658, 1659, 512, 513, 533, 1655, 1356, + /* 1780 */ 1625, 1356, 1356, 1356, 1356, 1356, 1356, 1356, 1356, 1356, + /* 1790 */ 1671, 1356, 1356, 1356, 1655, 1356, 1356, 1684, 514, 1356, + /* 1800 */ 270, 1656, 516, 1658, 1659, 512, 1671, 533, 513, 1356, + /* 1810 */ 1655, 1356, 1625, 1356, 514, 1356, 1356, 1356, 1356, 1356, + /* 1820 */ 1356, 1356, 1671, 1356, 513, 1356, 1356, 1356, 1625, 1684, + /* 1830 */ 514, 1356, 261, 1656, 516, 1658, 1659, 512, 1671, 533, + /* 1840 */ 513, 1356, 1356, 1356, 1625, 1684, 514, 1356, 271, 1656, + /* 1850 */ 516, 1658, 1659, 512, 1356, 533, 513, 1356, 1356, 1356, + /* 1860 */ 1625, 1684, 1655, 1356, 262, 1656, 516, 1658, 1659, 512, + /* 1870 */ 1356, 533, 1356, 1356, 1356, 1356, 1356, 1684, 1655, 1356, + /* 1880 */ 1667, 1656, 516, 1658, 1659, 512, 1356, 533, 1356, 1356, + /* 1890 */ 1671, 1356, 1356, 1356, 1356, 1356, 1356, 1356, 514, 1356, + /* 1900 */ 1356, 1356, 1356, 1356, 1356, 1356, 1671, 1356, 513, 1356, + /* 1910 */ 1655, 1356, 1625, 1356, 514, 1356, 1356, 1356, 1356, 1356, + /* 1920 */ 1356, 1356, 1356, 1356, 513, 1356, 1356, 1356, 1625, 1684, + /* 1930 */ 1356, 1356, 1666, 1656, 516, 1658, 1659, 512, 1671, 533, + /* 1940 */ 1356, 1356, 1655, 1356, 1356, 1684, 514, 1356, 1665, 1656, + /* 1950 */ 516, 1658, 1659, 512, 1356, 533, 513, 1356, 1356, 1356, + /* 1960 */ 1625, 1356, 1356, 1356, 1356, 1356, 1356, 1356, 1356, 1356, + /* 1970 */ 1671, 1356, 1655, 1356, 1356, 1356, 1356, 1684, 514, 1356, + /* 1980 */ 282, 1656, 516, 1658, 1659, 512, 1356, 533, 513, 1356, + /* 1990 */ 1356, 1356, 1625, 1356, 1356, 1356, 1356, 1356, 1356, 1356, + /* 2000 */ 1671, 1356, 1356, 1356, 1655, 1356, 1356, 1356, 514, 1684, + /* 2010 */ 1356, 1356, 281, 1656, 516, 1658, 1659, 512, 513, 533, + /* 2020 */ 1655, 1356, 1625, 1356, 1356, 1356, 1356, 1356, 1356, 1356, + /* 2030 */ 1356, 1356, 1671, 1356, 1356, 1356, 1356, 1356, 1356, 1684, + /* 2040 */ 514, 1356, 283, 1656, 516, 1658, 1659, 512, 1671, 533, + /* 2050 */ 513, 490, 1356, 1356, 1625, 1356, 514, 1356, 1356, 490, + /* 2060 */ 1356, 1356, 1356, 1356, 1356, 1356, 513, 1356, 1356, 1356, + /* 2070 */ 1625, 1684, 1356, 1356, 280, 1656, 516, 1658, 1659, 512, + /* 2080 */ 114, 533, 1356, 1356, 1356, 1356, 1356, 1684, 114, 1356, + /* 2090 */ 266, 1656, 516, 1658, 1659, 512, 1356, 533, 1356, 494, + /* 2100 */ 1356, 1356, 1356, 1356, 1356, 1356, 1356, 494, 1356, 1356, + /* 2110 */ 1356, 1356, 1356, 1356, 1356, 1356, 1356, 1356, 112, 1356, + /* 2120 */ 1356, 1356, 1356, 1356, 1356, 1356, 112, 1356, 1356, 1356, + /* 2130 */ 1356, 1356, 1356, 219, 1731, 489, 1356, 488, 1356, 1356, + /* 2140 */ 1786, 219, 1731, 489, 1356, 488, 1356, 1356, 1786, 1356, + /* 2150 */ 1356, 1356, 149, 1356, 1356, 1356, 1783, 1356, 1356, 1356, + /* 2160 */ 147, 1356, 1356, 1356, 1783, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 253, 335, 295, 270, 257, 270, 273, 243, 273, 245, - /* 10 */ 246, 250, 12, 13, 348, 269, 2, 240, 352, 239, - /* 20 */ 20, 241, 22, 290, 263, 290, 12, 13, 14, 15, - /* 30 */ 16, 335, 271, 12, 13, 14, 15, 16, 305, 306, - /* 40 */ 305, 306, 335, 20, 348, 268, 4, 47, 352, 316, - /* 50 */ 20, 316, 237, 276, 243, 348, 245, 246, 58, 352, - /* 60 */ 12, 13, 14, 286, 64, 4, 244, 290, 20, 247, - /* 70 */ 22, 12, 13, 14, 15, 16, 12, 13, 14, 15, - /* 80 */ 16, 81, 81, 306, 42, 43, 309, 310, 311, 312, - /* 90 */ 313, 314, 91, 316, 240, 47, 319, 0, 335, 14, - /* 100 */ 323, 324, 325, 103, 81, 20, 58, 0, 12, 13, - /* 110 */ 295, 348, 64, 113, 337, 352, 20, 58, 22, 295, - /* 120 */ 343, 344, 12, 13, 14, 15, 16, 250, 21, 81, - /* 130 */ 240, 24, 25, 26, 27, 28, 29, 30, 31, 32, - /* 140 */ 20, 82, 81, 47, 290, 278, 82, 260, 271, 90, - /* 150 */ 335, 103, 285, 286, 58, 268, 156, 60, 61, 335, - /* 160 */ 64, 113, 65, 348, 277, 68, 69, 352, 58, 72, - /* 170 */ 73, 74, 348, 20, 41, 22, 352, 81, 178, 179, - /* 180 */ 290, 181, 182, 183, 184, 185, 186, 187, 188, 189, - /* 190 */ 190, 191, 192, 193, 194, 195, 35, 14, 139, 103, - /* 200 */ 90, 81, 49, 20, 156, 81, 252, 21, 208, 113, - /* 210 */ 24, 25, 26, 27, 28, 29, 30, 31, 32, 160, - /* 220 */ 307, 12, 13, 14, 15, 16, 178, 179, 274, 181, + /* 0 */ 251, 248, 241, 271, 322, 323, 274, 0, 271, 249, + /* 10 */ 250, 274, 12, 13, 248, 12, 13, 14, 15, 16, + /* 20 */ 20, 272, 22, 291, 248, 248, 260, 270, 291, 244, + /* 30 */ 269, 246, 247, 267, 248, 282, 260, 296, 277, 307, + /* 40 */ 308, 20, 254, 277, 307, 308, 258, 47, 287, 240, + /* 50 */ 318, 242, 291, 277, 277, 318, 20, 296, 58, 248, + /* 60 */ 12, 13, 14, 277, 64, 0, 4, 253, 20, 308, + /* 70 */ 22, 260, 311, 312, 313, 314, 315, 316, 337, 318, + /* 80 */ 266, 81, 321, 4, 269, 82, 325, 326, 277, 275, + /* 90 */ 349, 276, 315, 287, 353, 47, 281, 20, 337, 293, + /* 100 */ 93, 315, 81, 103, 42, 43, 58, 330, 331, 332, + /* 110 */ 349, 334, 64, 113, 353, 329, 330, 331, 332, 112, + /* 120 */ 334, 114, 115, 116, 57, 60, 61, 62, 63, 81, + /* 130 */ 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + /* 140 */ 75, 76, 77, 78, 0, 12, 13, 14, 15, 16, + /* 150 */ 244, 103, 246, 247, 268, 279, 156, 20, 81, 269, + /* 160 */ 81, 113, 286, 287, 0, 21, 280, 277, 24, 25, + /* 170 */ 26, 27, 28, 29, 30, 31, 32, 143, 245, 179, + /* 180 */ 180, 248, 182, 183, 184, 185, 186, 187, 188, 189, + /* 190 */ 190, 191, 192, 193, 194, 195, 196, 12, 13, 14, + /* 200 */ 15, 16, 287, 313, 156, 290, 41, 21, 293, 209, + /* 210 */ 24, 25, 26, 27, 28, 29, 30, 31, 32, 35, + /* 220 */ 269, 57, 12, 13, 14, 15, 16, 179, 180, 278, /* 230 */ 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - /* 240 */ 192, 193, 194, 195, 331, 84, 240, 86, 87, 139, - /* 250 */ 89, 57, 156, 240, 93, 196, 197, 198, 199, 200, - /* 260 */ 201, 202, 203, 204, 205, 286, 20, 182, 289, 208, - /* 270 */ 160, 292, 320, 321, 178, 179, 115, 181, 182, 183, - /* 280 */ 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - /* 290 */ 194, 195, 12, 13, 0, 178, 290, 165, 166, 0, - /* 300 */ 20, 169, 22, 290, 247, 247, 196, 197, 198, 199, - /* 310 */ 200, 201, 202, 203, 204, 205, 259, 20, 24, 25, - /* 320 */ 26, 27, 28, 29, 30, 31, 32, 47, 208, 295, - /* 330 */ 93, 240, 208, 276, 217, 218, 219, 220, 221, 281, - /* 340 */ 12, 13, 14, 22, 64, 247, 260, 268, 20, 112, - /* 350 */ 22, 114, 115, 116, 268, 276, 57, 148, 247, 268, - /* 360 */ 227, 81, 240, 277, 276, 182, 247, 276, 47, 335, - /* 370 */ 259, 283, 268, 244, 276, 47, 247, 286, 259, 275, - /* 380 */ 20, 290, 348, 103, 280, 64, 352, 276, 12, 13, - /* 390 */ 311, 145, 64, 113, 240, 276, 20, 306, 22, 0, - /* 400 */ 309, 310, 311, 312, 313, 314, 81, 316, 2, 81, - /* 410 */ 319, 313, 290, 4, 323, 324, 325, 267, 12, 13, - /* 420 */ 14, 15, 16, 47, 103, 327, 328, 329, 330, 279, - /* 430 */ 332, 103, 252, 240, 113, 344, 156, 228, 60, 61, - /* 440 */ 64, 113, 145, 65, 290, 265, 68, 69, 1, 2, - /* 450 */ 72, 73, 74, 20, 274, 240, 57, 81, 178, 179, - /* 460 */ 143, 181, 182, 183, 184, 185, 186, 187, 188, 189, - /* 470 */ 190, 191, 192, 193, 194, 195, 20, 156, 260, 103, - /* 480 */ 155, 151, 157, 290, 156, 0, 268, 47, 208, 113, - /* 490 */ 12, 13, 14, 15, 16, 277, 268, 248, 249, 178, - /* 500 */ 179, 171, 172, 275, 64, 290, 178, 179, 280, 181, - /* 510 */ 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, - /* 520 */ 192, 193, 194, 195, 12, 13, 14, 15, 16, 82, - /* 530 */ 213, 214, 156, 208, 37, 94, 95, 96, 97, 98, - /* 540 */ 99, 100, 101, 102, 103, 104, 240, 106, 107, 108, - /* 550 */ 109, 110, 111, 240, 178, 179, 240, 181, 182, 183, - /* 560 */ 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - /* 570 */ 194, 195, 12, 13, 18, 75, 20, 0, 93, 247, - /* 580 */ 20, 247, 22, 27, 268, 307, 30, 261, 55, 14, - /* 590 */ 264, 259, 276, 259, 82, 20, 290, 112, 266, 114, - /* 600 */ 115, 116, 286, 290, 48, 268, 290, 47, 276, 331, - /* 610 */ 276, 240, 275, 80, 139, 286, 83, 280, 209, 119, - /* 620 */ 120, 292, 306, 145, 64, 309, 310, 311, 312, 313, - /* 630 */ 314, 315, 316, 317, 318, 160, 247, 60, 61, 62, - /* 640 */ 63, 81, 65, 66, 67, 68, 69, 70, 71, 72, - /* 650 */ 73, 74, 75, 76, 77, 78, 240, 14, 15, 16, - /* 660 */ 20, 290, 240, 103, 240, 276, 254, 255, 57, 254, - /* 670 */ 255, 196, 240, 113, 118, 0, 240, 121, 122, 123, - /* 680 */ 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, - /* 690 */ 134, 135, 136, 137, 138, 247, 285, 286, 247, 42, - /* 700 */ 43, 307, 313, 268, 207, 55, 290, 3, 240, 240, - /* 710 */ 259, 247, 290, 240, 290, 280, 156, 328, 329, 330, - /* 720 */ 20, 332, 290, 259, 276, 331, 290, 276, 268, 240, - /* 730 */ 266, 81, 256, 83, 258, 241, 270, 277, 178, 179, - /* 740 */ 276, 181, 182, 183, 184, 185, 186, 187, 188, 189, - /* 750 */ 190, 191, 192, 193, 194, 195, 290, 268, 290, 290, - /* 760 */ 240, 313, 0, 290, 93, 276, 47, 22, 93, 206, - /* 770 */ 207, 305, 306, 167, 168, 286, 328, 329, 330, 290, - /* 780 */ 332, 92, 316, 64, 295, 247, 115, 112, 268, 114, - /* 790 */ 115, 116, 47, 58, 41, 306, 276, 259, 309, 310, - /* 800 */ 311, 312, 313, 314, 41, 316, 286, 45, 319, 269, - /* 810 */ 290, 247, 323, 324, 276, 295, 85, 85, 270, 88, - /* 820 */ 88, 240, 182, 259, 335, 85, 306, 269, 88, 309, - /* 830 */ 310, 311, 312, 313, 314, 82, 316, 348, 290, 319, - /* 840 */ 276, 352, 44, 323, 324, 18, 12, 13, 247, 268, - /* 850 */ 23, 64, 21, 305, 306, 335, 22, 276, 208, 269, - /* 860 */ 259, 269, 35, 36, 316, 34, 39, 286, 348, 269, - /* 870 */ 247, 290, 352, 4, 247, 85, 41, 276, 88, 81, - /* 880 */ 41, 47, 259, 56, 240, 41, 259, 306, 19, 247, - /* 890 */ 309, 310, 311, 312, 313, 314, 47, 316, 64, 276, - /* 900 */ 319, 259, 33, 276, 323, 324, 325, 0, 81, 0, - /* 910 */ 0, 247, 268, 178, 45, 334, 269, 82, 276, 50, - /* 920 */ 276, 82, 257, 259, 55, 41, 82, 223, 47, 22, - /* 930 */ 286, 22, 22, 41, 290, 193, 194, 103, 41, 355, - /* 940 */ 276, 41, 1, 2, 117, 41, 41, 113, 299, 80, - /* 950 */ 306, 346, 83, 309, 310, 311, 312, 313, 314, 303, - /* 960 */ 316, 340, 113, 319, 268, 248, 82, 323, 324, 325, - /* 970 */ 240, 246, 41, 41, 82, 148, 149, 150, 334, 82, - /* 980 */ 153, 279, 82, 349, 308, 158, 82, 82, 225, 41, - /* 990 */ 156, 41, 41, 333, 113, 349, 20, 170, 268, 336, - /* 1000 */ 173, 240, 175, 176, 177, 247, 276, 349, 45, 41, - /* 1010 */ 304, 47, 178, 82, 82, 154, 286, 297, 41, 254, - /* 1020 */ 290, 247, 247, 189, 190, 191, 40, 284, 139, 268, - /* 1030 */ 82, 282, 82, 82, 282, 208, 306, 276, 247, 309, - /* 1040 */ 310, 311, 312, 313, 314, 20, 316, 286, 242, 319, - /* 1050 */ 82, 290, 242, 323, 324, 325, 295, 20, 247, 82, - /* 1060 */ 301, 252, 240, 286, 334, 252, 20, 306, 20, 296, - /* 1070 */ 309, 310, 311, 312, 313, 314, 294, 316, 252, 294, - /* 1080 */ 252, 276, 20, 287, 252, 252, 268, 276, 247, 252, - /* 1090 */ 268, 268, 242, 268, 64, 242, 335, 268, 276, 247, - /* 1100 */ 301, 163, 290, 300, 268, 250, 295, 286, 286, 348, - /* 1110 */ 268, 268, 290, 352, 268, 268, 268, 295, 250, 268, - /* 1120 */ 247, 250, 250, 20, 313, 240, 345, 287, 306, 216, - /* 1130 */ 215, 309, 310, 311, 312, 313, 314, 345, 316, 328, - /* 1140 */ 329, 330, 276, 332, 222, 147, 335, 290, 294, 291, - /* 1150 */ 291, 211, 290, 268, 290, 341, 240, 335, 342, 348, - /* 1160 */ 207, 276, 210, 352, 308, 20, 276, 307, 40, 339, - /* 1170 */ 348, 286, 226, 224, 352, 290, 338, 81, 291, 291, - /* 1180 */ 356, 290, 142, 229, 268, 290, 290, 288, 276, 326, - /* 1190 */ 287, 306, 276, 250, 309, 310, 311, 312, 313, 314, - /* 1200 */ 322, 316, 286, 264, 319, 250, 290, 81, 323, 324, - /* 1210 */ 240, 276, 247, 242, 272, 258, 351, 350, 298, 240, - /* 1220 */ 302, 351, 306, 250, 350, 309, 310, 311, 312, 313, - /* 1230 */ 314, 351, 316, 350, 262, 319, 262, 262, 268, 323, - /* 1240 */ 324, 251, 238, 0, 0, 72, 276, 268, 0, 47, - /* 1250 */ 174, 47, 47, 47, 174, 276, 286, 0, 47, 47, - /* 1260 */ 290, 174, 0, 47, 0, 286, 47, 0, 47, 290, - /* 1270 */ 0, 81, 160, 159, 113, 156, 306, 152, 151, 309, - /* 1280 */ 310, 311, 312, 313, 314, 306, 316, 0, 309, 310, - /* 1290 */ 311, 312, 313, 314, 0, 316, 19, 240, 319, 44, - /* 1300 */ 0, 0, 0, 324, 0, 0, 0, 0, 240, 0, - /* 1310 */ 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /* 1320 */ 0, 0, 45, 353, 354, 268, 0, 0, 51, 52, - /* 1330 */ 53, 54, 55, 276, 0, 40, 268, 0, 0, 0, - /* 1340 */ 0, 0, 0, 286, 276, 22, 0, 290, 40, 14, - /* 1350 */ 293, 0, 0, 0, 286, 0, 0, 80, 290, 37, - /* 1360 */ 83, 14, 37, 306, 0, 41, 309, 310, 311, 312, - /* 1370 */ 313, 314, 44, 316, 306, 44, 240, 309, 310, 311, - /* 1380 */ 312, 313, 314, 38, 316, 0, 37, 147, 0, 0, - /* 1390 */ 0, 37, 59, 116, 0, 0, 37, 47, 0, 37, - /* 1400 */ 0, 0, 45, 37, 268, 47, 45, 47, 45, 37, - /* 1410 */ 45, 0, 276, 0, 47, 347, 47, 88, 141, 0, - /* 1420 */ 0, 144, 286, 0, 22, 240, 290, 47, 47, 47, - /* 1430 */ 47, 47, 41, 90, 41, 0, 240, 22, 0, 162, - /* 1440 */ 22, 164, 306, 47, 47, 309, 310, 311, 312, 313, - /* 1450 */ 314, 48, 316, 268, 0, 22, 47, 0, 22, 0, - /* 1460 */ 0, 276, 22, 20, 268, 37, 22, 0, 47, 0, - /* 1470 */ 0, 286, 276, 0, 0, 290, 44, 145, 293, 206, - /* 1480 */ 81, 145, 286, 41, 212, 81, 290, 240, 41, 145, - /* 1490 */ 354, 306, 82, 41, 309, 310, 311, 312, 313, 314, - /* 1500 */ 44, 316, 306, 41, 82, 309, 310, 311, 312, 313, - /* 1510 */ 314, 81, 316, 142, 318, 268, 140, 2, 81, 41, - /* 1520 */ 41, 82, 81, 276, 33, 44, 240, 82, 82, 161, - /* 1530 */ 44, 82, 82, 286, 41, 47, 45, 290, 41, 240, - /* 1540 */ 293, 47, 51, 52, 53, 54, 55, 47, 47, 47, - /* 1550 */ 47, 178, 81, 306, 268, 44, 309, 310, 311, 312, - /* 1560 */ 313, 314, 276, 316, 82, 82, 81, 268, 81, 22, - /* 1570 */ 44, 80, 286, 81, 83, 276, 290, 82, 82, 293, - /* 1580 */ 81, 0, 212, 37, 22, 286, 212, 180, 240, 290, - /* 1590 */ 81, 143, 306, 81, 81, 309, 310, 311, 312, 313, - /* 1600 */ 314, 140, 316, 44, 44, 306, 82, 91, 309, 310, - /* 1610 */ 311, 312, 313, 314, 81, 316, 268, 240, 81, 81, - /* 1620 */ 92, 82, 47, 47, 276, 81, 47, 82, 105, 81, - /* 1630 */ 47, 140, 82, 142, 286, 144, 81, 146, 290, 47, - /* 1640 */ 82, 47, 81, 22, 81, 268, 82, 105, 105, 93, - /* 1650 */ 81, 81, 47, 276, 306, 164, 240, 309, 310, 311, - /* 1660 */ 312, 313, 314, 286, 316, 81, 113, 290, 22, 59, - /* 1670 */ 105, 58, 47, 64, 79, 41, 47, 47, 47, 47, - /* 1680 */ 47, 22, 47, 306, 268, 47, 309, 310, 311, 312, - /* 1690 */ 313, 314, 276, 316, 47, 47, 47, 47, 47, 47, - /* 1700 */ 47, 64, 286, 0, 47, 240, 290, 45, 37, 0, - /* 1710 */ 47, 45, 37, 0, 47, 45, 240, 37, 0, 47, - /* 1720 */ 37, 45, 306, 0, 47, 309, 310, 311, 312, 313, - /* 1730 */ 314, 0, 316, 268, 46, 0, 22, 21, 357, 357, - /* 1740 */ 21, 276, 22, 22, 268, 20, 357, 357, 357, 357, - /* 1750 */ 357, 286, 276, 357, 357, 290, 357, 357, 357, 357, - /* 1760 */ 357, 357, 286, 357, 357, 240, 290, 357, 357, 357, - /* 1770 */ 357, 306, 357, 357, 309, 310, 311, 312, 313, 314, - /* 1780 */ 240, 316, 306, 357, 357, 309, 310, 311, 312, 313, - /* 1790 */ 314, 357, 316, 268, 357, 357, 357, 240, 357, 357, - /* 1800 */ 357, 276, 357, 357, 357, 357, 357, 357, 268, 357, - /* 1810 */ 357, 286, 240, 357, 357, 290, 276, 357, 357, 357, - /* 1820 */ 357, 357, 357, 357, 357, 268, 286, 357, 357, 240, - /* 1830 */ 290, 306, 357, 276, 309, 310, 311, 312, 313, 314, - /* 1840 */ 268, 316, 357, 286, 357, 357, 306, 290, 276, 309, - /* 1850 */ 310, 311, 312, 313, 314, 357, 316, 268, 286, 357, - /* 1860 */ 357, 357, 290, 306, 357, 276, 309, 310, 311, 312, - /* 1870 */ 313, 314, 357, 316, 357, 286, 240, 357, 306, 290, - /* 1880 */ 357, 309, 310, 311, 312, 313, 314, 357, 316, 357, - /* 1890 */ 357, 357, 357, 357, 357, 306, 357, 357, 309, 310, - /* 1900 */ 311, 312, 313, 314, 268, 316, 357, 357, 240, 357, - /* 1910 */ 357, 357, 276, 357, 357, 357, 357, 357, 357, 357, - /* 1920 */ 357, 357, 286, 357, 357, 240, 290, 357, 357, 357, - /* 1930 */ 357, 357, 357, 357, 357, 357, 268, 357, 357, 357, - /* 1940 */ 357, 357, 306, 357, 276, 309, 310, 311, 312, 313, - /* 1950 */ 314, 357, 316, 268, 286, 357, 357, 240, 290, 357, - /* 1960 */ 357, 276, 357, 357, 357, 357, 357, 357, 357, 357, - /* 1970 */ 357, 286, 357, 357, 306, 290, 357, 309, 310, 311, - /* 1980 */ 312, 313, 314, 357, 316, 268, 357, 357, 357, 357, - /* 1990 */ 357, 306, 357, 276, 309, 310, 311, 312, 313, 314, - /* 2000 */ 357, 316, 357, 286, 357, 357, 240, 290, 357, 357, - /* 2010 */ 357, 357, 357, 357, 357, 357, 357, 357, 357, 247, - /* 2020 */ 357, 357, 357, 306, 357, 357, 309, 310, 311, 312, - /* 2030 */ 313, 314, 357, 316, 268, 357, 357, 357, 357, 357, - /* 2040 */ 357, 357, 276, 357, 357, 357, 357, 357, 276, 357, - /* 2050 */ 357, 357, 286, 357, 357, 357, 290, 357, 357, 357, - /* 2060 */ 357, 357, 357, 357, 357, 357, 357, 295, 357, 357, - /* 2070 */ 357, 357, 306, 357, 357, 309, 310, 311, 312, 313, - /* 2080 */ 314, 357, 316, 357, 357, 313, 357, 357, 357, 357, - /* 2090 */ 357, 357, 357, 357, 357, 357, 357, 357, 357, 357, - /* 2100 */ 328, 329, 330, 357, 332, 357, 357, 335, 357, 357, - /* 2110 */ 357, 357, 357, 357, 357, 357, 357, 357, 357, 357, - /* 2120 */ 348, 357, 357, 357, 352, 237, 237, 237, 237, 237, - /* 2130 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2140 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2150 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2160 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2170 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2180 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2190 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2200 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2210 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2220 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2230 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2240 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2250 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2260 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2270 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2280 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2290 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2300 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2310 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2320 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2330 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2340 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2350 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, - /* 2360 */ 237, 237, + /* 240 */ 192, 193, 194, 195, 196, 12, 13, 82, 214, 215, + /* 250 */ 238, 60, 61, 20, 241, 22, 65, 55, 262, 68, + /* 260 */ 69, 265, 75, 72, 73, 74, 12, 13, 84, 4, + /* 270 */ 86, 87, 47, 89, 20, 271, 22, 93, 145, 261, + /* 280 */ 47, 251, 269, 81, 19, 83, 209, 269, 209, 64, + /* 290 */ 277, 58, 82, 20, 264, 291, 278, 64, 33, 115, + /* 300 */ 287, 47, 272, 20, 291, 22, 119, 120, 296, 296, + /* 310 */ 45, 307, 308, 14, 81, 50, 20, 271, 64, 20, + /* 320 */ 55, 308, 318, 248, 311, 312, 313, 314, 315, 316, + /* 330 */ 2, 318, 49, 148, 321, 81, 103, 291, 325, 326, + /* 340 */ 12, 13, 14, 15, 16, 80, 113, 0, 83, 337, + /* 350 */ 337, 57, 277, 307, 308, 165, 166, 103, 241, 169, + /* 360 */ 92, 349, 349, 0, 318, 353, 353, 113, 0, 94, + /* 370 */ 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + /* 380 */ 241, 106, 107, 108, 109, 110, 111, 37, 81, 156, + /* 390 */ 315, 20, 24, 25, 26, 27, 28, 29, 30, 31, + /* 400 */ 32, 14, 15, 16, 57, 330, 331, 332, 291, 334, + /* 410 */ 156, 209, 179, 180, 229, 182, 183, 184, 185, 186, + /* 420 */ 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, + /* 430 */ 291, 64, 55, 179, 180, 300, 182, 183, 184, 185, + /* 440 */ 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, + /* 450 */ 196, 12, 13, 14, 2, 241, 93, 80, 296, 20, + /* 460 */ 83, 22, 93, 209, 12, 13, 14, 15, 16, 253, + /* 470 */ 0, 245, 12, 13, 248, 112, 241, 114, 115, 116, + /* 480 */ 20, 112, 22, 114, 115, 116, 47, 20, 248, 179, + /* 490 */ 248, 275, 22, 12, 13, 14, 15, 16, 337, 337, + /* 500 */ 260, 44, 260, 64, 269, 291, 270, 47, 81, 267, + /* 510 */ 349, 349, 277, 296, 353, 353, 209, 277, 241, 277, + /* 520 */ 81, 257, 287, 259, 64, 0, 291, 21, 218, 219, + /* 530 */ 220, 221, 222, 12, 13, 14, 15, 16, 81, 58, + /* 540 */ 34, 81, 103, 308, 255, 256, 311, 312, 313, 314, + /* 550 */ 315, 316, 113, 318, 337, 270, 321, 269, 208, 261, + /* 560 */ 325, 326, 327, 103, 276, 337, 349, 269, 291, 281, + /* 570 */ 353, 90, 14, 113, 339, 337, 278, 349, 20, 344, + /* 580 */ 345, 353, 155, 309, 157, 60, 61, 349, 248, 151, + /* 590 */ 65, 353, 269, 68, 69, 156, 269, 72, 73, 74, + /* 600 */ 260, 255, 256, 276, 281, 286, 287, 333, 281, 171, + /* 610 */ 172, 4, 145, 241, 42, 43, 156, 277, 179, 180, + /* 620 */ 139, 182, 183, 184, 185, 186, 187, 188, 189, 190, + /* 630 */ 191, 192, 193, 194, 195, 196, 209, 1, 2, 179, + /* 640 */ 180, 160, 182, 183, 184, 185, 186, 187, 188, 189, + /* 650 */ 190, 191, 192, 193, 194, 195, 196, 12, 13, 18, + /* 660 */ 241, 20, 248, 291, 248, 20, 241, 22, 27, 3, + /* 670 */ 241, 30, 241, 270, 260, 270, 260, 270, 197, 198, + /* 680 */ 199, 200, 201, 202, 203, 204, 205, 206, 269, 48, + /* 690 */ 248, 277, 47, 277, 14, 248, 277, 241, 277, 248, + /* 700 */ 20, 93, 260, 241, 261, 284, 287, 260, 20, 64, + /* 710 */ 291, 260, 269, 248, 167, 168, 291, 20, 82, 277, + /* 720 */ 291, 278, 291, 115, 277, 260, 81, 308, 277, 139, + /* 730 */ 311, 312, 313, 314, 315, 316, 22, 318, 241, 241, + /* 740 */ 321, 183, 277, 41, 325, 326, 327, 291, 103, 41, + /* 750 */ 160, 207, 208, 291, 241, 336, 241, 241, 113, 118, + /* 760 */ 241, 47, 121, 122, 123, 124, 125, 126, 127, 128, + /* 770 */ 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, + /* 780 */ 248, 18, 248, 248, 309, 242, 23, 197, 291, 291, + /* 790 */ 82, 309, 260, 241, 260, 260, 241, 41, 35, 36, + /* 800 */ 0, 156, 39, 241, 291, 58, 291, 291, 333, 277, + /* 810 */ 291, 277, 277, 85, 258, 333, 88, 210, 85, 56, + /* 820 */ 85, 88, 47, 88, 179, 180, 41, 182, 183, 184, + /* 830 */ 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + /* 840 */ 195, 196, 145, 291, 81, 45, 291, 12, 13, 14, + /* 850 */ 15, 16, 85, 291, 0, 88, 22, 0, 41, 1, + /* 860 */ 2, 194, 195, 183, 41, 41, 41, 82, 41, 41, + /* 870 */ 270, 183, 304, 41, 41, 81, 22, 41, 47, 22, + /* 880 */ 117, 47, 41, 356, 47, 91, 41, 347, 113, 41, + /* 890 */ 224, 341, 269, 58, 41, 247, 241, 280, 64, 82, + /* 900 */ 249, 64, 41, 310, 335, 82, 82, 82, 306, 82, + /* 910 */ 82, 148, 149, 150, 82, 82, 153, 82, 82, 20, + /* 920 */ 350, 158, 338, 82, 269, 90, 179, 82, 226, 350, + /* 930 */ 82, 350, 277, 170, 211, 82, 173, 103, 175, 176, + /* 940 */ 177, 178, 287, 82, 113, 45, 291, 113, 248, 305, + /* 950 */ 47, 255, 12, 13, 298, 154, 40, 248, 248, 285, + /* 960 */ 139, 283, 22, 308, 283, 248, 311, 312, 313, 314, + /* 970 */ 315, 316, 209, 318, 139, 241, 321, 20, 20, 243, + /* 980 */ 325, 326, 327, 243, 228, 302, 287, 47, 20, 295, + /* 990 */ 156, 253, 253, 20, 253, 160, 253, 277, 297, 295, + /* 1000 */ 345, 20, 253, 269, 64, 253, 288, 248, 253, 243, + /* 1010 */ 64, 277, 248, 179, 180, 269, 243, 269, 291, 269, + /* 1020 */ 269, 287, 269, 269, 302, 291, 269, 269, 241, 269, + /* 1030 */ 269, 269, 197, 198, 199, 200, 201, 202, 203, 204, + /* 1040 */ 205, 206, 308, 103, 251, 311, 312, 313, 314, 315, + /* 1050 */ 316, 163, 318, 113, 251, 321, 269, 287, 248, 325, + /* 1060 */ 326, 327, 251, 295, 277, 277, 301, 288, 217, 251, + /* 1070 */ 336, 20, 346, 292, 287, 216, 346, 291, 291, 241, + /* 1080 */ 291, 310, 292, 291, 342, 223, 147, 343, 212, 306, + /* 1090 */ 241, 208, 277, 20, 40, 308, 156, 227, 311, 312, + /* 1100 */ 313, 314, 315, 316, 309, 318, 328, 269, 321, 340, + /* 1110 */ 225, 230, 325, 326, 327, 277, 81, 291, 269, 179, + /* 1120 */ 291, 324, 292, 336, 352, 287, 277, 351, 351, 291, + /* 1130 */ 190, 191, 192, 351, 296, 292, 287, 357, 241, 142, + /* 1140 */ 291, 291, 277, 352, 289, 296, 308, 251, 352, 311, + /* 1150 */ 312, 313, 314, 315, 316, 288, 318, 308, 265, 251, + /* 1160 */ 311, 312, 313, 314, 315, 316, 269, 318, 277, 81, + /* 1170 */ 259, 273, 248, 251, 277, 337, 243, 299, 241, 252, + /* 1180 */ 303, 263, 263, 263, 287, 0, 337, 349, 291, 239, + /* 1190 */ 0, 353, 40, 0, 72, 0, 47, 174, 349, 174, + /* 1200 */ 47, 47, 353, 47, 0, 308, 269, 47, 311, 312, + /* 1210 */ 313, 314, 315, 316, 277, 318, 47, 174, 321, 0, + /* 1220 */ 33, 47, 325, 326, 287, 0, 47, 0, 291, 47, + /* 1230 */ 0, 81, 45, 113, 160, 159, 156, 241, 51, 52, + /* 1240 */ 53, 54, 55, 0, 0, 308, 152, 151, 311, 312, + /* 1250 */ 313, 314, 315, 316, 0, 318, 0, 44, 321, 22, + /* 1260 */ 40, 0, 325, 326, 0, 269, 0, 80, 0, 0, + /* 1270 */ 83, 0, 0, 277, 0, 0, 0, 0, 0, 0, + /* 1280 */ 0, 0, 0, 287, 0, 0, 0, 291, 0, 241, + /* 1290 */ 0, 0, 0, 0, 0, 0, 0, 0, 241, 0, + /* 1300 */ 0, 41, 40, 14, 308, 0, 0, 311, 312, 313, + /* 1310 */ 314, 315, 316, 317, 318, 319, 320, 269, 14, 0, + /* 1320 */ 37, 0, 147, 0, 37, 277, 269, 140, 0, 142, + /* 1330 */ 0, 144, 59, 146, 277, 287, 0, 0, 0, 291, + /* 1340 */ 38, 0, 44, 0, 287, 44, 37, 37, 291, 37, + /* 1350 */ 47, 164, 45, 37, 241, 47, 308, 0, 45, 311, + /* 1360 */ 312, 313, 314, 315, 316, 308, 318, 47, 311, 312, + /* 1370 */ 313, 314, 315, 316, 45, 318, 37, 45, 321, 19, + /* 1380 */ 47, 37, 269, 326, 0, 0, 0, 47, 22, 0, + /* 1390 */ 277, 41, 47, 33, 47, 47, 47, 41, 47, 47, + /* 1400 */ 287, 47, 354, 355, 291, 45, 88, 294, 241, 0, + /* 1410 */ 22, 51, 52, 53, 54, 55, 90, 241, 0, 22, + /* 1420 */ 48, 308, 0, 22, 311, 312, 313, 314, 315, 316, + /* 1430 */ 47, 318, 0, 22, 0, 22, 269, 20, 0, 47, + /* 1440 */ 80, 0, 22, 83, 277, 269, 0, 0, 0, 0, + /* 1450 */ 81, 37, 213, 277, 287, 82, 41, 41, 291, 145, + /* 1460 */ 145, 81, 145, 287, 142, 41, 41, 291, 213, 82, + /* 1470 */ 241, 140, 81, 44, 81, 308, 116, 82, 311, 312, + /* 1480 */ 313, 314, 315, 316, 308, 318, 161, 311, 312, 313, + /* 1490 */ 314, 315, 316, 81, 318, 44, 41, 44, 269, 41, + /* 1500 */ 241, 141, 41, 82, 144, 82, 277, 44, 82, 47, + /* 1510 */ 82, 47, 47, 47, 47, 348, 287, 47, 2, 41, + /* 1520 */ 291, 82, 162, 294, 164, 179, 81, 44, 269, 44, + /* 1530 */ 22, 355, 82, 81, 181, 81, 277, 308, 81, 143, + /* 1540 */ 311, 312, 313, 314, 315, 316, 287, 318, 213, 207, + /* 1550 */ 291, 82, 241, 82, 81, 81, 81, 0, 81, 37, + /* 1560 */ 81, 44, 44, 91, 82, 81, 22, 308, 241, 81, + /* 1570 */ 311, 312, 313, 314, 315, 316, 140, 318, 92, 320, + /* 1580 */ 269, 82, 47, 47, 81, 47, 82, 81, 277, 82, + /* 1590 */ 47, 81, 47, 82, 81, 47, 269, 82, 287, 105, + /* 1600 */ 81, 105, 291, 105, 277, 294, 105, 22, 241, 81, + /* 1610 */ 81, 47, 81, 93, 287, 113, 22, 59, 291, 308, + /* 1620 */ 241, 294, 311, 312, 313, 314, 315, 316, 58, 318, + /* 1630 */ 47, 64, 41, 79, 47, 308, 269, 47, 311, 312, + /* 1640 */ 313, 314, 315, 316, 277, 318, 47, 47, 269, 47, + /* 1650 */ 22, 47, 47, 64, 287, 47, 277, 47, 291, 47, + /* 1660 */ 47, 47, 47, 47, 0, 47, 287, 45, 241, 37, + /* 1670 */ 291, 0, 47, 37, 45, 308, 0, 47, 311, 312, + /* 1680 */ 313, 314, 315, 316, 45, 318, 37, 308, 0, 47, + /* 1690 */ 311, 312, 313, 314, 315, 316, 269, 318, 45, 37, + /* 1700 */ 241, 47, 46, 0, 277, 0, 0, 22, 21, 358, + /* 1710 */ 358, 22, 22, 21, 287, 20, 358, 358, 291, 358, + /* 1720 */ 358, 358, 358, 358, 358, 358, 358, 358, 269, 358, + /* 1730 */ 241, 358, 358, 358, 358, 308, 277, 358, 311, 312, + /* 1740 */ 313, 314, 315, 316, 358, 318, 287, 358, 358, 358, + /* 1750 */ 291, 358, 358, 358, 358, 358, 358, 358, 269, 358, + /* 1760 */ 358, 358, 241, 358, 358, 358, 277, 308, 358, 358, + /* 1770 */ 311, 312, 313, 314, 315, 316, 287, 318, 241, 358, + /* 1780 */ 291, 358, 358, 358, 358, 358, 358, 358, 358, 358, + /* 1790 */ 269, 358, 358, 358, 241, 358, 358, 308, 277, 358, + /* 1800 */ 311, 312, 313, 314, 315, 316, 269, 318, 287, 358, + /* 1810 */ 241, 358, 291, 358, 277, 358, 358, 358, 358, 358, + /* 1820 */ 358, 358, 269, 358, 287, 358, 358, 358, 291, 308, + /* 1830 */ 277, 358, 311, 312, 313, 314, 315, 316, 269, 318, + /* 1840 */ 287, 358, 358, 358, 291, 308, 277, 358, 311, 312, + /* 1850 */ 313, 314, 315, 316, 358, 318, 287, 358, 358, 358, + /* 1860 */ 291, 308, 241, 358, 311, 312, 313, 314, 315, 316, + /* 1870 */ 358, 318, 358, 358, 358, 358, 358, 308, 241, 358, + /* 1880 */ 311, 312, 313, 314, 315, 316, 358, 318, 358, 358, + /* 1890 */ 269, 358, 358, 358, 358, 358, 358, 358, 277, 358, + /* 1900 */ 358, 358, 358, 358, 358, 358, 269, 358, 287, 358, + /* 1910 */ 241, 358, 291, 358, 277, 358, 358, 358, 358, 358, + /* 1920 */ 358, 358, 358, 358, 287, 358, 358, 358, 291, 308, + /* 1930 */ 358, 358, 311, 312, 313, 314, 315, 316, 269, 318, + /* 1940 */ 358, 358, 241, 358, 358, 308, 277, 358, 311, 312, + /* 1950 */ 313, 314, 315, 316, 358, 318, 287, 358, 358, 358, + /* 1960 */ 291, 358, 358, 358, 358, 358, 358, 358, 358, 358, + /* 1970 */ 269, 358, 241, 358, 358, 358, 358, 308, 277, 358, + /* 1980 */ 311, 312, 313, 314, 315, 316, 358, 318, 287, 358, + /* 1990 */ 358, 358, 291, 358, 358, 358, 358, 358, 358, 358, + /* 2000 */ 269, 358, 358, 358, 241, 358, 358, 358, 277, 308, + /* 2010 */ 358, 358, 311, 312, 313, 314, 315, 316, 287, 318, + /* 2020 */ 241, 358, 291, 358, 358, 358, 358, 358, 358, 358, + /* 2030 */ 358, 358, 269, 358, 358, 358, 358, 358, 358, 308, + /* 2040 */ 277, 358, 311, 312, 313, 314, 315, 316, 269, 318, + /* 2050 */ 287, 248, 358, 358, 291, 358, 277, 358, 358, 248, + /* 2060 */ 358, 358, 358, 358, 358, 358, 287, 358, 358, 358, + /* 2070 */ 291, 308, 358, 358, 311, 312, 313, 314, 315, 316, + /* 2080 */ 277, 318, 358, 358, 358, 358, 358, 308, 277, 358, + /* 2090 */ 311, 312, 313, 314, 315, 316, 358, 318, 358, 296, + /* 2100 */ 358, 358, 358, 358, 358, 358, 358, 296, 358, 358, + /* 2110 */ 358, 358, 358, 358, 358, 358, 358, 358, 315, 358, + /* 2120 */ 358, 358, 358, 358, 358, 358, 315, 358, 358, 358, + /* 2130 */ 358, 358, 358, 330, 331, 332, 358, 334, 358, 358, + /* 2140 */ 337, 330, 331, 332, 358, 334, 358, 358, 337, 358, + /* 2150 */ 358, 358, 349, 358, 358, 358, 353, 358, 358, 358, + /* 2160 */ 349, 358, 358, 358, 353, }; -#define YY_SHIFT_COUNT (611) +#define YY_SHIFT_COUNT (615) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1735) +#define YY_SHIFT_MAX (1706) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 827, 0, 0, 48, 96, 96, 96, 96, 280, 280, - /* 10 */ 96, 96, 328, 376, 560, 376, 376, 376, 376, 376, - /* 20 */ 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, - /* 30 */ 376, 376, 376, 376, 376, 376, 376, 376, 120, 120, - /* 40 */ 23, 23, 23, 834, 834, 834, 834, 325, 650, 124, - /* 50 */ 30, 30, 42, 42, 61, 124, 124, 30, 30, 30, - /* 60 */ 30, 30, 30, 194, 30, 30, 360, 433, 456, 360, - /* 70 */ 30, 30, 360, 30, 360, 360, 456, 360, 30, 611, - /* 80 */ 556, 59, 110, 110, 186, 378, 321, 321, 321, 321, - /* 90 */ 321, 321, 321, 321, 321, 321, 321, 321, 321, 321, - /* 100 */ 321, 321, 321, 321, 321, 161, 153, 575, 575, 299, - /* 110 */ 440, 246, 246, 246, 399, 440, 700, 456, 360, 360, - /* 120 */ 456, 689, 787, 441, 441, 441, 441, 441, 441, 441, - /* 130 */ 1277, 107, 97, 209, 117, 132, 317, 85, 183, 657, - /* 140 */ 745, 671, 297, 563, 497, 563, 704, 704, 704, 409, - /* 150 */ 640, 976, 963, 964, 861, 976, 976, 986, 889, 889, - /* 160 */ 976, 1025, 1025, 1037, 194, 456, 194, 1046, 1048, 194, - /* 170 */ 1046, 194, 700, 1062, 194, 194, 976, 194, 1025, 360, - /* 180 */ 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, - /* 190 */ 976, 1025, 1030, 1037, 611, 938, 456, 611, 976, 1046, - /* 200 */ 611, 700, 1062, 611, 1103, 913, 915, 1030, 913, 915, - /* 210 */ 1030, 1030, 360, 922, 998, 940, 952, 953, 700, 1145, - /* 220 */ 1128, 946, 949, 954, 946, 949, 946, 949, 1096, 915, - /* 230 */ 1030, 1030, 915, 1030, 1040, 700, 1062, 611, 689, 611, - /* 240 */ 700, 1126, 787, 976, 611, 1025, 2125, 2125, 2125, 2125, - /* 250 */ 2125, 2125, 2125, 577, 1491, 294, 869, 64, 14, 406, - /* 260 */ 478, 512, 485, 675, 21, 21, 21, 21, 21, 21, - /* 270 */ 21, 21, 237, 330, 533, 500, 447, 475, 643, 643, - /* 280 */ 643, 643, 762, 753, 731, 732, 740, 790, 907, 909, - /* 290 */ 910, 831, 606, 835, 839, 844, 941, 742, 763, 133, - /* 300 */ 884, 735, 892, 798, 900, 904, 905, 931, 932, 849, - /* 310 */ 881, 897, 948, 950, 951, 968, 977, 1, 719, 1243, - /* 320 */ 1244, 1173, 1248, 1202, 1076, 1204, 1205, 1206, 1080, 1257, - /* 330 */ 1211, 1212, 1087, 1262, 1216, 1264, 1219, 1267, 1221, 1270, - /* 340 */ 1190, 1112, 1114, 1161, 1119, 1287, 1294, 1125, 1127, 1304, - /* 350 */ 1305, 1255, 1300, 1301, 1302, 1306, 1307, 1309, 1311, 1312, - /* 360 */ 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321, 1326, - /* 370 */ 1295, 1327, 1334, 1337, 1338, 1339, 1340, 1323, 1341, 1342, - /* 380 */ 1346, 1351, 1352, 1353, 1308, 1322, 1324, 1335, 1328, 1347, - /* 390 */ 1331, 1355, 1345, 1325, 1356, 1364, 1385, 1349, 1240, 1388, - /* 400 */ 1389, 1354, 1390, 1333, 1394, 1395, 1350, 1357, 1359, 1398, - /* 410 */ 1358, 1361, 1362, 1400, 1360, 1363, 1366, 1401, 1367, 1365, - /* 420 */ 1372, 1411, 1413, 1419, 1420, 1343, 1329, 1369, 1402, 1423, - /* 430 */ 1380, 1381, 1382, 1383, 1391, 1393, 1384, 1396, 1397, 1435, - /* 440 */ 1415, 1438, 1418, 1403, 1454, 1433, 1409, 1457, 1436, 1459, - /* 450 */ 1440, 1443, 1460, 1332, 1421, 1469, 1368, 1444, 1336, 1371, - /* 460 */ 1467, 1470, 1473, 1344, 1474, 1399, 1428, 1376, 1442, 1447, - /* 470 */ 1272, 1410, 1452, 1422, 1404, 1430, 1437, 1439, 1462, 1432, - /* 480 */ 1456, 1441, 1478, 1370, 1445, 1446, 1481, 1273, 1479, 1486, - /* 490 */ 1449, 1493, 1374, 1450, 1488, 1494, 1500, 1501, 1502, 1503, - /* 500 */ 1450, 1515, 1373, 1497, 1482, 1471, 1483, 1511, 1485, 1487, - /* 510 */ 1526, 1547, 1407, 1492, 1495, 1496, 1499, 1509, 1448, 1512, - /* 520 */ 1581, 1546, 1461, 1513, 1516, 1559, 1560, 1533, 1524, 1537, - /* 530 */ 1562, 1538, 1528, 1539, 1575, 1576, 1544, 1545, 1579, 1548, - /* 540 */ 1550, 1583, 1555, 1558, 1592, 1561, 1564, 1594, 1563, 1523, - /* 550 */ 1542, 1543, 1565, 1621, 1556, 1569, 1570, 1605, 1584, 1553, - /* 560 */ 1646, 1610, 1613, 1625, 1609, 1595, 1634, 1629, 1630, 1631, - /* 570 */ 1632, 1633, 1659, 1635, 1638, 1637, 1391, 1647, 1393, 1648, - /* 580 */ 1649, 1650, 1651, 1652, 1653, 1703, 1657, 1662, 1671, 1709, - /* 590 */ 1663, 1666, 1675, 1713, 1667, 1670, 1680, 1718, 1672, 1676, - /* 600 */ 1683, 1723, 1677, 1688, 1731, 1735, 1714, 1716, 1720, 1721, - /* 610 */ 1719, 1725, + /* 0 */ 763, 0, 0, 48, 233, 233, 233, 233, 254, 254, + /* 10 */ 233, 233, 439, 460, 645, 460, 460, 460, 460, 460, + /* 20 */ 460, 460, 460, 460, 460, 460, 460, 460, 460, 460, + /* 30 */ 460, 460, 460, 460, 460, 460, 460, 460, 77, 77, + /* 40 */ 21, 21, 21, 940, 940, 940, 940, 427, 202, 307, + /* 50 */ 36, 36, 62, 62, 79, 307, 307, 36, 36, 36, + /* 60 */ 36, 36, 36, 36, 67, 36, 36, 137, 273, 296, + /* 70 */ 137, 36, 36, 137, 36, 137, 137, 296, 137, 36, + /* 80 */ 294, 641, 835, 481, 481, 186, 191, 834, 834, 834, + /* 90 */ 834, 834, 834, 834, 834, 834, 834, 834, 834, 834, + /* 100 */ 834, 834, 834, 834, 834, 834, 184, 283, 299, 299, + /* 110 */ 164, 225, 467, 467, 467, 347, 225, 371, 296, 137, + /* 120 */ 137, 296, 268, 367, 275, 275, 275, 275, 275, 275, + /* 130 */ 275, 1360, 144, 525, 185, 310, 190, 34, 558, 680, + /* 140 */ 572, 714, 608, 697, 544, 350, 544, 666, 666, 666, + /* 150 */ 607, 688, 723, 899, 900, 903, 801, 899, 899, 916, + /* 160 */ 821, 821, 899, 957, 957, 958, 67, 296, 67, 968, + /* 170 */ 973, 67, 968, 67, 371, 981, 67, 67, 899, 67, + /* 180 */ 957, 137, 137, 137, 137, 137, 137, 137, 137, 137, + /* 190 */ 137, 137, 899, 957, 946, 958, 294, 888, 296, 294, + /* 200 */ 899, 968, 294, 371, 981, 294, 1051, 851, 859, 946, + /* 210 */ 851, 859, 946, 946, 137, 862, 939, 876, 723, 883, + /* 220 */ 371, 1073, 1054, 870, 885, 881, 870, 885, 870, 885, + /* 230 */ 1035, 859, 946, 946, 859, 946, 997, 371, 981, 294, + /* 240 */ 268, 294, 371, 1088, 367, 899, 294, 957, 2165, 2165, + /* 250 */ 2165, 2165, 2165, 2165, 2165, 65, 1187, 368, 265, 3, + /* 260 */ 328, 452, 133, 210, 7, 363, 521, 521, 521, 521, + /* 270 */ 521, 521, 521, 521, 369, 438, 377, 187, 636, 590, + /* 280 */ 387, 387, 387, 387, 800, 165, 728, 733, 735, 767, + /* 290 */ 470, 854, 857, 506, 547, 708, 785, 817, 858, 667, + /* 300 */ 702, 756, 824, 747, 825, 457, 827, 828, 832, 833, + /* 310 */ 836, 775, 831, 823, 841, 845, 848, 853, 861, 794, + /* 320 */ 837, 1185, 1190, 1152, 1193, 1122, 1195, 1149, 1023, 1153, + /* 330 */ 1154, 1156, 1025, 1204, 1160, 1169, 1043, 1219, 1174, 1225, + /* 340 */ 1179, 1227, 1182, 1230, 1150, 1074, 1076, 1120, 1080, 1243, + /* 350 */ 1244, 1094, 1096, 1254, 1256, 1213, 1264, 1266, 1268, 1269, + /* 360 */ 1271, 1272, 1274, 1275, 1276, 1277, 1278, 1279, 1280, 1281, + /* 370 */ 1282, 1284, 1285, 1286, 1220, 1261, 1288, 1290, 1291, 1292, + /* 380 */ 1293, 1237, 1294, 1295, 1296, 1297, 1299, 1300, 1262, 1283, + /* 390 */ 1260, 1289, 1298, 1304, 1301, 1305, 1302, 1287, 1306, 1319, + /* 400 */ 1321, 1309, 1175, 1323, 1328, 1310, 1330, 1273, 1336, 1337, + /* 410 */ 1303, 1307, 1312, 1338, 1308, 1313, 1316, 1341, 1320, 1329, + /* 420 */ 1339, 1343, 1333, 1332, 1344, 1357, 1384, 1385, 1386, 1326, + /* 430 */ 1318, 1340, 1366, 1389, 1345, 1347, 1348, 1349, 1350, 1356, + /* 440 */ 1351, 1352, 1354, 1409, 1388, 1418, 1397, 1372, 1422, 1401, + /* 450 */ 1383, 1432, 1411, 1434, 1413, 1417, 1438, 1314, 1392, 1441, + /* 460 */ 1325, 1420, 1315, 1322, 1446, 1447, 1448, 1317, 1449, 1369, + /* 470 */ 1414, 1331, 1415, 1416, 1239, 1373, 1424, 1387, 1380, 1391, + /* 480 */ 1393, 1395, 1425, 1429, 1451, 1412, 1455, 1255, 1421, 1423, + /* 490 */ 1453, 1342, 1458, 1463, 1426, 1461, 1335, 1428, 1462, 1464, + /* 500 */ 1465, 1466, 1467, 1470, 1428, 1516, 1346, 1478, 1439, 1445, + /* 510 */ 1450, 1483, 1452, 1454, 1485, 1508, 1353, 1457, 1469, 1471, + /* 520 */ 1473, 1474, 1396, 1475, 1557, 1522, 1436, 1477, 1472, 1517, + /* 530 */ 1518, 1479, 1482, 1484, 1544, 1488, 1486, 1499, 1535, 1536, + /* 540 */ 1503, 1504, 1538, 1506, 1507, 1543, 1510, 1511, 1545, 1513, + /* 550 */ 1515, 1548, 1519, 1494, 1496, 1498, 1501, 1585, 1520, 1528, + /* 560 */ 1529, 1564, 1531, 1502, 1594, 1558, 1570, 1583, 1567, 1554, + /* 570 */ 1591, 1587, 1590, 1599, 1600, 1602, 1628, 1604, 1605, 1589, + /* 580 */ 1350, 1608, 1356, 1610, 1612, 1613, 1614, 1615, 1616, 1664, + /* 590 */ 1618, 1622, 1632, 1671, 1625, 1629, 1636, 1676, 1630, 1639, + /* 600 */ 1649, 1688, 1642, 1653, 1662, 1705, 1654, 1656, 1703, 1706, + /* 610 */ 1685, 1687, 1689, 1690, 1692, 1695, }; -#define YY_REDUCE_COUNT (252) -#define YY_REDUCE_MIN (-334) -#define YY_REDUCE_MAX (1772) +#define YY_REDUCE_COUNT (254) +#define YY_REDUCE_MIN (-318) +#define YY_REDUCE_MAX (1811) static const short yy_reduce_ofst[] = { - /* 0 */ -185, 489, 520, -223, 91, 581, 644, 730, 761, 822, - /* 10 */ 885, 916, 316, 970, 979, 1057, 1068, 1136, 1185, 1196, - /* 20 */ 1247, 1286, 1299, 1348, 1377, 1416, 1465, 1476, 1525, 1540, - /* 30 */ 1557, 1572, 1589, 1636, 1668, 1685, 1717, 1766, 811, 1772, - /* 40 */ 98, 389, 448, -267, -265, 466, 548, -293, -176, 34, - /* 50 */ 332, 464, -236, -189, -334, -304, -237, 57, 111, 119, - /* 60 */ 334, 451, 538, 180, 564, 601, 104, 79, -21, -113, - /* 70 */ 623, 627, 228, 642, 86, 337, -133, 218, 664, -239, - /* 80 */ 58, -48, -48, -48, -220, -253, -146, -110, 6, 13, - /* 90 */ 122, 154, 193, 215, 306, 313, 371, 416, 422, 424, - /* 100 */ 432, 436, 468, 469, 473, 150, 249, -178, 129, -46, - /* 110 */ 412, -87, 278, 394, -123, 415, 88, 329, 460, 435, - /* 120 */ 411, 326, 476, -254, 540, 558, 590, 592, 600, 647, - /* 130 */ 649, 494, 665, 584, 605, 656, 621, 696, 696, 725, - /* 140 */ 717, 702, 676, 660, 660, 660, 634, 646, 658, 663, - /* 150 */ 696, 758, 706, 765, 720, 774, 775, 743, 749, 752, - /* 160 */ 791, 806, 810, 759, 809, 777, 813, 782, 773, 826, - /* 170 */ 785, 828, 805, 796, 832, 833, 841, 837, 850, 818, - /* 180 */ 823, 825, 829, 836, 842, 843, 846, 847, 848, 851, - /* 190 */ 852, 853, 812, 799, 855, 803, 821, 868, 873, 854, - /* 200 */ 871, 866, 840, 872, 856, 781, 858, 857, 792, 859, - /* 210 */ 862, 864, 696, 816, 814, 830, 838, 660, 890, 860, - /* 220 */ 863, 865, 867, 824, 870, 874, 880, 883, 878, 887, - /* 230 */ 891, 895, 888, 896, 899, 912, 903, 943, 939, 955, - /* 240 */ 935, 942, 957, 965, 973, 971, 920, 918, 972, 974, - /* 250 */ 975, 990, 1004, + /* 0 */ 12, -239, 13, 235, 655, 419, 734, 787, 838, 849, + /* 10 */ 897, 937, 996, 1048, 1057, 1113, 1167, 1176, 1229, 1259, + /* 20 */ 1311, 1327, 1367, 1379, 1427, 1459, 1489, 1521, 1537, 1553, + /* 30 */ 1569, 1621, 1637, 1669, 1701, 1731, 1763, 1779, 1803, 1811, + /* 40 */ -214, -223, 75, -268, -263, 4, 46, -259, 162, 217, + /* 50 */ -234, 242, -215, -94, 161, 228, 238, -224, -189, 240, + /* 60 */ 340, 414, 416, 442, -186, 447, 451, -185, -110, -85, + /* 70 */ 18, 465, 532, 288, 534, 298, 327, -124, 443, 535, + /* 80 */ 30, -247, -318, -318, -318, -191, -212, 117, 139, 214, + /* 90 */ 277, 372, 425, 429, 431, 456, 462, 497, 498, 513, + /* 100 */ 515, 516, 519, 552, 555, 562, -114, -240, -67, 226, + /* 110 */ 216, 289, 274, 475, 482, -251, 346, 421, -194, -49, + /* 120 */ 323, 319, -4, 264, -243, 236, 285, 403, 405, 407, + /* 130 */ 600, 135, 543, 556, 527, 540, 568, 550, 623, 623, + /* 140 */ 648, 651, 617, 593, 569, 569, 569, 570, 579, 581, + /* 150 */ 584, 623, 602, 700, 644, 696, 656, 709, 710, 674, + /* 160 */ 678, 681, 717, 736, 740, 683, 738, 699, 739, 694, + /* 170 */ 701, 741, 704, 743, 720, 718, 749, 752, 759, 755, + /* 180 */ 766, 746, 748, 750, 751, 753, 754, 757, 758, 760, + /* 190 */ 761, 762, 764, 773, 727, 722, 793, 765, 770, 803, + /* 200 */ 810, 768, 811, 788, 779, 818, 771, 726, 781, 786, + /* 210 */ 730, 790, 789, 792, 623, 744, 742, 769, 783, 569, + /* 220 */ 815, 795, 778, 772, 776, 780, 791, 777, 796, 782, + /* 230 */ 797, 830, 826, 829, 843, 850, 855, 865, 867, 896, + /* 240 */ 893, 908, 891, 898, 911, 924, 922, 933, 878, 877, + /* 250 */ 918, 919, 920, 927, 950, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 10 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 20 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 30 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 40 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 50 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 60 */ 1349, 1349, 1349, 1418, 1349, 1349, 1349, 1349, 1349, 1349, - /* 70 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1416, - /* 80 */ 1556, 1349, 1720, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 90 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 100 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1418, - /* 110 */ 1349, 1731, 1731, 1731, 1416, 1349, 1349, 1349, 1349, 1349, - /* 120 */ 1349, 1512, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 130 */ 1592, 1349, 1349, 1797, 1349, 1598, 1755, 1349, 1349, 1349, - /* 140 */ 1349, 1465, 1747, 1723, 1737, 1724, 1782, 1782, 1782, 1740, - /* 150 */ 1349, 1349, 1349, 1349, 1584, 1349, 1349, 1561, 1558, 1558, - /* 160 */ 1349, 1349, 1349, 1349, 1418, 1349, 1418, 1349, 1349, 1418, - /* 170 */ 1349, 1418, 1349, 1349, 1418, 1418, 1349, 1418, 1349, 1349, - /* 180 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 190 */ 1349, 1349, 1349, 1349, 1416, 1594, 1349, 1416, 1349, 1349, - /* 200 */ 1416, 1349, 1349, 1416, 1349, 1762, 1760, 1349, 1762, 1760, - /* 210 */ 1349, 1349, 1349, 1774, 1770, 1753, 1751, 1737, 1349, 1349, - /* 220 */ 1349, 1788, 1784, 1800, 1788, 1784, 1788, 1784, 1349, 1760, - /* 230 */ 1349, 1349, 1760, 1349, 1569, 1349, 1349, 1416, 1349, 1416, - /* 240 */ 1349, 1481, 1349, 1349, 1416, 1349, 1586, 1600, 1515, 1515, - /* 250 */ 1515, 1419, 1354, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 260 */ 1349, 1349, 1349, 1349, 1664, 1773, 1772, 1696, 1695, 1694, - /* 270 */ 1692, 1663, 1477, 1349, 1349, 1349, 1349, 1349, 1657, 1658, - /* 280 */ 1656, 1655, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 290 */ 1349, 1349, 1349, 1349, 1349, 1349, 1721, 1349, 1785, 1789, - /* 300 */ 1349, 1349, 1349, 1640, 1349, 1349, 1349, 1349, 1349, 1349, - /* 310 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 320 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 330 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 340 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 350 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 360 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 370 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 380 */ 1349, 1349, 1349, 1349, 1349, 1349, 1383, 1349, 1349, 1349, - /* 390 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 400 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 410 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 420 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 430 */ 1349, 1349, 1349, 1349, 1446, 1445, 1349, 1349, 1349, 1349, - /* 440 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 450 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 460 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1744, 1754, - /* 470 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 480 */ 1640, 1349, 1771, 1349, 1730, 1726, 1349, 1349, 1722, 1349, - /* 490 */ 1349, 1783, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 500 */ 1349, 1716, 1349, 1689, 1349, 1349, 1349, 1349, 1349, 1349, - /* 510 */ 1349, 1349, 1651, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 520 */ 1349, 1349, 1349, 1349, 1349, 1639, 1349, 1680, 1349, 1349, - /* 530 */ 1349, 1349, 1349, 1349, 1349, 1349, 1509, 1349, 1349, 1349, - /* 540 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1494, - /* 550 */ 1492, 1491, 1490, 1349, 1487, 1349, 1349, 1349, 1349, 1349, - /* 560 */ 1349, 1349, 1349, 1349, 1349, 1349, 1438, 1349, 1349, 1349, - /* 570 */ 1349, 1349, 1349, 1349, 1349, 1349, 1429, 1349, 1428, 1349, - /* 580 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 590 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 600 */ 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, 1349, - /* 610 */ 1349, 1349, + /* 0 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 10 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 20 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 30 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 40 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 50 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 60 */ 1354, 1354, 1354, 1354, 1423, 1354, 1354, 1354, 1354, 1354, + /* 70 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 80 */ 1421, 1561, 1354, 1726, 1354, 1354, 1354, 1354, 1354, 1354, + /* 90 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 100 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 110 */ 1423, 1354, 1737, 1737, 1737, 1421, 1354, 1354, 1354, 1354, + /* 120 */ 1354, 1354, 1517, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 130 */ 1354, 1597, 1354, 1354, 1803, 1354, 1603, 1761, 1354, 1354, + /* 140 */ 1354, 1354, 1470, 1753, 1729, 1743, 1730, 1788, 1788, 1788, + /* 150 */ 1746, 1354, 1757, 1354, 1354, 1354, 1589, 1354, 1354, 1566, + /* 160 */ 1563, 1563, 1354, 1354, 1354, 1354, 1423, 1354, 1423, 1354, + /* 170 */ 1354, 1423, 1354, 1423, 1354, 1354, 1423, 1423, 1354, 1423, + /* 180 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 190 */ 1354, 1354, 1354, 1354, 1354, 1354, 1421, 1599, 1354, 1421, + /* 200 */ 1354, 1354, 1421, 1354, 1354, 1421, 1354, 1768, 1766, 1354, + /* 210 */ 1768, 1766, 1354, 1354, 1354, 1780, 1776, 1759, 1757, 1743, + /* 220 */ 1354, 1354, 1354, 1794, 1790, 1806, 1794, 1790, 1794, 1790, + /* 230 */ 1354, 1766, 1354, 1354, 1766, 1354, 1574, 1354, 1354, 1421, + /* 240 */ 1354, 1421, 1354, 1486, 1354, 1354, 1421, 1354, 1591, 1605, + /* 250 */ 1520, 1520, 1520, 1424, 1359, 1354, 1354, 1354, 1354, 1354, + /* 260 */ 1354, 1354, 1354, 1354, 1354, 1354, 1670, 1779, 1778, 1702, + /* 270 */ 1701, 1700, 1698, 1669, 1482, 1354, 1354, 1354, 1354, 1354, + /* 280 */ 1663, 1664, 1662, 1661, 1354, 1354, 1354, 1354, 1354, 1354, + /* 290 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1727, 1354, + /* 300 */ 1791, 1795, 1354, 1354, 1354, 1646, 1354, 1354, 1354, 1354, + /* 310 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 320 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 330 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 340 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 350 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 360 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 370 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 380 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 390 */ 1388, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 400 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 410 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 420 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 430 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1451, 1450, + /* 440 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 450 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 460 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 470 */ 1354, 1354, 1750, 1760, 1354, 1354, 1354, 1354, 1354, 1354, + /* 480 */ 1354, 1354, 1354, 1354, 1646, 1354, 1777, 1354, 1736, 1732, + /* 490 */ 1354, 1354, 1728, 1354, 1354, 1789, 1354, 1354, 1354, 1354, + /* 500 */ 1354, 1354, 1354, 1354, 1354, 1722, 1354, 1695, 1354, 1354, + /* 510 */ 1354, 1354, 1354, 1354, 1354, 1354, 1657, 1354, 1354, 1354, + /* 520 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1645, + /* 530 */ 1354, 1686, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 540 */ 1514, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 550 */ 1354, 1354, 1354, 1499, 1497, 1496, 1495, 1354, 1492, 1354, + /* 560 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 570 */ 1443, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 580 */ 1434, 1354, 1433, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 590 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 600 */ 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, 1354, + /* 610 */ 1354, 1354, 1354, 1354, 1354, 1354, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1029,6 +1012,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* REDISTRIBUTE => nothing */ 0, /* SPLIT => nothing */ 0, /* SYNCDB => nothing */ + 0, /* DELETE => nothing */ 0, /* NULL => nothing */ 0, /* NK_QUESTION => nothing */ 0, /* NK_ARROW => nothing */ @@ -1082,12 +1066,12 @@ static const YYCODETYPE yyFallback[] = { 0, /* ASC => nothing */ 0, /* NULLS => nothing */ 0, /* ID => nothing */ - 230, /* NK_BITNOT => ID */ - 230, /* INSERT => ID */ - 230, /* VALUES => ID */ - 230, /* IMPORT => ID */ - 230, /* NK_SEMI => ID */ - 230, /* FILE => ID */ + 231, /* NK_BITNOT => ID */ + 231, /* INSERT => ID */ + 231, /* VALUES => ID */ + 231, /* IMPORT => ID */ + 231, /* NK_SEMI => ID */ + 231, /* FILE => ID */ }; #endif /* YYFALLBACK */ @@ -1353,185 +1337,186 @@ static const char *const yyTokenName[] = { /* 175 */ "REDISTRIBUTE", /* 176 */ "SPLIT", /* 177 */ "SYNCDB", - /* 178 */ "NULL", - /* 179 */ "NK_QUESTION", - /* 180 */ "NK_ARROW", - /* 181 */ "ROWTS", - /* 182 */ "TBNAME", - /* 183 */ "QSTARTTS", - /* 184 */ "QENDTS", - /* 185 */ "WSTARTTS", - /* 186 */ "WENDTS", - /* 187 */ "WDURATION", - /* 188 */ "CAST", - /* 189 */ "NOW", - /* 190 */ "TODAY", - /* 191 */ "TIMEZONE", - /* 192 */ "COUNT", - /* 193 */ "FIRST", - /* 194 */ "LAST", - /* 195 */ "LAST_ROW", - /* 196 */ "BETWEEN", - /* 197 */ "IS", - /* 198 */ "NK_LT", - /* 199 */ "NK_GT", - /* 200 */ "NK_LE", - /* 201 */ "NK_GE", - /* 202 */ "NK_NE", - /* 203 */ "MATCH", - /* 204 */ "NMATCH", - /* 205 */ "CONTAINS", - /* 206 */ "JOIN", - /* 207 */ "INNER", - /* 208 */ "SELECT", - /* 209 */ "DISTINCT", - /* 210 */ "WHERE", - /* 211 */ "PARTITION", - /* 212 */ "BY", - /* 213 */ "SESSION", - /* 214 */ "STATE_WINDOW", - /* 215 */ "SLIDING", - /* 216 */ "FILL", - /* 217 */ "VALUE", - /* 218 */ "NONE", - /* 219 */ "PREV", - /* 220 */ "LINEAR", - /* 221 */ "NEXT", - /* 222 */ "HAVING", - /* 223 */ "ORDER", - /* 224 */ "SLIMIT", - /* 225 */ "SOFFSET", - /* 226 */ "LIMIT", - /* 227 */ "OFFSET", - /* 228 */ "ASC", - /* 229 */ "NULLS", - /* 230 */ "ID", - /* 231 */ "NK_BITNOT", - /* 232 */ "INSERT", - /* 233 */ "VALUES", - /* 234 */ "IMPORT", - /* 235 */ "NK_SEMI", - /* 236 */ "FILE", - /* 237 */ "cmd", - /* 238 */ "account_options", - /* 239 */ "alter_account_options", - /* 240 */ "literal", - /* 241 */ "alter_account_option", - /* 242 */ "user_name", - /* 243 */ "privileges", - /* 244 */ "priv_level", - /* 245 */ "priv_type_list", - /* 246 */ "priv_type", - /* 247 */ "db_name", - /* 248 */ "dnode_endpoint", - /* 249 */ "dnode_host_name", - /* 250 */ "not_exists_opt", - /* 251 */ "db_options", - /* 252 */ "exists_opt", - /* 253 */ "alter_db_options", - /* 254 */ "integer_list", - /* 255 */ "variable_list", - /* 256 */ "retention_list", - /* 257 */ "alter_db_option", - /* 258 */ "retention", - /* 259 */ "full_table_name", - /* 260 */ "column_def_list", - /* 261 */ "tags_def_opt", - /* 262 */ "table_options", - /* 263 */ "multi_create_clause", - /* 264 */ "tags_def", - /* 265 */ "multi_drop_clause", - /* 266 */ "alter_table_clause", - /* 267 */ "alter_table_options", - /* 268 */ "column_name", - /* 269 */ "type_name", - /* 270 */ "signed_literal", - /* 271 */ "create_subtable_clause", - /* 272 */ "specific_tags_opt", - /* 273 */ "literal_list", - /* 274 */ "drop_table_clause", - /* 275 */ "col_name_list", - /* 276 */ "table_name", - /* 277 */ "column_def", - /* 278 */ "func_name_list", - /* 279 */ "alter_table_option", - /* 280 */ "col_name", - /* 281 */ "db_name_cond_opt", - /* 282 */ "like_pattern_opt", - /* 283 */ "table_name_cond", - /* 284 */ "from_db_opt", - /* 285 */ "func_name", - /* 286 */ "function_name", - /* 287 */ "index_name", - /* 288 */ "index_options", - /* 289 */ "func_list", - /* 290 */ "duration_literal", - /* 291 */ "sliding_opt", - /* 292 */ "func", - /* 293 */ "expression_list", - /* 294 */ "topic_name", - /* 295 */ "query_expression", - /* 296 */ "cgroup_name", - /* 297 */ "analyze_opt", - /* 298 */ "explain_options", - /* 299 */ "agg_func_opt", - /* 300 */ "bufsize_opt", - /* 301 */ "stream_name", - /* 302 */ "stream_options", - /* 303 */ "into_opt", - /* 304 */ "dnode_list", - /* 305 */ "signed", - /* 306 */ "literal_func", - /* 307 */ "table_alias", - /* 308 */ "column_alias", - /* 309 */ "expression", - /* 310 */ "pseudo_column", - /* 311 */ "column_reference", - /* 312 */ "function_expression", - /* 313 */ "subquery", - /* 314 */ "star_func", - /* 315 */ "star_func_para_list", - /* 316 */ "noarg_func", - /* 317 */ "other_para_list", - /* 318 */ "star_func_para", - /* 319 */ "predicate", - /* 320 */ "compare_op", - /* 321 */ "in_op", - /* 322 */ "in_predicate_value", - /* 323 */ "boolean_value_expression", - /* 324 */ "boolean_primary", - /* 325 */ "common_expression", - /* 326 */ "from_clause", - /* 327 */ "table_reference_list", - /* 328 */ "table_reference", - /* 329 */ "table_primary", - /* 330 */ "joined_table", - /* 331 */ "alias_opt", - /* 332 */ "parenthesized_joined_table", - /* 333 */ "join_type", - /* 334 */ "search_condition", - /* 335 */ "query_specification", - /* 336 */ "set_quantifier_opt", - /* 337 */ "select_list", - /* 338 */ "where_clause_opt", - /* 339 */ "partition_by_clause_opt", - /* 340 */ "twindow_clause_opt", - /* 341 */ "group_by_clause_opt", - /* 342 */ "having_clause_opt", - /* 343 */ "select_sublist", - /* 344 */ "select_item", - /* 345 */ "fill_opt", - /* 346 */ "fill_mode", - /* 347 */ "group_by_list", - /* 348 */ "query_expression_body", - /* 349 */ "order_by_clause_opt", - /* 350 */ "slimit_clause_opt", - /* 351 */ "limit_clause_opt", - /* 352 */ "query_primary", - /* 353 */ "sort_specification_list", - /* 354 */ "sort_specification", - /* 355 */ "ordering_specification_opt", - /* 356 */ "null_ordering_opt", + /* 178 */ "DELETE", + /* 179 */ "NULL", + /* 180 */ "NK_QUESTION", + /* 181 */ "NK_ARROW", + /* 182 */ "ROWTS", + /* 183 */ "TBNAME", + /* 184 */ "QSTARTTS", + /* 185 */ "QENDTS", + /* 186 */ "WSTARTTS", + /* 187 */ "WENDTS", + /* 188 */ "WDURATION", + /* 189 */ "CAST", + /* 190 */ "NOW", + /* 191 */ "TODAY", + /* 192 */ "TIMEZONE", + /* 193 */ "COUNT", + /* 194 */ "FIRST", + /* 195 */ "LAST", + /* 196 */ "LAST_ROW", + /* 197 */ "BETWEEN", + /* 198 */ "IS", + /* 199 */ "NK_LT", + /* 200 */ "NK_GT", + /* 201 */ "NK_LE", + /* 202 */ "NK_GE", + /* 203 */ "NK_NE", + /* 204 */ "MATCH", + /* 205 */ "NMATCH", + /* 206 */ "CONTAINS", + /* 207 */ "JOIN", + /* 208 */ "INNER", + /* 209 */ "SELECT", + /* 210 */ "DISTINCT", + /* 211 */ "WHERE", + /* 212 */ "PARTITION", + /* 213 */ "BY", + /* 214 */ "SESSION", + /* 215 */ "STATE_WINDOW", + /* 216 */ "SLIDING", + /* 217 */ "FILL", + /* 218 */ "VALUE", + /* 219 */ "NONE", + /* 220 */ "PREV", + /* 221 */ "LINEAR", + /* 222 */ "NEXT", + /* 223 */ "HAVING", + /* 224 */ "ORDER", + /* 225 */ "SLIMIT", + /* 226 */ "SOFFSET", + /* 227 */ "LIMIT", + /* 228 */ "OFFSET", + /* 229 */ "ASC", + /* 230 */ "NULLS", + /* 231 */ "ID", + /* 232 */ "NK_BITNOT", + /* 233 */ "INSERT", + /* 234 */ "VALUES", + /* 235 */ "IMPORT", + /* 236 */ "NK_SEMI", + /* 237 */ "FILE", + /* 238 */ "cmd", + /* 239 */ "account_options", + /* 240 */ "alter_account_options", + /* 241 */ "literal", + /* 242 */ "alter_account_option", + /* 243 */ "user_name", + /* 244 */ "privileges", + /* 245 */ "priv_level", + /* 246 */ "priv_type_list", + /* 247 */ "priv_type", + /* 248 */ "db_name", + /* 249 */ "dnode_endpoint", + /* 250 */ "dnode_host_name", + /* 251 */ "not_exists_opt", + /* 252 */ "db_options", + /* 253 */ "exists_opt", + /* 254 */ "alter_db_options", + /* 255 */ "integer_list", + /* 256 */ "variable_list", + /* 257 */ "retention_list", + /* 258 */ "alter_db_option", + /* 259 */ "retention", + /* 260 */ "full_table_name", + /* 261 */ "column_def_list", + /* 262 */ "tags_def_opt", + /* 263 */ "table_options", + /* 264 */ "multi_create_clause", + /* 265 */ "tags_def", + /* 266 */ "multi_drop_clause", + /* 267 */ "alter_table_clause", + /* 268 */ "alter_table_options", + /* 269 */ "column_name", + /* 270 */ "type_name", + /* 271 */ "signed_literal", + /* 272 */ "create_subtable_clause", + /* 273 */ "specific_tags_opt", + /* 274 */ "literal_list", + /* 275 */ "drop_table_clause", + /* 276 */ "col_name_list", + /* 277 */ "table_name", + /* 278 */ "column_def", + /* 279 */ "func_name_list", + /* 280 */ "alter_table_option", + /* 281 */ "col_name", + /* 282 */ "db_name_cond_opt", + /* 283 */ "like_pattern_opt", + /* 284 */ "table_name_cond", + /* 285 */ "from_db_opt", + /* 286 */ "func_name", + /* 287 */ "function_name", + /* 288 */ "index_name", + /* 289 */ "index_options", + /* 290 */ "func_list", + /* 291 */ "duration_literal", + /* 292 */ "sliding_opt", + /* 293 */ "func", + /* 294 */ "expression_list", + /* 295 */ "topic_name", + /* 296 */ "query_expression", + /* 297 */ "cgroup_name", + /* 298 */ "analyze_opt", + /* 299 */ "explain_options", + /* 300 */ "agg_func_opt", + /* 301 */ "bufsize_opt", + /* 302 */ "stream_name", + /* 303 */ "stream_options", + /* 304 */ "into_opt", + /* 305 */ "dnode_list", + /* 306 */ "where_clause_opt", + /* 307 */ "signed", + /* 308 */ "literal_func", + /* 309 */ "table_alias", + /* 310 */ "column_alias", + /* 311 */ "expression", + /* 312 */ "pseudo_column", + /* 313 */ "column_reference", + /* 314 */ "function_expression", + /* 315 */ "subquery", + /* 316 */ "star_func", + /* 317 */ "star_func_para_list", + /* 318 */ "noarg_func", + /* 319 */ "other_para_list", + /* 320 */ "star_func_para", + /* 321 */ "predicate", + /* 322 */ "compare_op", + /* 323 */ "in_op", + /* 324 */ "in_predicate_value", + /* 325 */ "boolean_value_expression", + /* 326 */ "boolean_primary", + /* 327 */ "common_expression", + /* 328 */ "from_clause", + /* 329 */ "table_reference_list", + /* 330 */ "table_reference", + /* 331 */ "table_primary", + /* 332 */ "joined_table", + /* 333 */ "alias_opt", + /* 334 */ "parenthesized_joined_table", + /* 335 */ "join_type", + /* 336 */ "search_condition", + /* 337 */ "query_specification", + /* 338 */ "set_quantifier_opt", + /* 339 */ "select_list", + /* 340 */ "partition_by_clause_opt", + /* 341 */ "twindow_clause_opt", + /* 342 */ "group_by_clause_opt", + /* 343 */ "having_clause_opt", + /* 344 */ "select_sublist", + /* 345 */ "select_item", + /* 346 */ "fill_opt", + /* 347 */ "fill_mode", + /* 348 */ "group_by_list", + /* 349 */ "query_expression_body", + /* 350 */ "order_by_clause_opt", + /* 351 */ "slimit_clause_opt", + /* 352 */ "limit_clause_opt", + /* 353 */ "query_primary", + /* 354 */ "sort_specification_list", + /* 355 */ "sort_specification", + /* 356 */ "ordering_specification_opt", + /* 357 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1800,196 +1785,197 @@ static const char *const yyRuleName[] = { /* 258 */ "dnode_list ::= DNODE NK_INTEGER", /* 259 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", /* 260 */ "cmd ::= SYNCDB db_name REPLICA", - /* 261 */ "cmd ::= query_expression", - /* 262 */ "literal ::= NK_INTEGER", - /* 263 */ "literal ::= NK_FLOAT", - /* 264 */ "literal ::= NK_STRING", - /* 265 */ "literal ::= NK_BOOL", - /* 266 */ "literal ::= TIMESTAMP NK_STRING", - /* 267 */ "literal ::= duration_literal", - /* 268 */ "literal ::= NULL", - /* 269 */ "literal ::= NK_QUESTION", - /* 270 */ "duration_literal ::= NK_VARIABLE", - /* 271 */ "signed ::= NK_INTEGER", - /* 272 */ "signed ::= NK_PLUS NK_INTEGER", - /* 273 */ "signed ::= NK_MINUS NK_INTEGER", - /* 274 */ "signed ::= NK_FLOAT", - /* 275 */ "signed ::= NK_PLUS NK_FLOAT", - /* 276 */ "signed ::= NK_MINUS NK_FLOAT", - /* 277 */ "signed_literal ::= signed", - /* 278 */ "signed_literal ::= NK_STRING", - /* 279 */ "signed_literal ::= NK_BOOL", - /* 280 */ "signed_literal ::= TIMESTAMP NK_STRING", - /* 281 */ "signed_literal ::= duration_literal", - /* 282 */ "signed_literal ::= NULL", - /* 283 */ "signed_literal ::= literal_func", - /* 284 */ "literal_list ::= signed_literal", - /* 285 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 286 */ "db_name ::= NK_ID", - /* 287 */ "table_name ::= NK_ID", - /* 288 */ "column_name ::= NK_ID", - /* 289 */ "function_name ::= NK_ID", - /* 290 */ "table_alias ::= NK_ID", - /* 291 */ "column_alias ::= NK_ID", - /* 292 */ "user_name ::= NK_ID", - /* 293 */ "index_name ::= NK_ID", - /* 294 */ "topic_name ::= NK_ID", - /* 295 */ "stream_name ::= NK_ID", - /* 296 */ "cgroup_name ::= NK_ID", - /* 297 */ "expression ::= literal", - /* 298 */ "expression ::= pseudo_column", - /* 299 */ "expression ::= column_reference", - /* 300 */ "expression ::= function_expression", - /* 301 */ "expression ::= subquery", - /* 302 */ "expression ::= NK_LP expression NK_RP", - /* 303 */ "expression ::= NK_PLUS expression", - /* 304 */ "expression ::= NK_MINUS expression", - /* 305 */ "expression ::= expression NK_PLUS expression", - /* 306 */ "expression ::= expression NK_MINUS expression", - /* 307 */ "expression ::= expression NK_STAR expression", - /* 308 */ "expression ::= expression NK_SLASH expression", - /* 309 */ "expression ::= expression NK_REM expression", - /* 310 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 311 */ "expression_list ::= expression", - /* 312 */ "expression_list ::= expression_list NK_COMMA expression", - /* 313 */ "column_reference ::= column_name", - /* 314 */ "column_reference ::= table_name NK_DOT column_name", - /* 315 */ "pseudo_column ::= ROWTS", - /* 316 */ "pseudo_column ::= TBNAME", - /* 317 */ "pseudo_column ::= table_name NK_DOT TBNAME", - /* 318 */ "pseudo_column ::= QSTARTTS", - /* 319 */ "pseudo_column ::= QENDTS", - /* 320 */ "pseudo_column ::= WSTARTTS", - /* 321 */ "pseudo_column ::= WENDTS", - /* 322 */ "pseudo_column ::= WDURATION", - /* 323 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 324 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 325 */ "function_expression ::= CAST NK_LP expression AS type_name NK_RP", - /* 326 */ "function_expression ::= literal_func", - /* 327 */ "literal_func ::= noarg_func NK_LP NK_RP", - /* 328 */ "literal_func ::= NOW", - /* 329 */ "noarg_func ::= NOW", - /* 330 */ "noarg_func ::= TODAY", - /* 331 */ "noarg_func ::= TIMEZONE", - /* 332 */ "star_func ::= COUNT", - /* 333 */ "star_func ::= FIRST", - /* 334 */ "star_func ::= LAST", - /* 335 */ "star_func ::= LAST_ROW", - /* 336 */ "star_func_para_list ::= NK_STAR", - /* 337 */ "star_func_para_list ::= other_para_list", - /* 338 */ "other_para_list ::= star_func_para", - /* 339 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 340 */ "star_func_para ::= expression", - /* 341 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 342 */ "predicate ::= expression compare_op expression", - /* 343 */ "predicate ::= expression BETWEEN expression AND expression", - /* 344 */ "predicate ::= expression NOT BETWEEN expression AND expression", - /* 345 */ "predicate ::= expression IS NULL", - /* 346 */ "predicate ::= expression IS NOT NULL", - /* 347 */ "predicate ::= expression in_op in_predicate_value", - /* 348 */ "compare_op ::= NK_LT", - /* 349 */ "compare_op ::= NK_GT", - /* 350 */ "compare_op ::= NK_LE", - /* 351 */ "compare_op ::= NK_GE", - /* 352 */ "compare_op ::= NK_NE", - /* 353 */ "compare_op ::= NK_EQ", - /* 354 */ "compare_op ::= LIKE", - /* 355 */ "compare_op ::= NOT LIKE", - /* 356 */ "compare_op ::= MATCH", - /* 357 */ "compare_op ::= NMATCH", - /* 358 */ "compare_op ::= CONTAINS", - /* 359 */ "in_op ::= IN", - /* 360 */ "in_op ::= NOT IN", - /* 361 */ "in_predicate_value ::= NK_LP expression_list NK_RP", - /* 362 */ "boolean_value_expression ::= boolean_primary", - /* 363 */ "boolean_value_expression ::= NOT boolean_primary", - /* 364 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 365 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 366 */ "boolean_primary ::= predicate", - /* 367 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 368 */ "common_expression ::= expression", - /* 369 */ "common_expression ::= boolean_value_expression", - /* 370 */ "from_clause ::= FROM table_reference_list", - /* 371 */ "table_reference_list ::= table_reference", - /* 372 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 373 */ "table_reference ::= table_primary", - /* 374 */ "table_reference ::= joined_table", - /* 375 */ "table_primary ::= table_name alias_opt", - /* 376 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 377 */ "table_primary ::= subquery alias_opt", - /* 378 */ "table_primary ::= parenthesized_joined_table", - /* 379 */ "alias_opt ::=", - /* 380 */ "alias_opt ::= table_alias", - /* 381 */ "alias_opt ::= AS table_alias", - /* 382 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 383 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 384 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 385 */ "join_type ::=", - /* 386 */ "join_type ::= INNER", - /* 387 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 388 */ "set_quantifier_opt ::=", - /* 389 */ "set_quantifier_opt ::= DISTINCT", - /* 390 */ "set_quantifier_opt ::= ALL", - /* 391 */ "select_list ::= NK_STAR", - /* 392 */ "select_list ::= select_sublist", - /* 393 */ "select_sublist ::= select_item", - /* 394 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 395 */ "select_item ::= common_expression", - /* 396 */ "select_item ::= common_expression column_alias", - /* 397 */ "select_item ::= common_expression AS column_alias", - /* 398 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 399 */ "where_clause_opt ::=", - /* 400 */ "where_clause_opt ::= WHERE search_condition", - /* 401 */ "partition_by_clause_opt ::=", - /* 402 */ "partition_by_clause_opt ::= PARTITION BY expression_list", - /* 403 */ "twindow_clause_opt ::=", - /* 404 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 405 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", - /* 406 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 407 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 408 */ "sliding_opt ::=", - /* 409 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 410 */ "fill_opt ::=", - /* 411 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 412 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 413 */ "fill_mode ::= NONE", - /* 414 */ "fill_mode ::= PREV", - /* 415 */ "fill_mode ::= NULL", - /* 416 */ "fill_mode ::= LINEAR", - /* 417 */ "fill_mode ::= NEXT", - /* 418 */ "group_by_clause_opt ::=", - /* 419 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 420 */ "group_by_list ::= expression", - /* 421 */ "group_by_list ::= group_by_list NK_COMMA expression", - /* 422 */ "having_clause_opt ::=", - /* 423 */ "having_clause_opt ::= HAVING search_condition", - /* 424 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 425 */ "query_expression_body ::= query_primary", - /* 426 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 427 */ "query_expression_body ::= query_expression_body UNION query_expression_body", - /* 428 */ "query_primary ::= query_specification", - /* 429 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP", - /* 430 */ "order_by_clause_opt ::=", - /* 431 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 432 */ "slimit_clause_opt ::=", - /* 433 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 434 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 435 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 436 */ "limit_clause_opt ::=", - /* 437 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 438 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 439 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 440 */ "subquery ::= NK_LP query_expression NK_RP", - /* 441 */ "search_condition ::= common_expression", - /* 442 */ "sort_specification_list ::= sort_specification", - /* 443 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 444 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 445 */ "ordering_specification_opt ::=", - /* 446 */ "ordering_specification_opt ::= ASC", - /* 447 */ "ordering_specification_opt ::= DESC", - /* 448 */ "null_ordering_opt ::=", - /* 449 */ "null_ordering_opt ::= NULLS FIRST", - /* 450 */ "null_ordering_opt ::= NULLS LAST", + /* 261 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", + /* 262 */ "cmd ::= query_expression", + /* 263 */ "literal ::= NK_INTEGER", + /* 264 */ "literal ::= NK_FLOAT", + /* 265 */ "literal ::= NK_STRING", + /* 266 */ "literal ::= NK_BOOL", + /* 267 */ "literal ::= TIMESTAMP NK_STRING", + /* 268 */ "literal ::= duration_literal", + /* 269 */ "literal ::= NULL", + /* 270 */ "literal ::= NK_QUESTION", + /* 271 */ "duration_literal ::= NK_VARIABLE", + /* 272 */ "signed ::= NK_INTEGER", + /* 273 */ "signed ::= NK_PLUS NK_INTEGER", + /* 274 */ "signed ::= NK_MINUS NK_INTEGER", + /* 275 */ "signed ::= NK_FLOAT", + /* 276 */ "signed ::= NK_PLUS NK_FLOAT", + /* 277 */ "signed ::= NK_MINUS NK_FLOAT", + /* 278 */ "signed_literal ::= signed", + /* 279 */ "signed_literal ::= NK_STRING", + /* 280 */ "signed_literal ::= NK_BOOL", + /* 281 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 282 */ "signed_literal ::= duration_literal", + /* 283 */ "signed_literal ::= NULL", + /* 284 */ "signed_literal ::= literal_func", + /* 285 */ "literal_list ::= signed_literal", + /* 286 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 287 */ "db_name ::= NK_ID", + /* 288 */ "table_name ::= NK_ID", + /* 289 */ "column_name ::= NK_ID", + /* 290 */ "function_name ::= NK_ID", + /* 291 */ "table_alias ::= NK_ID", + /* 292 */ "column_alias ::= NK_ID", + /* 293 */ "user_name ::= NK_ID", + /* 294 */ "index_name ::= NK_ID", + /* 295 */ "topic_name ::= NK_ID", + /* 296 */ "stream_name ::= NK_ID", + /* 297 */ "cgroup_name ::= NK_ID", + /* 298 */ "expression ::= literal", + /* 299 */ "expression ::= pseudo_column", + /* 300 */ "expression ::= column_reference", + /* 301 */ "expression ::= function_expression", + /* 302 */ "expression ::= subquery", + /* 303 */ "expression ::= NK_LP expression NK_RP", + /* 304 */ "expression ::= NK_PLUS expression", + /* 305 */ "expression ::= NK_MINUS expression", + /* 306 */ "expression ::= expression NK_PLUS expression", + /* 307 */ "expression ::= expression NK_MINUS expression", + /* 308 */ "expression ::= expression NK_STAR expression", + /* 309 */ "expression ::= expression NK_SLASH expression", + /* 310 */ "expression ::= expression NK_REM expression", + /* 311 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 312 */ "expression_list ::= expression", + /* 313 */ "expression_list ::= expression_list NK_COMMA expression", + /* 314 */ "column_reference ::= column_name", + /* 315 */ "column_reference ::= table_name NK_DOT column_name", + /* 316 */ "pseudo_column ::= ROWTS", + /* 317 */ "pseudo_column ::= TBNAME", + /* 318 */ "pseudo_column ::= table_name NK_DOT TBNAME", + /* 319 */ "pseudo_column ::= QSTARTTS", + /* 320 */ "pseudo_column ::= QENDTS", + /* 321 */ "pseudo_column ::= WSTARTTS", + /* 322 */ "pseudo_column ::= WENDTS", + /* 323 */ "pseudo_column ::= WDURATION", + /* 324 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 325 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 326 */ "function_expression ::= CAST NK_LP expression AS type_name NK_RP", + /* 327 */ "function_expression ::= literal_func", + /* 328 */ "literal_func ::= noarg_func NK_LP NK_RP", + /* 329 */ "literal_func ::= NOW", + /* 330 */ "noarg_func ::= NOW", + /* 331 */ "noarg_func ::= TODAY", + /* 332 */ "noarg_func ::= TIMEZONE", + /* 333 */ "star_func ::= COUNT", + /* 334 */ "star_func ::= FIRST", + /* 335 */ "star_func ::= LAST", + /* 336 */ "star_func ::= LAST_ROW", + /* 337 */ "star_func_para_list ::= NK_STAR", + /* 338 */ "star_func_para_list ::= other_para_list", + /* 339 */ "other_para_list ::= star_func_para", + /* 340 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 341 */ "star_func_para ::= expression", + /* 342 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 343 */ "predicate ::= expression compare_op expression", + /* 344 */ "predicate ::= expression BETWEEN expression AND expression", + /* 345 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 346 */ "predicate ::= expression IS NULL", + /* 347 */ "predicate ::= expression IS NOT NULL", + /* 348 */ "predicate ::= expression in_op in_predicate_value", + /* 349 */ "compare_op ::= NK_LT", + /* 350 */ "compare_op ::= NK_GT", + /* 351 */ "compare_op ::= NK_LE", + /* 352 */ "compare_op ::= NK_GE", + /* 353 */ "compare_op ::= NK_NE", + /* 354 */ "compare_op ::= NK_EQ", + /* 355 */ "compare_op ::= LIKE", + /* 356 */ "compare_op ::= NOT LIKE", + /* 357 */ "compare_op ::= MATCH", + /* 358 */ "compare_op ::= NMATCH", + /* 359 */ "compare_op ::= CONTAINS", + /* 360 */ "in_op ::= IN", + /* 361 */ "in_op ::= NOT IN", + /* 362 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 363 */ "boolean_value_expression ::= boolean_primary", + /* 364 */ "boolean_value_expression ::= NOT boolean_primary", + /* 365 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 366 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 367 */ "boolean_primary ::= predicate", + /* 368 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 369 */ "common_expression ::= expression", + /* 370 */ "common_expression ::= boolean_value_expression", + /* 371 */ "from_clause ::= FROM table_reference_list", + /* 372 */ "table_reference_list ::= table_reference", + /* 373 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 374 */ "table_reference ::= table_primary", + /* 375 */ "table_reference ::= joined_table", + /* 376 */ "table_primary ::= table_name alias_opt", + /* 377 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 378 */ "table_primary ::= subquery alias_opt", + /* 379 */ "table_primary ::= parenthesized_joined_table", + /* 380 */ "alias_opt ::=", + /* 381 */ "alias_opt ::= table_alias", + /* 382 */ "alias_opt ::= AS table_alias", + /* 383 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 384 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 385 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 386 */ "join_type ::=", + /* 387 */ "join_type ::= INNER", + /* 388 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 389 */ "set_quantifier_opt ::=", + /* 390 */ "set_quantifier_opt ::= DISTINCT", + /* 391 */ "set_quantifier_opt ::= ALL", + /* 392 */ "select_list ::= NK_STAR", + /* 393 */ "select_list ::= select_sublist", + /* 394 */ "select_sublist ::= select_item", + /* 395 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 396 */ "select_item ::= common_expression", + /* 397 */ "select_item ::= common_expression column_alias", + /* 398 */ "select_item ::= common_expression AS column_alias", + /* 399 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 400 */ "where_clause_opt ::=", + /* 401 */ "where_clause_opt ::= WHERE search_condition", + /* 402 */ "partition_by_clause_opt ::=", + /* 403 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 404 */ "twindow_clause_opt ::=", + /* 405 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 406 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", + /* 407 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 408 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 409 */ "sliding_opt ::=", + /* 410 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 411 */ "fill_opt ::=", + /* 412 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 413 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 414 */ "fill_mode ::= NONE", + /* 415 */ "fill_mode ::= PREV", + /* 416 */ "fill_mode ::= NULL", + /* 417 */ "fill_mode ::= LINEAR", + /* 418 */ "fill_mode ::= NEXT", + /* 419 */ "group_by_clause_opt ::=", + /* 420 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 421 */ "group_by_list ::= expression", + /* 422 */ "group_by_list ::= group_by_list NK_COMMA expression", + /* 423 */ "having_clause_opt ::=", + /* 424 */ "having_clause_opt ::= HAVING search_condition", + /* 425 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 426 */ "query_expression_body ::= query_primary", + /* 427 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 428 */ "query_expression_body ::= query_expression_body UNION query_expression_body", + /* 429 */ "query_primary ::= query_specification", + /* 430 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP", + /* 431 */ "order_by_clause_opt ::=", + /* 432 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 433 */ "slimit_clause_opt ::=", + /* 434 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 435 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 436 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 437 */ "limit_clause_opt ::=", + /* 438 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 439 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 440 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 441 */ "subquery ::= NK_LP query_expression NK_RP", + /* 442 */ "search_condition ::= common_expression", + /* 443 */ "sort_specification_list ::= sort_specification", + /* 444 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 445 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 446 */ "ordering_specification_opt ::=", + /* 447 */ "ordering_specification_opt ::= ASC", + /* 448 */ "ordering_specification_opt ::= DESC", + /* 449 */ "null_ordering_opt ::=", + /* 450 */ "null_ordering_opt ::= NULLS FIRST", + /* 451 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2116,174 +2102,174 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 237: /* cmd */ - case 240: /* literal */ - case 251: /* db_options */ - case 253: /* alter_db_options */ - case 258: /* retention */ - case 259: /* full_table_name */ - case 262: /* table_options */ - case 266: /* alter_table_clause */ - case 267: /* alter_table_options */ - case 270: /* signed_literal */ - case 271: /* create_subtable_clause */ - case 274: /* drop_table_clause */ - case 277: /* column_def */ - case 280: /* col_name */ - case 281: /* db_name_cond_opt */ - case 282: /* like_pattern_opt */ - case 283: /* table_name_cond */ - case 284: /* from_db_opt */ - case 285: /* func_name */ - case 288: /* index_options */ - case 290: /* duration_literal */ - case 291: /* sliding_opt */ - case 292: /* func */ - case 295: /* query_expression */ - case 298: /* explain_options */ - case 302: /* stream_options */ - case 303: /* into_opt */ - case 305: /* signed */ - case 306: /* literal_func */ - case 309: /* expression */ - case 310: /* pseudo_column */ - case 311: /* column_reference */ - case 312: /* function_expression */ - case 313: /* subquery */ - case 318: /* star_func_para */ - case 319: /* predicate */ - case 322: /* in_predicate_value */ - case 323: /* boolean_value_expression */ - case 324: /* boolean_primary */ - case 325: /* common_expression */ - case 326: /* from_clause */ - case 327: /* table_reference_list */ - case 328: /* table_reference */ - case 329: /* table_primary */ - case 330: /* joined_table */ - case 332: /* parenthesized_joined_table */ - case 334: /* search_condition */ - case 335: /* query_specification */ - case 338: /* where_clause_opt */ - case 340: /* twindow_clause_opt */ - case 342: /* having_clause_opt */ - case 344: /* select_item */ - case 345: /* fill_opt */ - case 348: /* query_expression_body */ - case 350: /* slimit_clause_opt */ - case 351: /* limit_clause_opt */ - case 352: /* query_primary */ - case 354: /* sort_specification */ + case 238: /* cmd */ + case 241: /* literal */ + case 252: /* db_options */ + case 254: /* alter_db_options */ + case 259: /* retention */ + case 260: /* full_table_name */ + case 263: /* table_options */ + case 267: /* alter_table_clause */ + case 268: /* alter_table_options */ + case 271: /* signed_literal */ + case 272: /* create_subtable_clause */ + case 275: /* drop_table_clause */ + case 278: /* column_def */ + case 281: /* col_name */ + case 282: /* db_name_cond_opt */ + case 283: /* like_pattern_opt */ + case 284: /* table_name_cond */ + case 285: /* from_db_opt */ + case 286: /* func_name */ + case 289: /* index_options */ + case 291: /* duration_literal */ + case 292: /* sliding_opt */ + case 293: /* func */ + case 296: /* query_expression */ + case 299: /* explain_options */ + case 303: /* stream_options */ + case 304: /* into_opt */ + case 306: /* where_clause_opt */ + case 307: /* signed */ + case 308: /* literal_func */ + case 311: /* expression */ + case 312: /* pseudo_column */ + case 313: /* column_reference */ + case 314: /* function_expression */ + case 315: /* subquery */ + case 320: /* star_func_para */ + case 321: /* predicate */ + case 324: /* in_predicate_value */ + case 325: /* boolean_value_expression */ + case 326: /* boolean_primary */ + case 327: /* common_expression */ + case 328: /* from_clause */ + case 329: /* table_reference_list */ + case 330: /* table_reference */ + case 331: /* table_primary */ + case 332: /* joined_table */ + case 334: /* parenthesized_joined_table */ + case 336: /* search_condition */ + case 337: /* query_specification */ + case 341: /* twindow_clause_opt */ + case 343: /* having_clause_opt */ + case 345: /* select_item */ + case 346: /* fill_opt */ + case 349: /* query_expression_body */ + case 351: /* slimit_clause_opt */ + case 352: /* limit_clause_opt */ + case 353: /* query_primary */ + case 355: /* sort_specification */ { - nodesDestroyNode((yypminor->yy686)); + nodesDestroyNode((yypminor->yy172)); } break; - case 238: /* account_options */ - case 239: /* alter_account_options */ - case 241: /* alter_account_option */ - case 300: /* bufsize_opt */ + case 239: /* account_options */ + case 240: /* alter_account_options */ + case 242: /* alter_account_option */ + case 301: /* bufsize_opt */ { } break; - case 242: /* user_name */ - case 244: /* priv_level */ - case 247: /* db_name */ - case 248: /* dnode_endpoint */ - case 249: /* dnode_host_name */ - case 268: /* column_name */ - case 276: /* table_name */ - case 286: /* function_name */ - case 287: /* index_name */ - case 294: /* topic_name */ - case 296: /* cgroup_name */ - case 301: /* stream_name */ - case 307: /* table_alias */ - case 308: /* column_alias */ - case 314: /* star_func */ - case 316: /* noarg_func */ - case 331: /* alias_opt */ + case 243: /* user_name */ + case 245: /* priv_level */ + case 248: /* db_name */ + case 249: /* dnode_endpoint */ + case 250: /* dnode_host_name */ + case 269: /* column_name */ + case 277: /* table_name */ + case 287: /* function_name */ + case 288: /* index_name */ + case 295: /* topic_name */ + case 297: /* cgroup_name */ + case 302: /* stream_name */ + case 309: /* table_alias */ + case 310: /* column_alias */ + case 316: /* star_func */ + case 318: /* noarg_func */ + case 333: /* alias_opt */ { } break; - case 243: /* privileges */ - case 245: /* priv_type_list */ - case 246: /* priv_type */ + case 244: /* privileges */ + case 246: /* priv_type_list */ + case 247: /* priv_type */ { } break; - case 250: /* not_exists_opt */ - case 252: /* exists_opt */ - case 297: /* analyze_opt */ - case 299: /* agg_func_opt */ - case 336: /* set_quantifier_opt */ + case 251: /* not_exists_opt */ + case 253: /* exists_opt */ + case 298: /* analyze_opt */ + case 300: /* agg_func_opt */ + case 338: /* set_quantifier_opt */ { } break; - case 254: /* integer_list */ - case 255: /* variable_list */ - case 256: /* retention_list */ - case 260: /* column_def_list */ - case 261: /* tags_def_opt */ - case 263: /* multi_create_clause */ - case 264: /* tags_def */ - case 265: /* multi_drop_clause */ - case 272: /* specific_tags_opt */ - case 273: /* literal_list */ - case 275: /* col_name_list */ - case 278: /* func_name_list */ - case 289: /* func_list */ - case 293: /* expression_list */ - case 304: /* dnode_list */ - case 315: /* star_func_para_list */ - case 317: /* other_para_list */ - case 337: /* select_list */ - case 339: /* partition_by_clause_opt */ - case 341: /* group_by_clause_opt */ - case 343: /* select_sublist */ - case 347: /* group_by_list */ - case 349: /* order_by_clause_opt */ - case 353: /* sort_specification_list */ + case 255: /* integer_list */ + case 256: /* variable_list */ + case 257: /* retention_list */ + case 261: /* column_def_list */ + case 262: /* tags_def_opt */ + case 264: /* multi_create_clause */ + case 265: /* tags_def */ + case 266: /* multi_drop_clause */ + case 273: /* specific_tags_opt */ + case 274: /* literal_list */ + case 276: /* col_name_list */ + case 279: /* func_name_list */ + case 290: /* func_list */ + case 294: /* expression_list */ + case 305: /* dnode_list */ + case 317: /* star_func_para_list */ + case 319: /* other_para_list */ + case 339: /* select_list */ + case 340: /* partition_by_clause_opt */ + case 342: /* group_by_clause_opt */ + case 344: /* select_sublist */ + case 348: /* group_by_list */ + case 350: /* order_by_clause_opt */ + case 354: /* sort_specification_list */ { - nodesDestroyList((yypminor->yy670)); + nodesDestroyList((yypminor->yy60)); } break; - case 257: /* alter_db_option */ - case 279: /* alter_table_option */ + case 258: /* alter_db_option */ + case 280: /* alter_table_option */ { } break; - case 269: /* type_name */ + case 270: /* type_name */ { } break; - case 320: /* compare_op */ - case 321: /* in_op */ + case 322: /* compare_op */ + case 323: /* in_op */ { } break; - case 333: /* join_type */ + case 335: /* join_type */ { } break; - case 346: /* fill_mode */ + case 347: /* fill_mode */ { } break; - case 355: /* ordering_specification_opt */ + case 356: /* ordering_specification_opt */ { } break; - case 356: /* null_ordering_opt */ + case 357: /* null_ordering_opt */ { } @@ -2411,18 +2397,15 @@ static YYACTIONTYPE yy_find_shift_action( do{ i = yy_shift_ofst[stateno]; assert( i>=0 ); - assert( i<=YY_ACTTAB_COUNT ); - assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); + /* assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); */ assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; - assert( i<(int)YY_NLOOKAHEAD ); - if( yy_lookahead[i]!=iLookAhead ){ + if( i>=YY_NLOOKAHEAD || yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ - assert( iLookAhead %s\n", @@ -2437,8 +2420,16 @@ static YYACTIONTYPE yy_find_shift_action( #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; - assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); - if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ + if( +#if YY_SHIFT_MIN+YYWILDCARD<0 + j>=0 && +#endif +#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT + j0 + ){ #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", @@ -2452,7 +2443,6 @@ static YYACTIONTYPE yy_find_shift_action( #endif /* YYWILDCARD */ return yy_default[stateno]; }else{ - assert( i>=0 && iyytos; #ifndef NDEBUG if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfoNRhs[yyruleno]; + yysize = yyRuleInfo[yyruleno].nrhs; if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", + fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", yyTracePrompt, - yyruleno, yyRuleName[yyruleno], - yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ yypParser->yyhwm++; @@ -3572,11 +3108,11 @@ static YYACTIONTYPE yy_reduce( YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,238,&yymsp[0].minor); + yy_destructor(yypParser,239,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,239,&yymsp[0].minor); + yy_destructor(yypParser,240,&yymsp[0].minor); break; case 2: /* account_options ::= */ { } @@ -3590,20 +3126,20 @@ static YYACTIONTYPE yy_reduce( case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); -{ yy_destructor(yypParser,238,&yymsp[-2].minor); +{ yy_destructor(yypParser,239,&yymsp[-2].minor); { } - yy_destructor(yypParser,240,&yymsp[0].minor); + yy_destructor(yypParser,241,&yymsp[0].minor); } break; case 12: /* alter_account_options ::= alter_account_option */ -{ yy_destructor(yypParser,241,&yymsp[0].minor); +{ yy_destructor(yypParser,242,&yymsp[0].minor); { } } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ -{ yy_destructor(yypParser,239,&yymsp[-1].minor); +{ yy_destructor(yypParser,240,&yymsp[-1].minor); { } - yy_destructor(yypParser,241,&yymsp[0].minor); + yy_destructor(yypParser,242,&yymsp[0].minor); } break; case 14: /* alter_account_option ::= PASS literal */ @@ -3617,63 +3153,63 @@ static YYACTIONTYPE yy_reduce( case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); { } - yy_destructor(yypParser,240,&yymsp[0].minor); + yy_destructor(yypParser,241,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy0); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy113, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy105, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy113, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy105, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy113); } +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy105); } break; case 28: /* cmd ::= GRANT privileges ON priv_level TO user_name */ -{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy123, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113); } +{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy593, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy105); } break; case 29: /* cmd ::= REVOKE privileges ON priv_level FROM user_name */ -{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy123, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113); } +{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy593, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy105); } break; case 30: /* privileges ::= ALL */ -{ yymsp[0].minor.yy123 = PRIVILEGE_TYPE_ALL; } +{ yymsp[0].minor.yy593 = PRIVILEGE_TYPE_ALL; } break; case 31: /* privileges ::= priv_type_list */ case 32: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==32); -{ yylhsminor.yy123 = yymsp[0].minor.yy123; } - yymsp[0].minor.yy123 = yylhsminor.yy123; +{ yylhsminor.yy593 = yymsp[0].minor.yy593; } + yymsp[0].minor.yy593 = yylhsminor.yy593; break; case 33: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ -{ yylhsminor.yy123 = yymsp[-2].minor.yy123 | yymsp[0].minor.yy123; } - yymsp[-2].minor.yy123 = yylhsminor.yy123; +{ yylhsminor.yy593 = yymsp[-2].minor.yy593 | yymsp[0].minor.yy593; } + yymsp[-2].minor.yy593 = yylhsminor.yy593; break; case 34: /* priv_type ::= READ */ -{ yymsp[0].minor.yy123 = PRIVILEGE_TYPE_READ; } +{ yymsp[0].minor.yy593 = PRIVILEGE_TYPE_READ; } break; case 35: /* priv_type ::= WRITE */ -{ yymsp[0].minor.yy123 = PRIVILEGE_TYPE_WRITE; } +{ yymsp[0].minor.yy593 = PRIVILEGE_TYPE_WRITE; } break; case 36: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ -{ yylhsminor.yy113 = yymsp[-2].minor.yy0; } - yymsp[-2].minor.yy113 = yylhsminor.yy113; +{ yylhsminor.yy105 = yymsp[-2].minor.yy0; } + yymsp[-2].minor.yy105 = yylhsminor.yy105; break; case 37: /* priv_level ::= db_name NK_DOT NK_STAR */ -{ yylhsminor.yy113 = yymsp[-2].minor.yy113; } - yymsp[-2].minor.yy113 = yylhsminor.yy113; +{ yylhsminor.yy105 = yymsp[-2].minor.yy105; } + yymsp[-2].minor.yy105 = yylhsminor.yy105; break; case 38: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy113, NULL); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy105, NULL); } break; case 39: /* cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy0); } break; case 40: /* cmd ::= DROP DNODE NK_INTEGER */ { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy0); } break; case 41: /* cmd ::= DROP DNODE dnode_endpoint */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy113); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy105); } break; case 42: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } @@ -3690,26 +3226,26 @@ static YYACTIONTYPE yy_reduce( case 46: /* dnode_endpoint ::= NK_STRING */ case 47: /* dnode_host_name ::= NK_ID */ yytestcase(yyruleno==47); case 48: /* dnode_host_name ::= NK_IPTOKEN */ yytestcase(yyruleno==48); - case 286: /* db_name ::= NK_ID */ yytestcase(yyruleno==286); - case 287: /* table_name ::= NK_ID */ yytestcase(yyruleno==287); - case 288: /* column_name ::= NK_ID */ yytestcase(yyruleno==288); - case 289: /* function_name ::= NK_ID */ yytestcase(yyruleno==289); - case 290: /* table_alias ::= NK_ID */ yytestcase(yyruleno==290); - case 291: /* column_alias ::= NK_ID */ yytestcase(yyruleno==291); - case 292: /* user_name ::= NK_ID */ yytestcase(yyruleno==292); - case 293: /* index_name ::= NK_ID */ yytestcase(yyruleno==293); - case 294: /* topic_name ::= NK_ID */ yytestcase(yyruleno==294); - case 295: /* stream_name ::= NK_ID */ yytestcase(yyruleno==295); - case 296: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==296); - case 329: /* noarg_func ::= NOW */ yytestcase(yyruleno==329); - case 330: /* noarg_func ::= TODAY */ yytestcase(yyruleno==330); - case 331: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==331); - case 332: /* star_func ::= COUNT */ yytestcase(yyruleno==332); - case 333: /* star_func ::= FIRST */ yytestcase(yyruleno==333); - case 334: /* star_func ::= LAST */ yytestcase(yyruleno==334); - case 335: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==335); -{ yylhsminor.yy113 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy113 = yylhsminor.yy113; + case 287: /* db_name ::= NK_ID */ yytestcase(yyruleno==287); + case 288: /* table_name ::= NK_ID */ yytestcase(yyruleno==288); + case 289: /* column_name ::= NK_ID */ yytestcase(yyruleno==289); + case 290: /* function_name ::= NK_ID */ yytestcase(yyruleno==290); + case 291: /* table_alias ::= NK_ID */ yytestcase(yyruleno==291); + case 292: /* column_alias ::= NK_ID */ yytestcase(yyruleno==292); + case 293: /* user_name ::= NK_ID */ yytestcase(yyruleno==293); + case 294: /* index_name ::= NK_ID */ yytestcase(yyruleno==294); + case 295: /* topic_name ::= NK_ID */ yytestcase(yyruleno==295); + case 296: /* stream_name ::= NK_ID */ yytestcase(yyruleno==296); + case 297: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==297); + case 330: /* noarg_func ::= NOW */ yytestcase(yyruleno==330); + case 331: /* noarg_func ::= TODAY */ yytestcase(yyruleno==331); + case 332: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==332); + case 333: /* star_func ::= COUNT */ yytestcase(yyruleno==333); + case 334: /* star_func ::= FIRST */ yytestcase(yyruleno==334); + case 335: /* star_func ::= LAST */ yytestcase(yyruleno==335); + case 336: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==336); +{ yylhsminor.yy105 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy105 = yylhsminor.yy105; break; case 49: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } @@ -3742,156 +3278,156 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } break; case 59: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy131, &yymsp[-1].minor.yy113, yymsp[0].minor.yy686); } +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy617, &yymsp[-1].minor.yy105, yymsp[0].minor.yy172); } break; case 60: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy131, &yymsp[0].minor.yy113); } +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy617, &yymsp[0].minor.yy105); } break; case 61: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy113); } +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy105); } break; case 62: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy113, yymsp[0].minor.yy686); } +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy105, yymsp[0].minor.yy172); } break; case 63: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy131 = true; } +{ yymsp[-2].minor.yy617 = true; } break; case 64: /* not_exists_opt ::= */ case 66: /* exists_opt ::= */ yytestcase(yyruleno==66); case 232: /* analyze_opt ::= */ yytestcase(yyruleno==232); case 240: /* agg_func_opt ::= */ yytestcase(yyruleno==240); - case 388: /* set_quantifier_opt ::= */ yytestcase(yyruleno==388); -{ yymsp[1].minor.yy131 = false; } + case 389: /* set_quantifier_opt ::= */ yytestcase(yyruleno==389); +{ yymsp[1].minor.yy617 = false; } break; case 65: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy131 = true; } +{ yymsp[-1].minor.yy617 = true; } break; case 67: /* db_options ::= */ -{ yymsp[1].minor.yy686 = createDefaultDatabaseOptions(pCxt); } +{ yymsp[1].minor.yy172 = createDefaultDatabaseOptions(pCxt); } break; case 68: /* db_options ::= db_options BUFFER NK_INTEGER */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 69: /* db_options ::= db_options CACHELAST NK_INTEGER */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_CACHELAST, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_CACHELAST, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 70: /* db_options ::= db_options COMP NK_INTEGER */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_COMP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 71: /* db_options ::= db_options DAYS NK_INTEGER */ case 72: /* db_options ::= db_options DAYS NK_VARIABLE */ yytestcase(yyruleno==72); -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 73: /* db_options ::= db_options FSYNC NK_INTEGER */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 74: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 75: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 76: /* db_options ::= db_options KEEP integer_list */ case 77: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==77); -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_KEEP, yymsp[0].minor.yy670); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_KEEP, yymsp[0].minor.yy60); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 78: /* db_options ::= db_options PAGES NK_INTEGER */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 79: /* db_options ::= db_options PAGESIZE NK_INTEGER */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 80: /* db_options ::= db_options PRECISION NK_STRING */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 81: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 82: /* db_options ::= db_options STRICT NK_INTEGER */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_STRICT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_STRICT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 83: /* db_options ::= db_options WAL NK_INTEGER */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_WAL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 84: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 85: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 86: /* db_options ::= db_options RETENTIONS retention_list */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_RETENTIONS, yymsp[0].minor.yy670); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_RETENTIONS, yymsp[0].minor.yy60); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 87: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ -{ yylhsminor.yy686 = setDatabaseOption(pCxt, yymsp[-2].minor.yy686, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setDatabaseOption(pCxt, yymsp[-2].minor.yy172, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 88: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy686 = createAlterDatabaseOptions(pCxt); yylhsminor.yy686 = setAlterDatabaseOption(pCxt, yylhsminor.yy686, &yymsp[0].minor.yy53); } - yymsp[0].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createAlterDatabaseOptions(pCxt); yylhsminor.yy172 = setAlterDatabaseOption(pCxt, yylhsminor.yy172, &yymsp[0].minor.yy609); } + yymsp[0].minor.yy172 = yylhsminor.yy172; break; case 89: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy686 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy686, &yymsp[0].minor.yy53); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy172, &yymsp[0].minor.yy609); } + yymsp[-1].minor.yy172 = yylhsminor.yy172; break; case 90: /* alter_db_option ::= BUFFER NK_INTEGER */ -{ yymsp[-1].minor.yy53.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy609.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy609.val = yymsp[0].minor.yy0; } break; case 91: /* alter_db_option ::= CACHELAST NK_INTEGER */ -{ yymsp[-1].minor.yy53.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy609.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy609.val = yymsp[0].minor.yy0; } break; case 92: /* alter_db_option ::= FSYNC NK_INTEGER */ -{ yymsp[-1].minor.yy53.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy609.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy609.val = yymsp[0].minor.yy0; } break; case 93: /* alter_db_option ::= KEEP integer_list */ case 94: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==94); -{ yymsp[-1].minor.yy53.type = DB_OPTION_KEEP; yymsp[-1].minor.yy53.pList = yymsp[0].minor.yy670; } +{ yymsp[-1].minor.yy609.type = DB_OPTION_KEEP; yymsp[-1].minor.yy609.pList = yymsp[0].minor.yy60; } break; case 95: /* alter_db_option ::= PAGES NK_INTEGER */ -{ yymsp[-1].minor.yy53.type = DB_OPTION_PAGES; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy609.type = DB_OPTION_PAGES; yymsp[-1].minor.yy609.val = yymsp[0].minor.yy0; } break; case 96: /* alter_db_option ::= REPLICA NK_INTEGER */ -{ yymsp[-1].minor.yy53.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy609.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy609.val = yymsp[0].minor.yy0; } break; case 97: /* alter_db_option ::= STRICT NK_INTEGER */ -{ yymsp[-1].minor.yy53.type = DB_OPTION_STRICT; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy609.type = DB_OPTION_STRICT; yymsp[-1].minor.yy609.val = yymsp[0].minor.yy0; } break; case 98: /* alter_db_option ::= WAL NK_INTEGER */ -{ yymsp[-1].minor.yy53.type = DB_OPTION_WAL; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy609.type = DB_OPTION_WAL; yymsp[-1].minor.yy609.val = yymsp[0].minor.yy0; } break; case 99: /* integer_list ::= NK_INTEGER */ -{ yylhsminor.yy670 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy670 = yylhsminor.yy670; +{ yylhsminor.yy60 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy60 = yylhsminor.yy60; break; case 100: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ case 259: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==259); -{ yylhsminor.yy670 = addNodeToList(pCxt, yymsp[-2].minor.yy670, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy670 = yylhsminor.yy670; +{ yylhsminor.yy60 = addNodeToList(pCxt, yymsp[-2].minor.yy60, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy60 = yylhsminor.yy60; break; case 101: /* variable_list ::= NK_VARIABLE */ -{ yylhsminor.yy670 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy670 = yylhsminor.yy670; +{ yylhsminor.yy60 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy60 = yylhsminor.yy60; break; case 102: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ -{ yylhsminor.yy670 = addNodeToList(pCxt, yymsp[-2].minor.yy670, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy670 = yylhsminor.yy670; +{ yylhsminor.yy60 = addNodeToList(pCxt, yymsp[-2].minor.yy60, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy60 = yylhsminor.yy60; break; case 103: /* retention_list ::= retention */ case 123: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==123); @@ -3900,242 +3436,242 @@ static YYACTIONTYPE yy_reduce( case 173: /* col_name_list ::= col_name */ yytestcase(yyruleno==173); case 211: /* func_name_list ::= func_name */ yytestcase(yyruleno==211); case 220: /* func_list ::= func */ yytestcase(yyruleno==220); - case 284: /* literal_list ::= signed_literal */ yytestcase(yyruleno==284); - case 338: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==338); - case 393: /* select_sublist ::= select_item */ yytestcase(yyruleno==393); - case 442: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==442); -{ yylhsminor.yy670 = createNodeList(pCxt, yymsp[0].minor.yy686); } - yymsp[0].minor.yy670 = yylhsminor.yy670; + case 285: /* literal_list ::= signed_literal */ yytestcase(yyruleno==285); + case 339: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==339); + case 394: /* select_sublist ::= select_item */ yytestcase(yyruleno==394); + case 443: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==443); +{ yylhsminor.yy60 = createNodeList(pCxt, yymsp[0].minor.yy172); } + yymsp[0].minor.yy60 = yylhsminor.yy60; break; case 104: /* retention_list ::= retention_list NK_COMMA retention */ case 134: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==134); case 174: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==174); case 212: /* func_name_list ::= func_name_list NK_COMMA func_name */ yytestcase(yyruleno==212); case 221: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==221); - case 285: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==285); - case 339: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==339); - case 394: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==394); - case 443: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==443); -{ yylhsminor.yy670 = addNodeToList(pCxt, yymsp[-2].minor.yy670, yymsp[0].minor.yy686); } - yymsp[-2].minor.yy670 = yylhsminor.yy670; + case 286: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==286); + case 340: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==340); + case 395: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==395); + case 444: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==444); +{ yylhsminor.yy60 = addNodeToList(pCxt, yymsp[-2].minor.yy60, yymsp[0].minor.yy172); } + yymsp[-2].minor.yy60 = yylhsminor.yy60; break; case 105: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -{ yylhsminor.yy686 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 106: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ case 108: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==108); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy131, yymsp[-5].minor.yy686, yymsp[-3].minor.yy670, yymsp[-1].minor.yy670, yymsp[0].minor.yy686); } +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy617, yymsp[-5].minor.yy172, yymsp[-3].minor.yy60, yymsp[-1].minor.yy60, yymsp[0].minor.yy172); } break; case 107: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy670); } +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy60); } break; case 109: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy670); } +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy60); } break; case 110: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy131, yymsp[0].minor.yy686); } +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy617, yymsp[0].minor.yy172); } break; case 111: /* cmd ::= ALTER TABLE alter_table_clause */ case 112: /* cmd ::= ALTER STABLE alter_table_clause */ yytestcase(yyruleno==112); - case 261: /* cmd ::= query_expression */ yytestcase(yyruleno==261); -{ pCxt->pRootNode = yymsp[0].minor.yy686; } + case 262: /* cmd ::= query_expression */ yytestcase(yyruleno==262); +{ pCxt->pRootNode = yymsp[0].minor.yy172; } break; case 113: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy686 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy686, yymsp[0].minor.yy686); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy172, yymsp[0].minor.yy172); } + yymsp[-1].minor.yy172 = yylhsminor.yy172; break; case 114: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy686 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy686, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy113, yymsp[0].minor.yy490); } - yymsp[-4].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy172, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy105, yymsp[0].minor.yy248); } + yymsp[-4].minor.yy172 = yylhsminor.yy172; break; case 115: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy686 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy686, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy113); } - yymsp[-3].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy172, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy105); } + yymsp[-3].minor.yy172 = yylhsminor.yy172; break; case 116: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy686 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy686, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy113, yymsp[0].minor.yy490); } - yymsp[-4].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy172, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy105, yymsp[0].minor.yy248); } + yymsp[-4].minor.yy172 = yylhsminor.yy172; break; case 117: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy686 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy686, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy113, &yymsp[0].minor.yy113); } - yymsp[-4].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy172, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy105, &yymsp[0].minor.yy105); } + yymsp[-4].minor.yy172 = yylhsminor.yy172; break; case 118: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy686 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy686, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy113, yymsp[0].minor.yy490); } - yymsp[-4].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy172, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy105, yymsp[0].minor.yy248); } + yymsp[-4].minor.yy172 = yylhsminor.yy172; break; case 119: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy686 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy686, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy113); } - yymsp[-3].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy172, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy105); } + yymsp[-3].minor.yy172 = yylhsminor.yy172; break; case 120: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy686 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy686, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy113, yymsp[0].minor.yy490); } - yymsp[-4].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy172, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy105, yymsp[0].minor.yy248); } + yymsp[-4].minor.yy172 = yylhsminor.yy172; break; case 121: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy686 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy686, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy113, &yymsp[0].minor.yy113); } - yymsp[-4].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy172, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy105, &yymsp[0].minor.yy105); } + yymsp[-4].minor.yy172 = yylhsminor.yy172; break; case 122: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ -{ yylhsminor.yy686 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy686, &yymsp[-2].minor.yy113, yymsp[0].minor.yy686); } - yymsp[-5].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy172, &yymsp[-2].minor.yy105, yymsp[0].minor.yy172); } + yymsp[-5].minor.yy172 = yylhsminor.yy172; break; case 124: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ case 127: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==127); -{ yylhsminor.yy670 = addNodeToList(pCxt, yymsp[-1].minor.yy670, yymsp[0].minor.yy686); } - yymsp[-1].minor.yy670 = yylhsminor.yy670; +{ yylhsminor.yy60 = addNodeToList(pCxt, yymsp[-1].minor.yy60, yymsp[0].minor.yy172); } + yymsp[-1].minor.yy60 = yylhsminor.yy60; break; case 125: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options */ -{ yylhsminor.yy686 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy131, yymsp[-8].minor.yy686, yymsp[-6].minor.yy686, yymsp[-5].minor.yy670, yymsp[-2].minor.yy670, yymsp[0].minor.yy686); } - yymsp[-9].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy617, yymsp[-8].minor.yy172, yymsp[-6].minor.yy172, yymsp[-5].minor.yy60, yymsp[-2].minor.yy60, yymsp[0].minor.yy172); } + yymsp[-9].minor.yy172 = yylhsminor.yy172; break; case 128: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy686 = createDropTableClause(pCxt, yymsp[-1].minor.yy131, yymsp[0].minor.yy686); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createDropTableClause(pCxt, yymsp[-1].minor.yy617, yymsp[0].minor.yy172); } + yymsp[-1].minor.yy172 = yylhsminor.yy172; break; case 129: /* specific_tags_opt ::= */ case 160: /* tags_def_opt ::= */ yytestcase(yyruleno==160); - case 401: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==401); - case 418: /* group_by_clause_opt ::= */ yytestcase(yyruleno==418); - case 430: /* order_by_clause_opt ::= */ yytestcase(yyruleno==430); -{ yymsp[1].minor.yy670 = NULL; } + case 402: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==402); + case 419: /* group_by_clause_opt ::= */ yytestcase(yyruleno==419); + case 431: /* order_by_clause_opt ::= */ yytestcase(yyruleno==431); +{ yymsp[1].minor.yy60 = NULL; } break; case 130: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ -{ yymsp[-2].minor.yy670 = yymsp[-1].minor.yy670; } +{ yymsp[-2].minor.yy60 = yymsp[-1].minor.yy60; } break; case 131: /* full_table_name ::= table_name */ -{ yylhsminor.yy686 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy113, NULL); } - yymsp[0].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy105, NULL); } + yymsp[0].minor.yy172 = yylhsminor.yy172; break; case 132: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy686 = createRealTableNode(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113, NULL); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createRealTableNode(pCxt, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy105, NULL); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 135: /* column_def ::= column_name type_name */ -{ yylhsminor.yy686 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy113, yymsp[0].minor.yy490, NULL); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy105, yymsp[0].minor.yy248, NULL); } + yymsp[-1].minor.yy172 = yylhsminor.yy172; break; case 136: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy686 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy113, yymsp[-2].minor.yy490, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy105, yymsp[-2].minor.yy248, &yymsp[0].minor.yy0); } + yymsp[-3].minor.yy172 = yylhsminor.yy172; break; case 137: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy490 = createDataType(TSDB_DATA_TYPE_BOOL); } +{ yymsp[0].minor.yy248 = createDataType(TSDB_DATA_TYPE_BOOL); } break; case 138: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy490 = createDataType(TSDB_DATA_TYPE_TINYINT); } +{ yymsp[0].minor.yy248 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; case 139: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy490 = createDataType(TSDB_DATA_TYPE_SMALLINT); } +{ yymsp[0].minor.yy248 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; case 140: /* type_name ::= INT */ case 141: /* type_name ::= INTEGER */ yytestcase(yyruleno==141); -{ yymsp[0].minor.yy490 = createDataType(TSDB_DATA_TYPE_INT); } +{ yymsp[0].minor.yy248 = createDataType(TSDB_DATA_TYPE_INT); } break; case 142: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy490 = createDataType(TSDB_DATA_TYPE_BIGINT); } +{ yymsp[0].minor.yy248 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; case 143: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy490 = createDataType(TSDB_DATA_TYPE_FLOAT); } +{ yymsp[0].minor.yy248 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; case 144: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy490 = createDataType(TSDB_DATA_TYPE_DOUBLE); } +{ yymsp[0].minor.yy248 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; case 145: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy490 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy248 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; case 146: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy490 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } +{ yymsp[0].minor.yy248 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; case 147: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy490 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy248 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; case 148: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy490 = createDataType(TSDB_DATA_TYPE_UTINYINT); } +{ yymsp[-1].minor.yy248 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; case 149: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy490 = createDataType(TSDB_DATA_TYPE_USMALLINT); } +{ yymsp[-1].minor.yy248 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; case 150: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy490 = createDataType(TSDB_DATA_TYPE_UINT); } +{ yymsp[-1].minor.yy248 = createDataType(TSDB_DATA_TYPE_UINT); } break; case 151: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy490 = createDataType(TSDB_DATA_TYPE_UBIGINT); } +{ yymsp[-1].minor.yy248 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; case 152: /* type_name ::= JSON */ -{ yymsp[0].minor.yy490 = createDataType(TSDB_DATA_TYPE_JSON); } +{ yymsp[0].minor.yy248 = createDataType(TSDB_DATA_TYPE_JSON); } break; case 153: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy490 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy248 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; case 154: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy490 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } +{ yymsp[0].minor.yy248 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; case 155: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy490 = createDataType(TSDB_DATA_TYPE_BLOB); } +{ yymsp[0].minor.yy248 = createDataType(TSDB_DATA_TYPE_BLOB); } break; case 156: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy490 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy248 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; case 157: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy490 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[0].minor.yy248 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 158: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy490 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[-3].minor.yy248 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 159: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy490 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[-5].minor.yy248 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 161: /* tags_def_opt ::= tags_def */ - case 337: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==337); - case 392: /* select_list ::= select_sublist */ yytestcase(yyruleno==392); -{ yylhsminor.yy670 = yymsp[0].minor.yy670; } - yymsp[0].minor.yy670 = yylhsminor.yy670; + case 338: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==338); + case 393: /* select_list ::= select_sublist */ yytestcase(yyruleno==393); +{ yylhsminor.yy60 = yymsp[0].minor.yy60; } + yymsp[0].minor.yy60 = yylhsminor.yy60; break; case 162: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ -{ yymsp[-3].minor.yy670 = yymsp[-1].minor.yy670; } +{ yymsp[-3].minor.yy60 = yymsp[-1].minor.yy60; } break; case 163: /* table_options ::= */ -{ yymsp[1].minor.yy686 = createDefaultTableOptions(pCxt); } +{ yymsp[1].minor.yy172 = createDefaultTableOptions(pCxt); } break; case 164: /* table_options ::= table_options COMMENT NK_STRING */ -{ yylhsminor.yy686 = setTableOption(pCxt, yymsp[-2].minor.yy686, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setTableOption(pCxt, yymsp[-2].minor.yy172, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 165: /* table_options ::= table_options FILE_FACTOR NK_FLOAT */ -{ yylhsminor.yy686 = setTableOption(pCxt, yymsp[-2].minor.yy686, TABLE_OPTION_FILE_FACTOR, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setTableOption(pCxt, yymsp[-2].minor.yy172, TABLE_OPTION_FILE_FACTOR, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 166: /* table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ -{ yylhsminor.yy686 = setTableOption(pCxt, yymsp[-4].minor.yy686, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy670); } - yymsp[-4].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setTableOption(pCxt, yymsp[-4].minor.yy172, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy60); } + yymsp[-4].minor.yy172 = yylhsminor.yy172; break; case 167: /* table_options ::= table_options TTL NK_INTEGER */ -{ yylhsminor.yy686 = setTableOption(pCxt, yymsp[-2].minor.yy686, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setTableOption(pCxt, yymsp[-2].minor.yy172, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 168: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ yylhsminor.yy686 = setTableOption(pCxt, yymsp[-4].minor.yy686, TABLE_OPTION_SMA, yymsp[-1].minor.yy670); } - yymsp[-4].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setTableOption(pCxt, yymsp[-4].minor.yy172, TABLE_OPTION_SMA, yymsp[-1].minor.yy60); } + yymsp[-4].minor.yy172 = yylhsminor.yy172; break; case 169: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy686 = createAlterTableOptions(pCxt); yylhsminor.yy686 = setTableOption(pCxt, yylhsminor.yy686, yymsp[0].minor.yy53.type, &yymsp[0].minor.yy53.val); } - yymsp[0].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createAlterTableOptions(pCxt); yylhsminor.yy172 = setTableOption(pCxt, yylhsminor.yy172, yymsp[0].minor.yy609.type, &yymsp[0].minor.yy609.val); } + yymsp[0].minor.yy172 = yylhsminor.yy172; break; case 170: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy686 = setTableOption(pCxt, yymsp[-1].minor.yy686, yymsp[0].minor.yy53.type, &yymsp[0].minor.yy53.val); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setTableOption(pCxt, yymsp[-1].minor.yy172, yymsp[0].minor.yy609.type, &yymsp[0].minor.yy609.val); } + yymsp[-1].minor.yy172 = yylhsminor.yy172; break; case 171: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy53.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy609.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy609.val = yymsp[0].minor.yy0; } break; case 172: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy53.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } +{ yymsp[-1].minor.yy609.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy609.val = yymsp[0].minor.yy0; } break; case 175: /* col_name ::= column_name */ -{ yylhsminor.yy686 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy113); } - yymsp[0].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy105); } + yymsp[0].minor.yy172 = yylhsminor.yy172; break; case 176: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL, NULL); } @@ -4147,13 +3683,13 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL, NULL); } break; case 179: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy686, yymsp[0].minor.yy686); } +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy172, yymsp[0].minor.yy172); } break; case 180: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy686, yymsp[0].minor.yy686); } +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy172, yymsp[0].minor.yy172); } break; case 181: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy686, NULL); } +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy172, NULL); } break; case 182: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT, NULL, NULL); } @@ -4168,7 +3704,7 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT, NULL, NULL); } break; case 186: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[-1].minor.yy686, yymsp[0].minor.yy686); } +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[-1].minor.yy172, yymsp[0].minor.yy172); } break; case 187: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT, NULL, NULL); } @@ -4187,13 +3723,13 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCE_STMT, NULL, NULL); } break; case 193: /* cmd ::= SHOW CREATE DATABASE db_name */ -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy113); } +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy105); } break; case 194: /* cmd ::= SHOW CREATE TABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy686); } +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy172); } break; case 195: /* cmd ::= SHOW CREATE STABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy686); } +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy172); } break; case 196: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT, NULL, NULL); } @@ -4221,139 +3757,139 @@ static YYACTIONTYPE yy_reduce( break; case 204: /* db_name_cond_opt ::= */ case 209: /* from_db_opt ::= */ yytestcase(yyruleno==209); -{ yymsp[1].minor.yy686 = createDefaultDatabaseCondValue(pCxt); } +{ yymsp[1].minor.yy172 = createDefaultDatabaseCondValue(pCxt); } break; case 205: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy686 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy113); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy105); } + yymsp[-1].minor.yy172 = yylhsminor.yy172; break; case 206: /* like_pattern_opt ::= */ case 217: /* index_options ::= */ yytestcase(yyruleno==217); case 246: /* into_opt ::= */ yytestcase(yyruleno==246); - case 399: /* where_clause_opt ::= */ yytestcase(yyruleno==399); - case 403: /* twindow_clause_opt ::= */ yytestcase(yyruleno==403); - case 408: /* sliding_opt ::= */ yytestcase(yyruleno==408); - case 410: /* fill_opt ::= */ yytestcase(yyruleno==410); - case 422: /* having_clause_opt ::= */ yytestcase(yyruleno==422); - case 432: /* slimit_clause_opt ::= */ yytestcase(yyruleno==432); - case 436: /* limit_clause_opt ::= */ yytestcase(yyruleno==436); -{ yymsp[1].minor.yy686 = NULL; } + case 400: /* where_clause_opt ::= */ yytestcase(yyruleno==400); + case 404: /* twindow_clause_opt ::= */ yytestcase(yyruleno==404); + case 409: /* sliding_opt ::= */ yytestcase(yyruleno==409); + case 411: /* fill_opt ::= */ yytestcase(yyruleno==411); + case 423: /* having_clause_opt ::= */ yytestcase(yyruleno==423); + case 433: /* slimit_clause_opt ::= */ yytestcase(yyruleno==433); + case 437: /* limit_clause_opt ::= */ yytestcase(yyruleno==437); +{ yymsp[1].minor.yy172 = NULL; } break; case 207: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy686 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy172 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; case 208: /* table_name_cond ::= table_name */ -{ yylhsminor.yy686 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy113); } - yymsp[0].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy105); } + yymsp[0].minor.yy172 = yylhsminor.yy172; break; case 210: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy686 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy113); } +{ yymsp[-1].minor.yy172 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy105); } break; case 213: /* func_name ::= function_name */ -{ yylhsminor.yy686 = createFunctionNode(pCxt, &yymsp[0].minor.yy113, NULL); } - yymsp[0].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createFunctionNode(pCxt, &yymsp[0].minor.yy105, NULL); } + yymsp[0].minor.yy172 = yylhsminor.yy172; break; case 214: /* cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy131, &yymsp[-3].minor.yy113, &yymsp[-1].minor.yy113, NULL, yymsp[0].minor.yy686); } +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy617, &yymsp[-3].minor.yy105, &yymsp[-1].minor.yy105, NULL, yymsp[0].minor.yy172); } break; case 215: /* cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, yymsp[-6].minor.yy131, &yymsp[-5].minor.yy113, &yymsp[-3].minor.yy113, yymsp[-1].minor.yy670, NULL); } +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, yymsp[-6].minor.yy617, &yymsp[-5].minor.yy105, &yymsp[-3].minor.yy105, yymsp[-1].minor.yy60, NULL); } break; case 216: /* cmd ::= DROP INDEX exists_opt index_name ON table_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-3].minor.yy131, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113); } +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-3].minor.yy617, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy105); } break; case 218: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ -{ yymsp[-8].minor.yy686 = createIndexOption(pCxt, yymsp[-6].minor.yy670, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), NULL, yymsp[0].minor.yy686); } +{ yymsp[-8].minor.yy172 = createIndexOption(pCxt, yymsp[-6].minor.yy60, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), NULL, yymsp[0].minor.yy172); } break; case 219: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ -{ yymsp[-10].minor.yy686 = createIndexOption(pCxt, yymsp[-8].minor.yy670, releaseRawExprNode(pCxt, yymsp[-4].minor.yy686), releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), yymsp[0].minor.yy686); } +{ yymsp[-10].minor.yy172 = createIndexOption(pCxt, yymsp[-8].minor.yy60, releaseRawExprNode(pCxt, yymsp[-4].minor.yy172), releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), yymsp[0].minor.yy172); } break; case 222: /* func ::= function_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy686 = createFunctionNode(pCxt, &yymsp[-3].minor.yy113, yymsp[-1].minor.yy670); } - yymsp[-3].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = createFunctionNode(pCxt, &yymsp[-3].minor.yy105, yymsp[-1].minor.yy60); } + yymsp[-3].minor.yy172 = yylhsminor.yy172; break; case 223: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy131, &yymsp[-2].minor.yy113, yymsp[0].minor.yy686, NULL, NULL); } +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy617, &yymsp[-2].minor.yy105, yymsp[0].minor.yy172, NULL, NULL); } break; case 224: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-4].minor.yy131, &yymsp[-3].minor.yy113, NULL, &yymsp[0].minor.yy113, NULL); } +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-4].minor.yy617, &yymsp[-3].minor.yy105, NULL, &yymsp[0].minor.yy105, NULL); } break; case 225: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-4].minor.yy131, &yymsp[-3].minor.yy113, NULL, NULL, yymsp[0].minor.yy686); } +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-4].minor.yy617, &yymsp[-3].minor.yy105, NULL, NULL, yymsp[0].minor.yy172); } break; case 226: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy131, &yymsp[0].minor.yy113); } +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy617, &yymsp[0].minor.yy105); } break; case 227: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ -{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy131, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113); } +{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy617, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy105); } break; case 228: /* cmd ::= DESC full_table_name */ case 229: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==229); -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy686); } +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy172); } break; case 230: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; case 231: /* cmd ::= EXPLAIN analyze_opt explain_options query_expression */ -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy131, yymsp[-1].minor.yy686, yymsp[0].minor.yy686); } +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy617, yymsp[-1].minor.yy172, yymsp[0].minor.yy172); } break; case 233: /* analyze_opt ::= ANALYZE */ case 241: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==241); - case 389: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==389); -{ yymsp[0].minor.yy131 = true; } + case 390: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==390); +{ yymsp[0].minor.yy617 = true; } break; case 234: /* explain_options ::= */ -{ yymsp[1].minor.yy686 = createDefaultExplainOptions(pCxt); } +{ yymsp[1].minor.yy172 = createDefaultExplainOptions(pCxt); } break; case 235: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -{ yylhsminor.yy686 = setExplainVerbose(pCxt, yymsp[-2].minor.yy686, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setExplainVerbose(pCxt, yymsp[-2].minor.yy172, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 236: /* explain_options ::= explain_options RATIO NK_FLOAT */ -{ yylhsminor.yy686 = setExplainRatio(pCxt, yymsp[-2].minor.yy686, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ yylhsminor.yy172 = setExplainRatio(pCxt, yymsp[-2].minor.yy172, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 237: /* cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ -{ pCxt->pRootNode = createCompactStmt(pCxt, yymsp[-1].minor.yy670); } +{ pCxt->pRootNode = createCompactStmt(pCxt, yymsp[-1].minor.yy60); } break; case 238: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy131, yymsp[-8].minor.yy131, &yymsp[-5].minor.yy113, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy490, yymsp[0].minor.yy550); } +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy617, yymsp[-8].minor.yy617, &yymsp[-5].minor.yy105, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy248, yymsp[0].minor.yy140); } break; case 239: /* cmd ::= DROP FUNCTION exists_opt function_name */ -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy131, &yymsp[0].minor.yy113); } +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy617, &yymsp[0].minor.yy105); } break; case 242: /* bufsize_opt ::= */ -{ yymsp[1].minor.yy550 = 0; } +{ yymsp[1].minor.yy140 = 0; } break; case 243: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ -{ yymsp[-1].minor.yy550 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy140 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } break; case 244: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-5].minor.yy131, &yymsp[-4].minor.yy113, yymsp[-2].minor.yy686, yymsp[-3].minor.yy686, yymsp[0].minor.yy686); } +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-5].minor.yy617, &yymsp[-4].minor.yy105, yymsp[-2].minor.yy172, yymsp[-3].minor.yy172, yymsp[0].minor.yy172); } break; case 245: /* cmd ::= DROP STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy131, &yymsp[0].minor.yy113); } +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy617, &yymsp[0].minor.yy105); } break; case 247: /* into_opt ::= INTO full_table_name */ - case 370: /* from_clause ::= FROM table_reference_list */ yytestcase(yyruleno==370); - case 400: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==400); - case 423: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==423); -{ yymsp[-1].minor.yy686 = yymsp[0].minor.yy686; } + case 371: /* from_clause ::= FROM table_reference_list */ yytestcase(yyruleno==371); + case 401: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==401); + case 424: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==424); +{ yymsp[-1].minor.yy172 = yymsp[0].minor.yy172; } break; case 248: /* stream_options ::= */ -{ yymsp[1].minor.yy686 = createStreamOptions(pCxt); } +{ yymsp[1].minor.yy172 = createStreamOptions(pCxt); } break; case 249: /* stream_options ::= stream_options TRIGGER AT_ONCE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy686)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy686 = yymsp[-2].minor.yy686; } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ ((SStreamOptions*)yymsp[-2].minor.yy172)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy172 = yymsp[-2].minor.yy172; } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 250: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy686)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy686 = yymsp[-2].minor.yy686; } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ ((SStreamOptions*)yymsp[-2].minor.yy172)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy172 = yymsp[-2].minor.yy172; } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 251: /* stream_options ::= stream_options WATERMARK duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy686)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy686); yylhsminor.yy686 = yymsp[-2].minor.yy686; } - yymsp[-2].minor.yy686 = yylhsminor.yy686; +{ ((SStreamOptions*)yymsp[-2].minor.yy172)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy172); yylhsminor.yy172 = yymsp[-2].minor.yy172; } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; case 252: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } @@ -4368,527 +3904,530 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 256: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy670); } +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy60); } break; case 257: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; case 258: /* dnode_list ::= DNODE NK_INTEGER */ -{ yymsp[-1].minor.yy670 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } +{ yymsp[-1].minor.yy60 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } break; case 260: /* cmd ::= SYNCDB db_name REPLICA */ -{ pCxt->pRootNode = createSyncdbStmt(pCxt, &yymsp[-1].minor.yy113); } - break; - case 262: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy686 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy686 = yylhsminor.yy686; - break; - case 263: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy686 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy686 = yylhsminor.yy686; - break; - case 264: /* literal ::= NK_STRING */ -{ yylhsminor.yy686 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy686 = yylhsminor.yy686; - break; - case 265: /* literal ::= NK_BOOL */ -{ yylhsminor.yy686 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy686 = yylhsminor.yy686; - break; - case 266: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy686 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; - break; - case 267: /* literal ::= duration_literal */ - case 277: /* signed_literal ::= signed */ yytestcase(yyruleno==277); - case 297: /* expression ::= literal */ yytestcase(yyruleno==297); - case 298: /* expression ::= pseudo_column */ yytestcase(yyruleno==298); - case 299: /* expression ::= column_reference */ yytestcase(yyruleno==299); - case 300: /* expression ::= function_expression */ yytestcase(yyruleno==300); - case 301: /* expression ::= subquery */ yytestcase(yyruleno==301); - case 326: /* function_expression ::= literal_func */ yytestcase(yyruleno==326); - case 362: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==362); - case 366: /* boolean_primary ::= predicate */ yytestcase(yyruleno==366); - case 368: /* common_expression ::= expression */ yytestcase(yyruleno==368); - case 369: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==369); - case 371: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==371); - case 373: /* table_reference ::= table_primary */ yytestcase(yyruleno==373); - case 374: /* table_reference ::= joined_table */ yytestcase(yyruleno==374); - case 378: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==378); - case 425: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==425); - case 428: /* query_primary ::= query_specification */ yytestcase(yyruleno==428); -{ yylhsminor.yy686 = yymsp[0].minor.yy686; } - yymsp[0].minor.yy686 = yylhsminor.yy686; - break; - case 268: /* literal ::= NULL */ -{ yylhsminor.yy686 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy686 = yylhsminor.yy686; - break; - case 269: /* literal ::= NK_QUESTION */ -{ yylhsminor.yy686 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy686 = yylhsminor.yy686; - break; - case 270: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy686 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy686 = yylhsminor.yy686; - break; - case 271: /* signed ::= NK_INTEGER */ -{ yylhsminor.yy686 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy686 = yylhsminor.yy686; - break; - case 272: /* signed ::= NK_PLUS NK_INTEGER */ -{ yymsp[-1].minor.yy686 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - break; - case 273: /* signed ::= NK_MINUS NK_INTEGER */ +{ pCxt->pRootNode = createSyncdbStmt(pCxt, &yymsp[-1].minor.yy105); } + break; + case 261: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ +{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy172, yymsp[0].minor.yy172); } + break; + case 263: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy172 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy172 = yylhsminor.yy172; + break; + case 264: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy172 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy172 = yylhsminor.yy172; + break; + case 265: /* literal ::= NK_STRING */ +{ yylhsminor.yy172 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy172 = yylhsminor.yy172; + break; + case 266: /* literal ::= NK_BOOL */ +{ yylhsminor.yy172 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy172 = yylhsminor.yy172; + break; + case 267: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy172 = yylhsminor.yy172; + break; + case 268: /* literal ::= duration_literal */ + case 278: /* signed_literal ::= signed */ yytestcase(yyruleno==278); + case 298: /* expression ::= literal */ yytestcase(yyruleno==298); + case 299: /* expression ::= pseudo_column */ yytestcase(yyruleno==299); + case 300: /* expression ::= column_reference */ yytestcase(yyruleno==300); + case 301: /* expression ::= function_expression */ yytestcase(yyruleno==301); + case 302: /* expression ::= subquery */ yytestcase(yyruleno==302); + case 327: /* function_expression ::= literal_func */ yytestcase(yyruleno==327); + case 363: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==363); + case 367: /* boolean_primary ::= predicate */ yytestcase(yyruleno==367); + case 369: /* common_expression ::= expression */ yytestcase(yyruleno==369); + case 370: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==370); + case 372: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==372); + case 374: /* table_reference ::= table_primary */ yytestcase(yyruleno==374); + case 375: /* table_reference ::= joined_table */ yytestcase(yyruleno==375); + case 379: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==379); + case 426: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==426); + case 429: /* query_primary ::= query_specification */ yytestcase(yyruleno==429); +{ yylhsminor.yy172 = yymsp[0].minor.yy172; } + yymsp[0].minor.yy172 = yylhsminor.yy172; + break; + case 269: /* literal ::= NULL */ +{ yylhsminor.yy172 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy172 = yylhsminor.yy172; + break; + case 270: /* literal ::= NK_QUESTION */ +{ yylhsminor.yy172 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy172 = yylhsminor.yy172; + break; + case 271: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy172 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy172 = yylhsminor.yy172; + break; + case 272: /* signed ::= NK_INTEGER */ +{ yylhsminor.yy172 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy172 = yylhsminor.yy172; + break; + case 273: /* signed ::= NK_PLUS NK_INTEGER */ +{ yymsp[-1].minor.yy172 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + break; + case 274: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy686 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy172 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; + yymsp[-1].minor.yy172 = yylhsminor.yy172; break; - case 274: /* signed ::= NK_FLOAT */ -{ yylhsminor.yy686 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy686 = yylhsminor.yy686; + case 275: /* signed ::= NK_FLOAT */ +{ yylhsminor.yy172 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy172 = yylhsminor.yy172; break; - case 275: /* signed ::= NK_PLUS NK_FLOAT */ -{ yymsp[-1].minor.yy686 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + case 276: /* signed ::= NK_PLUS NK_FLOAT */ +{ yymsp[-1].minor.yy172 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; - case 276: /* signed ::= NK_MINUS NK_FLOAT */ + case 277: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy686 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy172 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; + yymsp[-1].minor.yy172 = yylhsminor.yy172; break; - case 278: /* signed_literal ::= NK_STRING */ -{ yylhsminor.yy686 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy686 = yylhsminor.yy686; + case 279: /* signed_literal ::= NK_STRING */ +{ yylhsminor.yy172 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy172 = yylhsminor.yy172; break; - case 279: /* signed_literal ::= NK_BOOL */ -{ yylhsminor.yy686 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy686 = yylhsminor.yy686; + case 280: /* signed_literal ::= NK_BOOL */ +{ yylhsminor.yy172 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy172 = yylhsminor.yy172; break; - case 280: /* signed_literal ::= TIMESTAMP NK_STRING */ -{ yymsp[-1].minor.yy686 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + case 281: /* signed_literal ::= TIMESTAMP NK_STRING */ +{ yymsp[-1].minor.yy172 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } break; - case 281: /* signed_literal ::= duration_literal */ - case 283: /* signed_literal ::= literal_func */ yytestcase(yyruleno==283); - case 340: /* star_func_para ::= expression */ yytestcase(yyruleno==340); - case 395: /* select_item ::= common_expression */ yytestcase(yyruleno==395); - case 441: /* search_condition ::= common_expression */ yytestcase(yyruleno==441); -{ yylhsminor.yy686 = releaseRawExprNode(pCxt, yymsp[0].minor.yy686); } - yymsp[0].minor.yy686 = yylhsminor.yy686; + case 282: /* signed_literal ::= duration_literal */ + case 284: /* signed_literal ::= literal_func */ yytestcase(yyruleno==284); + case 341: /* star_func_para ::= expression */ yytestcase(yyruleno==341); + case 396: /* select_item ::= common_expression */ yytestcase(yyruleno==396); + case 442: /* search_condition ::= common_expression */ yytestcase(yyruleno==442); +{ yylhsminor.yy172 = releaseRawExprNode(pCxt, yymsp[0].minor.yy172); } + yymsp[0].minor.yy172 = yylhsminor.yy172; break; - case 282: /* signed_literal ::= NULL */ -{ yylhsminor.yy686 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy686 = yylhsminor.yy686; + case 283: /* signed_literal ::= NULL */ +{ yylhsminor.yy172 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy172 = yylhsminor.yy172; break; - case 302: /* expression ::= NK_LP expression NK_RP */ - case 367: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==367); -{ yylhsminor.yy686 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy686)); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + case 303: /* expression ::= NK_LP expression NK_RP */ + case 368: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==368); +{ yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy172)); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 303: /* expression ::= NK_PLUS expression */ + case 304: /* expression ::= NK_PLUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy686)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy172)); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; + yymsp[-1].minor.yy172 = yylhsminor.yy172; break; - case 304: /* expression ::= NK_MINUS expression */ + case 305: /* expression ::= NK_MINUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy686), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy172), NULL)); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; + yymsp[-1].minor.yy172 = yylhsminor.yy172; break; - case 305: /* expression ::= expression NK_PLUS expression */ + case 306: /* expression ::= expression NK_PLUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy686); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), releaseRawExprNode(pCxt, yymsp[0].minor.yy686))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 306: /* expression ::= expression NK_MINUS expression */ + case 307: /* expression ::= expression NK_MINUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy686); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), releaseRawExprNode(pCxt, yymsp[0].minor.yy686))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 307: /* expression ::= expression NK_STAR expression */ + case 308: /* expression ::= expression NK_STAR expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy686); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), releaseRawExprNode(pCxt, yymsp[0].minor.yy686))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 308: /* expression ::= expression NK_SLASH expression */ + case 309: /* expression ::= expression NK_SLASH expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy686); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), releaseRawExprNode(pCxt, yymsp[0].minor.yy686))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 309: /* expression ::= expression NK_REM expression */ + case 310: /* expression ::= expression NK_REM expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy686); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), releaseRawExprNode(pCxt, yymsp[0].minor.yy686))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 310: /* expression ::= column_reference NK_ARROW NK_STRING */ + case 311: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; - break; - case 311: /* expression_list ::= expression */ -{ yylhsminor.yy670 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy686)); } - yymsp[0].minor.yy670 = yylhsminor.yy670; - break; - case 312: /* expression_list ::= expression_list NK_COMMA expression */ -{ yylhsminor.yy670 = addNodeToList(pCxt, yymsp[-2].minor.yy670, releaseRawExprNode(pCxt, yymsp[0].minor.yy686)); } - yymsp[-2].minor.yy670 = yylhsminor.yy670; - break; - case 313: /* column_reference ::= column_name */ -{ yylhsminor.yy686 = createRawExprNode(pCxt, &yymsp[0].minor.yy113, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy113)); } - yymsp[0].minor.yy686 = yylhsminor.yy686; - break; - case 314: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy686 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113, createColumnNode(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113)); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; - break; - case 315: /* pseudo_column ::= ROWTS */ - case 316: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==316); - case 318: /* pseudo_column ::= QSTARTTS */ yytestcase(yyruleno==318); - case 319: /* pseudo_column ::= QENDTS */ yytestcase(yyruleno==319); - case 320: /* pseudo_column ::= WSTARTTS */ yytestcase(yyruleno==320); - case 321: /* pseudo_column ::= WENDTS */ yytestcase(yyruleno==321); - case 322: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==322); - case 328: /* literal_func ::= NOW */ yytestcase(yyruleno==328); -{ yylhsminor.yy686 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } - yymsp[0].minor.yy686 = yylhsminor.yy686; - break; - case 317: /* pseudo_column ::= table_name NK_DOT TBNAME */ -{ yylhsminor.yy686 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy113)))); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; - break; - case 323: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 324: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==324); -{ yylhsminor.yy686 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy113, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy113, yymsp[-1].minor.yy670)); } - yymsp[-3].minor.yy686 = yylhsminor.yy686; - break; - case 325: /* function_expression ::= CAST NK_LP expression AS type_name NK_RP */ -{ yylhsminor.yy686 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy686), yymsp[-1].minor.yy490)); } - yymsp[-5].minor.yy686 = yylhsminor.yy686; - break; - case 327: /* literal_func ::= noarg_func NK_LP NK_RP */ -{ yylhsminor.yy686 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy113, NULL)); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; - break; - case 336: /* star_func_para_list ::= NK_STAR */ -{ yylhsminor.yy670 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy670 = yylhsminor.yy670; - break; - case 341: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 398: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==398); -{ yylhsminor.yy686 = createColumnNode(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; - break; - case 342: /* predicate ::= expression compare_op expression */ - case 347: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==347); + yymsp[-2].minor.yy172 = yylhsminor.yy172; + break; + case 312: /* expression_list ::= expression */ +{ yylhsminor.yy60 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy172)); } + yymsp[0].minor.yy60 = yylhsminor.yy60; + break; + case 313: /* expression_list ::= expression_list NK_COMMA expression */ +{ yylhsminor.yy60 = addNodeToList(pCxt, yymsp[-2].minor.yy60, releaseRawExprNode(pCxt, yymsp[0].minor.yy172)); } + yymsp[-2].minor.yy60 = yylhsminor.yy60; + break; + case 314: /* column_reference ::= column_name */ +{ yylhsminor.yy172 = createRawExprNode(pCxt, &yymsp[0].minor.yy105, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy105)); } + yymsp[0].minor.yy172 = yylhsminor.yy172; + break; + case 315: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy105, createColumnNode(pCxt, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy105)); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; + break; + case 316: /* pseudo_column ::= ROWTS */ + case 317: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==317); + case 319: /* pseudo_column ::= QSTARTTS */ yytestcase(yyruleno==319); + case 320: /* pseudo_column ::= QENDTS */ yytestcase(yyruleno==320); + case 321: /* pseudo_column ::= WSTARTTS */ yytestcase(yyruleno==321); + case 322: /* pseudo_column ::= WENDTS */ yytestcase(yyruleno==322); + case 323: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==323); + case 329: /* literal_func ::= NOW */ yytestcase(yyruleno==329); +{ yylhsminor.yy172 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy172 = yylhsminor.yy172; + break; + case 318: /* pseudo_column ::= table_name NK_DOT TBNAME */ +{ yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy105)))); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; + break; + case 324: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 325: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==325); +{ yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy105, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy105, yymsp[-1].minor.yy60)); } + yymsp[-3].minor.yy172 = yylhsminor.yy172; + break; + case 326: /* function_expression ::= CAST NK_LP expression AS type_name NK_RP */ +{ yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy172), yymsp[-1].minor.yy248)); } + yymsp[-5].minor.yy172 = yylhsminor.yy172; + break; + case 328: /* literal_func ::= noarg_func NK_LP NK_RP */ +{ yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy105, NULL)); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; + break; + case 337: /* star_func_para_list ::= NK_STAR */ +{ yylhsminor.yy60 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy60 = yylhsminor.yy60; + break; + case 342: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 399: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==399); +{ yylhsminor.yy172 = createColumnNode(pCxt, &yymsp[-2].minor.yy105, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; + break; + case 343: /* predicate ::= expression compare_op expression */ + case 348: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==348); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy686); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy632, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), releaseRawExprNode(pCxt, yymsp[0].minor.yy686))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy572, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 343: /* predicate ::= expression BETWEEN expression AND expression */ + case 344: /* predicate ::= expression BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy686); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy686), releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), releaseRawExprNode(pCxt, yymsp[0].minor.yy686))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy172); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy172), releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); } - yymsp[-4].minor.yy686 = yylhsminor.yy686; + yymsp[-4].minor.yy172 = yylhsminor.yy172; break; - case 344: /* predicate ::= expression NOT BETWEEN expression AND expression */ + case 345: /* predicate ::= expression NOT BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy686); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy686), releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), releaseRawExprNode(pCxt, yymsp[0].minor.yy686))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy172); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy172), releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); } - yymsp[-5].minor.yy686 = yylhsminor.yy686; + yymsp[-5].minor.yy172 = yylhsminor.yy172; break; - case 345: /* predicate ::= expression IS NULL */ + case 346: /* predicate ::= expression IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), NULL)); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 346: /* predicate ::= expression IS NOT NULL */ + case 347: /* predicate ::= expression IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy686), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy172), NULL)); } - yymsp[-3].minor.yy686 = yylhsminor.yy686; + yymsp[-3].minor.yy172 = yylhsminor.yy172; break; - case 348: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy632 = OP_TYPE_LOWER_THAN; } + case 349: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy572 = OP_TYPE_LOWER_THAN; } break; - case 349: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy632 = OP_TYPE_GREATER_THAN; } + case 350: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy572 = OP_TYPE_GREATER_THAN; } break; - case 350: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy632 = OP_TYPE_LOWER_EQUAL; } + case 351: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy572 = OP_TYPE_LOWER_EQUAL; } break; - case 351: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy632 = OP_TYPE_GREATER_EQUAL; } + case 352: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy572 = OP_TYPE_GREATER_EQUAL; } break; - case 352: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy632 = OP_TYPE_NOT_EQUAL; } + case 353: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy572 = OP_TYPE_NOT_EQUAL; } break; - case 353: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy632 = OP_TYPE_EQUAL; } + case 354: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy572 = OP_TYPE_EQUAL; } break; - case 354: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy632 = OP_TYPE_LIKE; } + case 355: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy572 = OP_TYPE_LIKE; } break; - case 355: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy632 = OP_TYPE_NOT_LIKE; } + case 356: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy572 = OP_TYPE_NOT_LIKE; } break; - case 356: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy632 = OP_TYPE_MATCH; } + case 357: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy572 = OP_TYPE_MATCH; } break; - case 357: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy632 = OP_TYPE_NMATCH; } + case 358: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy572 = OP_TYPE_NMATCH; } break; - case 358: /* compare_op ::= CONTAINS */ -{ yymsp[0].minor.yy632 = OP_TYPE_JSON_CONTAINS; } + case 359: /* compare_op ::= CONTAINS */ +{ yymsp[0].minor.yy572 = OP_TYPE_JSON_CONTAINS; } break; - case 359: /* in_op ::= IN */ -{ yymsp[0].minor.yy632 = OP_TYPE_IN; } + case 360: /* in_op ::= IN */ +{ yymsp[0].minor.yy572 = OP_TYPE_IN; } break; - case 360: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy632 = OP_TYPE_NOT_IN; } + case 361: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy572 = OP_TYPE_NOT_IN; } break; - case 361: /* in_predicate_value ::= NK_LP expression_list NK_RP */ -{ yylhsminor.yy686 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy670)); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + case 362: /* in_predicate_value ::= NK_LP expression_list NK_RP */ +{ yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy60)); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 363: /* boolean_value_expression ::= NOT boolean_primary */ + case 364: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy686), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy172), NULL)); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; + yymsp[-1].minor.yy172 = yylhsminor.yy172; break; - case 364: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 365: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy686); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), releaseRawExprNode(pCxt, yymsp[0].minor.yy686))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 365: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 366: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy686); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy686); - yylhsminor.yy686 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), releaseRawExprNode(pCxt, yymsp[0].minor.yy686))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy172); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy172); + yylhsminor.yy172 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 372: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy686 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy686, yymsp[0].minor.yy686, NULL); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + case 373: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy172 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy172, yymsp[0].minor.yy172, NULL); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 375: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy686 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy113, &yymsp[0].minor.yy113); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; + case 376: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy172 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy105, &yymsp[0].minor.yy105); } + yymsp[-1].minor.yy172 = yylhsminor.yy172; break; - case 376: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy686 = createRealTableNode(pCxt, &yymsp[-3].minor.yy113, &yymsp[-1].minor.yy113, &yymsp[0].minor.yy113); } - yymsp[-3].minor.yy686 = yylhsminor.yy686; + case 377: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy172 = createRealTableNode(pCxt, &yymsp[-3].minor.yy105, &yymsp[-1].minor.yy105, &yymsp[0].minor.yy105); } + yymsp[-3].minor.yy172 = yylhsminor.yy172; break; - case 377: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy686 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy686), &yymsp[0].minor.yy113); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; + case 378: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy172 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy172), &yymsp[0].minor.yy105); } + yymsp[-1].minor.yy172 = yylhsminor.yy172; break; - case 379: /* alias_opt ::= */ -{ yymsp[1].minor.yy113 = nil_token; } + case 380: /* alias_opt ::= */ +{ yymsp[1].minor.yy105 = nil_token; } break; - case 380: /* alias_opt ::= table_alias */ -{ yylhsminor.yy113 = yymsp[0].minor.yy113; } - yymsp[0].minor.yy113 = yylhsminor.yy113; + case 381: /* alias_opt ::= table_alias */ +{ yylhsminor.yy105 = yymsp[0].minor.yy105; } + yymsp[0].minor.yy105 = yylhsminor.yy105; break; - case 381: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy113 = yymsp[0].minor.yy113; } + case 382: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy105 = yymsp[0].minor.yy105; } break; - case 382: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 383: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==383); -{ yymsp[-2].minor.yy686 = yymsp[-1].minor.yy686; } + case 383: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 384: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==384); +{ yymsp[-2].minor.yy172 = yymsp[-1].minor.yy172; } break; - case 384: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy686 = createJoinTableNode(pCxt, yymsp[-4].minor.yy120, yymsp[-5].minor.yy686, yymsp[-2].minor.yy686, yymsp[0].minor.yy686); } - yymsp[-5].minor.yy686 = yylhsminor.yy686; + case 385: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy172 = createJoinTableNode(pCxt, yymsp[-4].minor.yy636, yymsp[-5].minor.yy172, yymsp[-2].minor.yy172, yymsp[0].minor.yy172); } + yymsp[-5].minor.yy172 = yylhsminor.yy172; break; - case 385: /* join_type ::= */ -{ yymsp[1].minor.yy120 = JOIN_TYPE_INNER; } + case 386: /* join_type ::= */ +{ yymsp[1].minor.yy636 = JOIN_TYPE_INNER; } break; - case 386: /* join_type ::= INNER */ -{ yymsp[0].minor.yy120 = JOIN_TYPE_INNER; } + case 387: /* join_type ::= INNER */ +{ yymsp[0].minor.yy636 = JOIN_TYPE_INNER; } break; - case 387: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 388: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-8].minor.yy686 = createSelectStmt(pCxt, yymsp[-7].minor.yy131, yymsp[-6].minor.yy670, yymsp[-5].minor.yy686); - yymsp[-8].minor.yy686 = addWhereClause(pCxt, yymsp[-8].minor.yy686, yymsp[-4].minor.yy686); - yymsp[-8].minor.yy686 = addPartitionByClause(pCxt, yymsp[-8].minor.yy686, yymsp[-3].minor.yy670); - yymsp[-8].minor.yy686 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy686, yymsp[-2].minor.yy686); - yymsp[-8].minor.yy686 = addGroupByClause(pCxt, yymsp[-8].minor.yy686, yymsp[-1].minor.yy670); - yymsp[-8].minor.yy686 = addHavingClause(pCxt, yymsp[-8].minor.yy686, yymsp[0].minor.yy686); + yymsp[-8].minor.yy172 = createSelectStmt(pCxt, yymsp[-7].minor.yy617, yymsp[-6].minor.yy60, yymsp[-5].minor.yy172); + yymsp[-8].minor.yy172 = addWhereClause(pCxt, yymsp[-8].minor.yy172, yymsp[-4].minor.yy172); + yymsp[-8].minor.yy172 = addPartitionByClause(pCxt, yymsp[-8].minor.yy172, yymsp[-3].minor.yy60); + yymsp[-8].minor.yy172 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy172, yymsp[-2].minor.yy172); + yymsp[-8].minor.yy172 = addGroupByClause(pCxt, yymsp[-8].minor.yy172, yymsp[-1].minor.yy60); + yymsp[-8].minor.yy172 = addHavingClause(pCxt, yymsp[-8].minor.yy172, yymsp[0].minor.yy172); } break; - case 390: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy131 = false; } + case 391: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy617 = false; } break; - case 391: /* select_list ::= NK_STAR */ -{ yymsp[0].minor.yy670 = NULL; } + case 392: /* select_list ::= NK_STAR */ +{ yymsp[0].minor.yy60 = NULL; } break; - case 396: /* select_item ::= common_expression column_alias */ -{ yylhsminor.yy686 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy686), &yymsp[0].minor.yy113); } - yymsp[-1].minor.yy686 = yylhsminor.yy686; + case 397: /* select_item ::= common_expression column_alias */ +{ yylhsminor.yy172 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy172), &yymsp[0].minor.yy105); } + yymsp[-1].minor.yy172 = yylhsminor.yy172; break; - case 397: /* select_item ::= common_expression AS column_alias */ -{ yylhsminor.yy686 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), &yymsp[0].minor.yy113); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + case 398: /* select_item ::= common_expression AS column_alias */ +{ yylhsminor.yy172 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), &yymsp[0].minor.yy105); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 402: /* partition_by_clause_opt ::= PARTITION BY expression_list */ - case 419: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==419); - case 431: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==431); -{ yymsp[-2].minor.yy670 = yymsp[0].minor.yy670; } + case 403: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 420: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==420); + case 432: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==432); +{ yymsp[-2].minor.yy60 = yymsp[0].minor.yy60; } break; - case 404: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -{ yymsp[-5].minor.yy686 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy686), releaseRawExprNode(pCxt, yymsp[-1].minor.yy686)); } + case 405: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy172 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy172), releaseRawExprNode(pCxt, yymsp[-1].minor.yy172)); } break; - case 405: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ -{ yymsp[-3].minor.yy686 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy686)); } + case 406: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ +{ yymsp[-3].minor.yy172 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy172)); } break; - case 406: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy686 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy686), NULL, yymsp[-1].minor.yy686, yymsp[0].minor.yy686); } + case 407: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy172 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy172), NULL, yymsp[-1].minor.yy172, yymsp[0].minor.yy172); } break; - case 407: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy686 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy686), releaseRawExprNode(pCxt, yymsp[-3].minor.yy686), yymsp[-1].minor.yy686, yymsp[0].minor.yy686); } + case 408: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy172 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy172), releaseRawExprNode(pCxt, yymsp[-3].minor.yy172), yymsp[-1].minor.yy172, yymsp[0].minor.yy172); } break; - case 409: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ -{ yymsp[-3].minor.yy686 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy686); } + case 410: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ +{ yymsp[-3].minor.yy172 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy172); } break; - case 411: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy686 = createFillNode(pCxt, yymsp[-1].minor.yy522, NULL); } + case 412: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy172 = createFillNode(pCxt, yymsp[-1].minor.yy202, NULL); } break; - case 412: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy686 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy670)); } + case 413: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy172 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy60)); } break; - case 413: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy522 = FILL_MODE_NONE; } + case 414: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy202 = FILL_MODE_NONE; } break; - case 414: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy522 = FILL_MODE_PREV; } + case 415: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy202 = FILL_MODE_PREV; } break; - case 415: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy522 = FILL_MODE_NULL; } + case 416: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy202 = FILL_MODE_NULL; } break; - case 416: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy522 = FILL_MODE_LINEAR; } + case 417: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy202 = FILL_MODE_LINEAR; } break; - case 417: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy522 = FILL_MODE_NEXT; } + case 418: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy202 = FILL_MODE_NEXT; } break; - case 420: /* group_by_list ::= expression */ -{ yylhsminor.yy670 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy686))); } - yymsp[0].minor.yy670 = yylhsminor.yy670; + case 421: /* group_by_list ::= expression */ +{ yylhsminor.yy60 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); } + yymsp[0].minor.yy60 = yylhsminor.yy60; break; - case 421: /* group_by_list ::= group_by_list NK_COMMA expression */ -{ yylhsminor.yy670 = addNodeToList(pCxt, yymsp[-2].minor.yy670, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy686))); } - yymsp[-2].minor.yy670 = yylhsminor.yy670; + case 422: /* group_by_list ::= group_by_list NK_COMMA expression */ +{ yylhsminor.yy60 = addNodeToList(pCxt, yymsp[-2].minor.yy60, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy172))); } + yymsp[-2].minor.yy60 = yylhsminor.yy60; break; - case 424: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 425: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy686 = addOrderByClause(pCxt, yymsp[-3].minor.yy686, yymsp[-2].minor.yy670); - yylhsminor.yy686 = addSlimitClause(pCxt, yylhsminor.yy686, yymsp[-1].minor.yy686); - yylhsminor.yy686 = addLimitClause(pCxt, yylhsminor.yy686, yymsp[0].minor.yy686); + yylhsminor.yy172 = addOrderByClause(pCxt, yymsp[-3].minor.yy172, yymsp[-2].minor.yy60); + yylhsminor.yy172 = addSlimitClause(pCxt, yylhsminor.yy172, yymsp[-1].minor.yy172); + yylhsminor.yy172 = addLimitClause(pCxt, yylhsminor.yy172, yymsp[0].minor.yy172); } - yymsp[-3].minor.yy686 = yylhsminor.yy686; + yymsp[-3].minor.yy172 = yylhsminor.yy172; break; - case 426: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ yylhsminor.yy686 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy686, yymsp[0].minor.yy686); } - yymsp[-3].minor.yy686 = yylhsminor.yy686; + case 427: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ yylhsminor.yy172 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy172, yymsp[0].minor.yy172); } + yymsp[-3].minor.yy172 = yylhsminor.yy172; break; - case 427: /* query_expression_body ::= query_expression_body UNION query_expression_body */ -{ yylhsminor.yy686 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy686, yymsp[0].minor.yy686); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + case 428: /* query_expression_body ::= query_expression_body UNION query_expression_body */ +{ yylhsminor.yy172 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy172, yymsp[0].minor.yy172); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 429: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ -{ yymsp[-5].minor.yy686 = yymsp[-4].minor.yy686; } - yy_destructor(yypParser,349,&yymsp[-3].minor); - yy_destructor(yypParser,350,&yymsp[-2].minor); - yy_destructor(yypParser,351,&yymsp[-1].minor); + case 430: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ +{ yymsp[-5].minor.yy172 = yymsp[-4].minor.yy172; } + yy_destructor(yypParser,350,&yymsp[-3].minor); + yy_destructor(yypParser,351,&yymsp[-2].minor); + yy_destructor(yypParser,352,&yymsp[-1].minor); break; - case 433: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 437: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==437); -{ yymsp[-1].minor.yy686 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 434: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 438: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==438); +{ yymsp[-1].minor.yy172 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 434: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 438: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==438); -{ yymsp[-3].minor.yy686 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 435: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 439: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==439); +{ yymsp[-3].minor.yy172 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 435: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 439: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==439); -{ yymsp[-3].minor.yy686 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 436: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 440: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==440); +{ yymsp[-3].minor.yy172 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 440: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy686 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy686); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + case 441: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy172 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy172); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 444: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy686 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy686), yymsp[-1].minor.yy428, yymsp[0].minor.yy109); } - yymsp[-2].minor.yy686 = yylhsminor.yy686; + case 445: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy172 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy172), yymsp[-1].minor.yy14, yymsp[0].minor.yy17); } + yymsp[-2].minor.yy172 = yylhsminor.yy172; break; - case 445: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy428 = ORDER_ASC; } + case 446: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy14 = ORDER_ASC; } break; - case 446: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy428 = ORDER_ASC; } + case 447: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy14 = ORDER_ASC; } break; - case 447: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy428 = ORDER_DESC; } + case 448: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy14 = ORDER_DESC; } break; - case 448: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy109 = NULL_ORDER_DEFAULT; } + case 449: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy17 = NULL_ORDER_DEFAULT; } break; - case 449: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy109 = NULL_ORDER_FIRST; } + case 450: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy17 = NULL_ORDER_FIRST; } break; - case 450: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy109 = NULL_ORDER_LAST; } + case 451: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy17 = NULL_ORDER_LAST; } break; default: break; /********** End reduce actions ************************************************/ }; - assert( yyruleno now - 2d and ts < now - 1d"); + + run("DELETE FROM st1"); + + run("DELETE FROM st1 WHERE ts > now - 2d and ts < now - 1d AND tag1 = 10"); +} + +TEST_F(ParserInitialDTest, deleteSemanticCheck) { + useDb("root", "test"); + + run("DELETE FROM t1 WHERE c1 > 10", TSDB_CODE_PAR_INVALID_DELETE_WHERE, PARSER_STAGE_TRANSLATE); +} + // todo desc // todo describe -// todo drop account +// todo DROP account TEST_F(ParserInitialDTest, dropBnode) { useDb("root", "test"); @@ -62,51 +80,61 @@ TEST_F(ParserInitialDTest, dropCGroup) { run("DROP CONSUMER GROUP IF EXISTS cg1 ON tp1"); } -// todo drop database -// todo drop dnode -// todo drop function +// todo DROP database +// todo DROP dnode +// todo DROP function TEST_F(ParserInitialDTest, dropIndex) { useDb("root", "test"); - run("drop index index1 on t1"); + run("DROP index index1 on t1"); } TEST_F(ParserInitialDTest, dropMnode) { useDb("root", "test"); - run("drop mnode on dnode 1"); + run("DROP mnode on dnode 1"); } TEST_F(ParserInitialDTest, dropQnode) { useDb("root", "test"); - run("drop qnode on dnode 1"); + run("DROP qnode on dnode 1"); } TEST_F(ParserInitialDTest, dropSnode) { useDb("root", "test"); - run("drop snode on dnode 1"); + run("DROP snode on dnode 1"); +} + +TEST_F(ParserInitialDTest, dropSTable) { + useDb("root", "test"); + + run("DROP STABLE st1"); } -// todo drop stable -// todo drop stream -// todo drop table +// todo DROP stream + +TEST_F(ParserInitialDTest, dropTable) { + useDb("root", "test"); + + run("DROP TABLE t1"); +} TEST_F(ParserInitialDTest, dropTopic) { useDb("root", "test"); - run("drop topic tp1"); + run("DROP topic tp1"); - run("drop topic if exists tp1"); + run("DROP topic if exists tp1"); } TEST_F(ParserInitialDTest, dropUser) { login("root"); useDb("root", "test"); - run("drop user wxy"); + run("DROP user wxy"); } } // namespace ParserTest diff --git a/source/libs/parser/test/parInsertTest.cpp b/source/libs/parser/test/parInsertTest.cpp index 4d313fca766e8ab8f8d6ba404f7faf2fe833e9e6..eae83e222385f8e7a7af1be56bb3647b3e806358 100644 --- a/source/libs/parser/test/parInsertTest.cpp +++ b/source/libs/parser/test/parInsertTest.cpp @@ -59,6 +59,7 @@ class InsertTest : public Test { } int32_t runAsync() { + cxt_.async = true; code_ = parseInsertSyntax(&cxt_, &res_); if (code_ != TSDB_CODE_SUCCESS) { cout << "parseInsertSyntax code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index a5192595f0be83afa459429748dab3d8e9b65c4e..154e28a02c7fa88d5ba0f527463f7508af9e023c 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -33,6 +33,8 @@ TEST_F(ParserSelectTest, basic) { run("SELECT ts, t.c1 FROM (SELECT * FROM t1) t"); run("SELECT * FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); + + run("SELECT * FROM st1"); } TEST_F(ParserSelectTest, constant) { @@ -227,14 +229,14 @@ TEST_F(ParserSelectTest, subquery) { run("SELECT SUM(a) FROM (SELECT MAX(c1) a, _wstartts FROM st1s1 PARTITION BY TBNAME INTERVAL(1m)) INTERVAL(1n)"); } -TEST_F(ParserSelectTest, subquerySemanticError) { +TEST_F(ParserSelectTest, subquerySemanticCheck) { useDb("root", "test"); run("SELECT SUM(a) FROM (SELECT MAX(c1) a FROM st1s1 INTERVAL(1m)) INTERVAL(1n)", TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY, PARSER_STAGE_TRANSLATE); } -TEST_F(ParserSelectTest, semanticError) { +TEST_F(ParserSelectTest, semanticCheck) { useDb("root", "test"); // TSDB_CODE_PAR_INVALID_COLUMN diff --git a/source/libs/parser/test/parTestMain.cpp b/source/libs/parser/test/parTestMain.cpp index 820b8cca3cdc02633982a3ea797aa605db1e3fd3..ecc535feb0cdf08aa8eba88ff1b0e3d79718e9f7 100644 --- a/source/libs/parser/test/parTestMain.cpp +++ b/source/libs/parser/test/parTestMain.cpp @@ -35,6 +35,7 @@ namespace ParserTest { class ParserEnv : public testing::Environment { public: virtual void SetUp() { + fmFuncMgtInit(); initMetaDataEnv(); generateMetaData(); initLog(TD_TMP_DIR_PATH "td"); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 02d415af6c0b4ca3d1c717cde22e9c61c6c96888..c3aad632c7905fa24a5d756a60c718db5c007abe 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -21,8 +21,10 @@ typedef struct SLogicPlanContext { SPlanContext* pPlanCxt; } SLogicPlanContext; -typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, SSelectStmt*, SLogicNode**); +typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, void*, SLogicNode**); +typedef int32_t (*FCreateSelectLogicNode)(SLogicPlanContext*, SSelectStmt*, SLogicNode**); typedef int32_t (*FCreateSetOpLogicNode)(SLogicPlanContext*, SSetOperator*, SLogicNode**); +typedef int32_t (*FCreateDeleteLogicNode)(SLogicPlanContext*, SDeleteStmt*, SLogicNode**); static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode); @@ -96,7 +98,15 @@ static int32_t rewriteExprForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESq return cxt.errCode; } +static int32_t rewriteExpr(SNodeList* pExprs, SNode** pTarget) { + nodesWalkExprs(pExprs, doNameExpr, NULL); + SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs}; + nodesRewriteExpr(pTarget, doRewriteExpr, &cxt); + return cxt.errCode; +} + static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) { + nodesWalkExprs(pExprs, doNameExpr, NULL); SRewriteExprCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs}; nodesRewriteExprs(pTarget, doRewriteExpr, &cxt); return cxt.errCode; @@ -119,12 +129,12 @@ static int32_t pushLogicNode(SLogicPlanContext* pCxt, SLogicNode** pOldRoot, SLo return TSDB_CODE_SUCCESS; } -static int32_t createChildLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, FCreateLogicNode func, - SLogicNode** pRoot) { +static int32_t createRootLogicNode(SLogicPlanContext* pCxt, void* pStmt, uint8_t precision, FCreateLogicNode func, + SLogicNode** pRoot) { SLogicNode* pNode = NULL; - int32_t code = func(pCxt, pSelect, &pNode); + int32_t code = func(pCxt, pStmt, &pNode); if (TSDB_CODE_SUCCESS == code && NULL != pNode) { - pNode->precision = pSelect->precision; + pNode->precision = precision; code = pushLogicNode(pCxt, pRoot, pNode); } if (TSDB_CODE_SUCCESS != code) { @@ -133,59 +143,13 @@ static int32_t createChildLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelec return code; } -// typedef struct SCreateColumnCxt { -// int32_t errCode; -// SNodeList* pList; -// } SCreateColumnCxt; - -// static EDealRes doCreateColumn(SNode* pNode, void* pContext) { -// SCreateColumnCxt* pCxt = (SCreateColumnCxt*)pContext; -// switch (nodeType(pNode)) { -// case QUERY_NODE_COLUMN: { -// SNode* pCol = nodesCloneNode(pNode); -// if (NULL == pCol) { -// return DEAL_RES_ERROR; -// } -// return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); -// } -// case QUERY_NODE_OPERATOR: -// case QUERY_NODE_LOGIC_CONDITION: -// case QUERY_NODE_FUNCTION: { -// SExprNode* pExpr = (SExprNode*)pNode; -// SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); -// if (NULL == pCol) { -// return DEAL_RES_ERROR; -// } -// pCol->node.resType = pExpr->resType; -// strcpy(pCol->colName, pExpr->aliasName); -// return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); -// } -// default: -// break; -// } - -// return DEAL_RES_CONTINUE; -// } - -// static int32_t createColumnByRewriteExps(SNodeList* pExprs, SNodeList** pList) { -// SCreateColumnCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList)}; -// if (NULL == cxt.pList) { -// return TSDB_CODE_OUT_OF_MEMORY; -// } - -// nodesWalkExprs(pExprs, doCreateColumn, &cxt); -// if (TSDB_CODE_SUCCESS != cxt.errCode) { -// nodesDestroyList(cxt.pList); -// return cxt.errCode; -// } -// if (NULL == *pList) { -// *pList = cxt.pList; -// } -// return cxt.errCode; -// } +static int32_t createSelectRootLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, FCreateSelectLogicNode func, + SLogicNode** pRoot) { + return createRootLogicNode(pCxt, pSelect, pSelect->precision, (FCreateLogicNode)func, pRoot); +} static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols, SNodeList* pScanCols, - STableMeta* pMeta) { + int8_t tableType) { if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) { return SCAN_TYPE_STREAM; } @@ -195,7 +159,7 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols return NULL == pScanPseudoCols ? SCAN_TYPE_TABLE : SCAN_TYPE_TAG; } - if (TSDB_SYSTEM_TABLE == pMeta->tableType) { + if (TSDB_SYSTEM_TABLE == tableType) { return SCAN_TYPE_SYSTEM_TABLE; } @@ -248,16 +212,18 @@ static int32_t addPrimaryKeyCol(uint64_t tableId, SNodeList** pCols) { return TSDB_CODE_SUCCESS; } -static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, - SLogicNode** pLogicNode) { +static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealTable, bool hasRepeatScanFuncs, + SLogicNode** pLogicNode) { SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } - TSWAP(pScan->pMeta, pRealTable->pMeta); + // TSWAP(pScan->pMeta, pRealTable->pMeta); TSWAP(pScan->pVgroupList, pRealTable->pVgroupList); - pScan->scanSeq[0] = pSelect->hasRepeatScanFuncs ? 2 : 1; + pScan->tableId = pRealTable->pMeta->uid; + pScan->tableType = pRealTable->pMeta->tableType; + pScan->scanSeq[0] = hasRepeatScanFuncs ? 2 : 1; pScan->scanSeq[1] = 0; pScan->scanRange = TSWINDOW_INITIALIZER; pScan->tableName.type = TSDB_TABLE_NAME_T; @@ -268,9 +234,21 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pScan->ratio = pRealTable->ratio; pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; + *pLogicNode = (SLogicNode*)pScan; + + return TSDB_CODE_SUCCESS; +} + +static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, + SLogicNode** pLogicNode) { + SScanLogicNode* pScan = NULL; + int32_t code = makeScanLogicNode(pCxt, pRealTable, pSelect->hasRepeatScanFuncs, (SLogicNode**)&pScan); + // set columns to scan - int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_COL, - &pScan->pScanCols); + if (TSDB_CODE_SUCCESS == code) { + code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_COL, + &pScan->pScanCols); + } if (TSDB_CODE_SUCCESS == code) { code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_TAG, @@ -286,10 +264,10 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect code = rewriteExprForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM); } - pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->pMeta); + pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType); if (TSDB_CODE_SUCCESS == code) { - code = addPrimaryKeyCol(pScan->pMeta->uid, &pScan->pScanCols); + code = addPrimaryKeyCol(pScan->tableId, &pScan->pScanCols); } // set output @@ -562,6 +540,7 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva pWindow->sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : pWindow->interval); pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); + pWindow->intervalAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; pWindow->pTspk = nodesCloneNode(pInterval->pCol); if (NULL == pWindow->pTspk) { @@ -782,25 +761,25 @@ static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele SLogicNode* pRoot = NULL; int32_t code = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable, &pRoot); if (TSDB_CODE_SUCCESS == code) { - code = createChildLogicNode(pCxt, pSelect, createPartitionLogicNode, &pRoot); + code = createSelectRootLogicNode(pCxt, pSelect, createPartitionLogicNode, &pRoot); } if (TSDB_CODE_SUCCESS == code) { - code = createChildLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot); + code = createSelectRootLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot); } if (TSDB_CODE_SUCCESS == code) { - code = createChildLogicNode(pCxt, pSelect, createFillLogicNode, &pRoot); + code = createSelectRootLogicNode(pCxt, pSelect, createFillLogicNode, &pRoot); } if (TSDB_CODE_SUCCESS == code) { - code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot); + code = createSelectRootLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot); } if (TSDB_CODE_SUCCESS == code) { - code = createChildLogicNode(pCxt, pSelect, createDistinctLogicNode, &pRoot); + code = createSelectRootLogicNode(pCxt, pSelect, createDistinctLogicNode, &pRoot); } if (TSDB_CODE_SUCCESS == code) { - code = createChildLogicNode(pCxt, pSelect, createSortLogicNode, &pRoot); + code = createSelectRootLogicNode(pCxt, pSelect, createSortLogicNode, &pRoot); } if (TSDB_CODE_SUCCESS == code) { - code = createChildLogicNode(pCxt, pSelect, createProjectLogicNode, &pRoot); + code = createSelectRootLogicNode(pCxt, pSelect, createProjectLogicNode, &pRoot); } if (TSDB_CODE_SUCCESS == code) { @@ -812,17 +791,9 @@ static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele return code; } -static int32_t createSetOpChildLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, - FCreateSetOpLogicNode func, SLogicNode** pRoot) { - SLogicNode* pNode = NULL; - int32_t code = func(pCxt, pSetOperator, &pNode); - if (TSDB_CODE_SUCCESS == code && NULL != pNode) { - code = pushLogicNode(pCxt, pRoot, pNode); - } - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyNode(pNode); - } - return code; +static int32_t createSetOpRootLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, FCreateSetOpLogicNode func, + SLogicNode** pRoot) { + return createRootLogicNode(pCxt, pSetOperator, pSetOperator->precision, (FCreateLogicNode)func, pRoot); } static int32_t createSetOpSortLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) { @@ -969,7 +940,7 @@ static int32_t createSetOperatorLogicNode(SLogicPlanContext* pCxt, SSetOperator* SLogicNode* pRoot = NULL; int32_t code = createSetOpLogicNode(pCxt, pSetOperator, &pRoot); if (TSDB_CODE_SUCCESS == code) { - code = createSetOpChildLogicNode(pCxt, pSetOperator, createSetOpSortLogicNode, &pRoot); + code = createSetOpRootLogicNode(pCxt, pSetOperator, createSetOpSortLogicNode, &pRoot); } if (TSDB_CODE_SUCCESS == code) { @@ -997,16 +968,124 @@ static int32_t getMsgType(ENodeType sqlType) { } static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt, SLogicNode** pLogicNode) { - SVnodeModifLogicNode* pModif = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIF); + SVnodeModifyLogicNode* pModif = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY); if (NULL == pModif) { return TSDB_CODE_OUT_OF_MEMORY; } + pModif->modifyType = MODIFY_TABLE_TYPE_INSERT; TSWAP(pModif->pDataBlocks, pStmt->pDataBlocks); pModif->msgType = getMsgType(pStmt->sqlNodeType); *pLogicNode = (SLogicNode*)pModif; return TSDB_CODE_SUCCESS; } +static int32_t createDeleteRootLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, FCreateDeleteLogicNode func, + SLogicNode** pRoot) { + return createRootLogicNode(pCxt, pDelete, pDelete->precision, (FCreateLogicNode)func, pRoot); +} + +static int32_t createDeleteScanLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, SLogicNode** pLogicNode) { + SScanLogicNode* pScan = NULL; + int32_t code = makeScanLogicNode(pCxt, (SRealTableNode*)pDelete->pFromTable, false, (SLogicNode**)&pScan); + + // set columns to scan + if (TSDB_CODE_SUCCESS == code) { + pScan->scanType = SCAN_TYPE_TABLE; + pScan->pScanCols = nodesCloneList(((SFunctionNode*)pDelete->pCountFunc)->pParameterList); + if (NULL == pScan->pScanCols) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code && NULL != pDelete->pTagIndexCond) { + pScan->pTagCond = nodesCloneNode(pDelete->pTagIndexCond); + if (NULL == pScan->pTagCond) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + // set output + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pScan->pScanCols, &pScan->node.pTargets); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pScan; + } else { + nodesDestroyNode(pScan); + } + + return code; +} + +static int32_t createDeleteAggLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, SLogicNode** pLogicNode) { + SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG); + if (NULL == pAgg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, nodesCloneNode(pDelete->pCountFunc)); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pAgg->pAggFuncs, &pDelete->pCountFunc); + } + // set the output + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pAgg->pAggFuncs, &pAgg->node.pTargets); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pAgg; + } else { + nodesDestroyNode(pAgg); + } + + return code; +} + +static int32_t createVnodeModifLogicNodeByDelete(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, + SLogicNode** pLogicNode) { + SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY); + if (NULL == pModify) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SRealTableNode* pRealTable = (SRealTableNode*)pDelete->pFromTable; + + pModify->modifyType = MODIFY_TABLE_TYPE_DELETE; + pModify->tableId = pRealTable->pMeta->uid; + pModify->tableType = pRealTable->pMeta->tableType; + snprintf(pModify->tableFName, sizeof(pModify->tableFName), "%d.%s.%s", pCxt->pPlanCxt->acctId, + pRealTable->table.dbName, pRealTable->table.tableName); + pModify->deleteTimeRange = pDelete->timeRange; + pModify->pModifyRows = nodesCloneNode(pDelete->pCountFunc); + if (NULL == pModify->pModifyRows) { + nodesDestroyNode(pModify); + return TSDB_CODE_OUT_OF_MEMORY; + } + + *pLogicNode = (SLogicNode*)pModify; + return TSDB_CODE_SUCCESS; +} + +static int32_t createDeleteLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, SLogicNode** pLogicNode) { + SLogicNode* pRoot = NULL; + int32_t code = createDeleteScanLogicNode(pCxt, pDelete, &pRoot); + if (TSDB_CODE_SUCCESS == code) { + code = createDeleteRootLogicNode(pCxt, pDelete, createDeleteAggLogicNode, &pRoot); + } + if (TSDB_CODE_SUCCESS == code) { + code = createDeleteRootLogicNode(pCxt, pDelete, createVnodeModifLogicNodeByDelete, &pRoot); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = pRoot; + } else { + nodesDestroyNode(pRoot); + } + + return code; +} + static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogicNode** pLogicNode) { switch (nodeType(pStmt)) { case QUERY_NODE_SELECT_STMT: @@ -1017,6 +1096,8 @@ static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogi return createQueryLogicNode(pCxt, ((SExplainStmt*)pStmt)->pQuery, pLogicNode); case QUERY_NODE_SET_OPERATOR: return createSetOperatorLogicNode(pCxt, (SSetOperator*)pStmt, pLogicNode); + case QUERY_NODE_DELETE_STMT: + return createDeleteLogicNode(pCxt, (SDeleteStmt*)pStmt, pLogicNode); default: break; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 5f88fc40e54c5e000a6e4506b30a2063acfbc8f1..8346aca76f5a082ec44f0fa61edb7d080a0636bb 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -99,7 +99,7 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { return false; } // todo: release after function splitting - if (TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->pMeta->tableType && + if (TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->tableType && SCAN_TYPE_STREAM != ((SScanLogicNode*)pNode)->scanType) { return false; } @@ -108,7 +108,8 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { return false; } if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent)) { - return (WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType); + return true; + // return (WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType); } return !osdHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); } @@ -217,8 +218,7 @@ static int32_t osdGetDataRequired(SNodeList* pFuncs) { } static void setScanWindowInfo(SScanLogicNode* pScan) { - if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pScan->node.pParent) && - WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pScan->node.pParent)->winType) { + if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pScan->node.pParent)) { pScan->interval = ((SWindowLogicNode*)pScan->node.pParent)->interval; pScan->offset = ((SWindowLogicNode*)pScan->node.pParent)->offset; pScan->sliding = ((SWindowLogicNode*)pScan->node.pParent)->sliding; @@ -268,30 +268,6 @@ static int32_t cpdMergeCond(SNode** pDst, SNode** pSrc) { return code; } -static int32_t cpdMergeConds(SNode** pDst, SNodeList** pSrc) { - if (NULL == *pSrc) { - return TSDB_CODE_SUCCESS; - } - - if (1 == LIST_LENGTH(*pSrc)) { - *pDst = nodesListGetNode(*pSrc, 0); - nodesClearList(*pSrc); - } else { - SLogicConditionNode* pLogicCond = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); - if (NULL == pLogicCond) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL; - pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - pLogicCond->condType = LOGIC_COND_TYPE_AND; - pLogicCond->pParameterList = *pSrc; - *pDst = (SNode*)pLogicCond; - } - *pSrc = NULL; - - return TSDB_CODE_SUCCESS; -} - static int32_t cpdCondAppend(SNode** pCond, SNode** pAdditionalCond) { if (NULL == *pCond) { TSWAP(*pCond, *pAdditionalCond); @@ -310,119 +286,6 @@ static int32_t cpdCondAppend(SNode** pCond, SNode** pAdditionalCond) { return code; } -static EDealRes cpdIsPrimaryKeyCondImpl(SNode* pNode, void* pContext) { - if (QUERY_NODE_COLUMN == nodeType(pNode)) { - *((bool*)pContext) = ((PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) ? true : false); - return *((bool*)pContext) ? DEAL_RES_CONTINUE : DEAL_RES_END; - } - return DEAL_RES_CONTINUE; -} - -static bool cpdIsPrimaryKeyCond(SNode* pNode) { - if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { - return false; - } - bool isPrimaryKeyCond = false; - nodesWalkExpr(pNode, cpdIsPrimaryKeyCondImpl, &isPrimaryKeyCond); - return isPrimaryKeyCond; -} - -static EDealRes cpdIsTagCondImpl(SNode* pNode, void* pContext) { - if (QUERY_NODE_COLUMN == nodeType(pNode)) { - *((bool*)pContext) = ((COLUMN_TYPE_TAG == ((SColumnNode*)pNode)->colType) ? true : false); - return *((bool*)pContext) ? DEAL_RES_CONTINUE : DEAL_RES_END; - } - return DEAL_RES_CONTINUE; -} - -static bool cpdIsTagCond(SNode* pNode) { - if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { - return false; - } - bool isTagCond = false; - nodesWalkExpr(pNode, cpdIsTagCondImpl, &isTagCond); - return isTagCond; -} - -static int32_t cpdPartitionScanLogicCond(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pTagCond, - SNode** pOtherCond) { - SLogicConditionNode* pLogicCond = (SLogicConditionNode*)pScan->node.pConditions; - - if (LOGIC_COND_TYPE_AND != pLogicCond->condType) { - *pPrimaryKeyCond = NULL; - *pOtherCond = pScan->node.pConditions; - pScan->node.pConditions = NULL; - return TSDB_CODE_SUCCESS; - } - - int32_t code = TSDB_CODE_SUCCESS; - - SNodeList* pPrimaryKeyConds = NULL; - SNodeList* pTagConds = NULL; - SNodeList* pOtherConds = NULL; - SNode* pCond = NULL; - FOREACH(pCond, pLogicCond->pParameterList) { - if (cpdIsPrimaryKeyCond(pCond)) { - code = nodesListMakeAppend(&pPrimaryKeyConds, nodesCloneNode(pCond)); - } else if (cpdIsTagCond(pScan->node.pConditions)) { - code = nodesListMakeAppend(&pTagConds, nodesCloneNode(pCond)); - } else { - code = nodesListMakeAppend(&pOtherConds, nodesCloneNode(pCond)); - } - if (TSDB_CODE_SUCCESS != code) { - break; - } - } - - SNode* pTempPrimaryKeyCond = NULL; - SNode* pTempTagCond = NULL; - SNode* pTempOtherCond = NULL; - if (TSDB_CODE_SUCCESS == code) { - code = cpdMergeConds(&pTempPrimaryKeyCond, &pPrimaryKeyConds); - } - if (TSDB_CODE_SUCCESS == code) { - code = cpdMergeConds(&pTempTagCond, &pTagConds); - } - if (TSDB_CODE_SUCCESS == code) { - code = cpdMergeConds(&pTempOtherCond, &pOtherConds); - } - - if (TSDB_CODE_SUCCESS == code) { - *pPrimaryKeyCond = pTempPrimaryKeyCond; - *pTagCond = pTempTagCond; - *pOtherCond = pTempOtherCond; - nodesDestroyNode(pScan->node.pConditions); - pScan->node.pConditions = NULL; - } else { - nodesDestroyList(pPrimaryKeyConds); - nodesDestroyList(pTagConds); - nodesDestroyList(pOtherConds); - nodesDestroyNode(pTempPrimaryKeyCond); - nodesDestroyNode(pTempTagCond); - nodesDestroyNode(pTempOtherCond); - } - - return code; -} - -static int32_t cpdPartitionScanCond(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pTagCond, - SNode** pOtherCond) { - if (QUERY_NODE_LOGIC_CONDITION == nodeType(pScan->node.pConditions)) { - return cpdPartitionScanLogicCond(pScan, pPrimaryKeyCond, pTagCond, pOtherCond); - } - - if (cpdIsPrimaryKeyCond(pScan->node.pConditions)) { - *pPrimaryKeyCond = pScan->node.pConditions; - } else if (cpdIsTagCond(pScan->node.pConditions)) { - *pTagCond = pScan->node.pConditions; - } else { - *pOtherCond = pScan->node.pConditions; - } - pScan->node.pConditions = NULL; - - return TSDB_CODE_SUCCESS; -} - static int32_t cpdCalcTimeRange(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pOtherCond) { bool isStrict = false; int32_t code = filterGetTimeRange(*pPrimaryKeyCond, &pScan->scanRange, &isStrict); @@ -465,14 +328,14 @@ static int32_t cpdApplyTagIndex(SScanLogicNode* pScan, SNode** pTagCond, SNode** static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) { if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD) || - TSDB_SYSTEM_TABLE == pScan->pMeta->tableType) { + TSDB_SYSTEM_TABLE == pScan->tableType) { return TSDB_CODE_SUCCESS; } SNode* pPrimaryKeyCond = NULL; SNode* pTagCond = NULL; SNode* pOtherCond = NULL; - int32_t code = cpdPartitionScanCond(pScan, &pPrimaryKeyCond, &pTagCond, &pOtherCond); + int32_t code = nodesPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pTagCond, &pOtherCond); if (TSDB_CODE_SUCCESS == code && NULL != pPrimaryKeyCond) { code = cpdCalcTimeRange(pScan, &pPrimaryKeyCond, &pOtherCond); } @@ -565,16 +428,16 @@ static int32_t cpdPartitionLogicCond(SJoinLogicNode* pJoin, SNode** pOnCond, SNo SNode* pTempRightChildCond = NULL; SNode* pTempRemainCond = NULL; if (TSDB_CODE_SUCCESS == code) { - code = cpdMergeConds(&pTempOnCond, &pOnConds); + code = nodesMergeConds(&pTempOnCond, &pOnConds); } if (TSDB_CODE_SUCCESS == code) { - code = cpdMergeConds(&pTempLeftChildCond, &pLeftChildConds); + code = nodesMergeConds(&pTempLeftChildCond, &pLeftChildConds); } if (TSDB_CODE_SUCCESS == code) { - code = cpdMergeConds(&pTempRightChildCond, &pRightChildConds); + code = nodesMergeConds(&pTempRightChildCond, &pRightChildConds); } if (TSDB_CODE_SUCCESS == code) { - code = cpdMergeConds(&pTempRemainCond, &pRemainConds); + code = nodesMergeConds(&pTempRemainCond, &pRemainConds); } if (TSDB_CODE_SUCCESS == code) { @@ -799,7 +662,7 @@ static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeL switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: - if (TSDB_SUPER_TABLE != ((SScanLogicNode*)pNode)->pMeta->tableType) { + if (TSDB_SUPER_TABLE != ((SScanLogicNode*)pNode)->tableType) { return nodesListMakeAppend(pScanNodes, pNode); } break; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e1580d3eee4e2f08cb230344e65be5e8960c3ba3..3c000e3ff7f8156555390c499aba39adc3cee9c8 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -342,18 +342,7 @@ static int32_t setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, i return TSDB_CODE_SUCCESS; } -static uint8_t getPrecision(SNodeList* pChildren) { - if (1 == LIST_LENGTH(pChildren)) { - return (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc)->precision; - } else if (2 == LIST_LENGTH(pChildren)) { - uint8_t lp = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc)->precision; - uint8_t rp = (((SPhysiNode*)nodesListGetNode(pChildren, 1))->pOutputDataBlockDesc)->precision; - return (lp > rp ? rp : lp); - } - return 0; -} - -static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, uint8_t precision, SLogicNode* pLogicNode, ENodeType type) { +static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, ENodeType type) { SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type); if (NULL == pPhysiNode) { return NULL; @@ -364,7 +353,7 @@ static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, uint8_t precision, SLo nodesDestroyNode(pPhysiNode); return NULL; } - pPhysiNode->pOutputDataBlockDesc->precision = precision; + pPhysiNode->pOutputDataBlockDesc->precision = pLogicNode->precision; return pPhysiNode; } @@ -435,8 +424,8 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS } if (TSDB_CODE_SUCCESS == code) { - pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; - pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; + pScanPhysiNode->uid = pScanLogicNode->tableId; + pScanPhysiNode->tableType = pScanLogicNode->tableType; memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName)); if (NULL != pScanLogicNode->pTagCond) { pSubplan->pTagCond = nodesCloneNode(pScanLogicNode->pTagCond); @@ -462,8 +451,8 @@ static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAdd static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { - STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode( - pCxt, pScanLogicNode->pMeta->tableInfo.precision, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); + STagScanPhysiNode* pTagScan = + (STagScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); if (NULL == pTagScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -476,8 +465,7 @@ static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubpla static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { STableScanPhysiNode* pTableScan = - (STableScanPhysiNode*)makePhysiNode(pCxt, pScanLogicNode->pMeta->tableInfo.precision, (SLogicNode*)pScanLogicNode, - QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + (STableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); if (NULL == pTableScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -515,9 +503,8 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { - SSystemTableScanPhysiNode* pScan = - (SSystemTableScanPhysiNode*)makePhysiNode(pCxt, pScanLogicNode->pMeta->tableInfo.precision, - (SLogicNode*)pScanLogicNode, QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN); + SSystemTableScanPhysiNode* pScan = (SSystemTableScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, + QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN); if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -526,10 +513,10 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pScan->accountId = pCxt->pPlanCxt->acctId; if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLES)) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); - SQueryNodeLoad node = { .addr = pSubplan->execNode, .load = 0}; + SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0}; taosArrayPush(pCxt->pExecNodeList, &pSubplan->execNode); } else { - SQueryNodeLoad node = { .addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0}; + SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0}; taosArrayPush(pCxt->pExecNodeList, &node); } pScan->mgmtEpSet = pCxt->pPlanCxt->mgmtEpSet; @@ -567,8 +554,8 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, static int32_t createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode, SPhysiNode** pPhyNode) { - SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pJoinLogicNode, - QUERY_NODE_PHYSICAL_PLAN_JOIN); + SJoinPhysiNode* pJoin = + (SJoinPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pJoinLogicNode, QUERY_NODE_PHYSICAL_PLAN_JOIN); if (NULL == pJoin) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -750,8 +737,7 @@ static int32_t rewritePrecalcExpr(SPhysiPlanContext* pCxt, SNode* pNode, SNodeLi static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode, SPhysiNode** pPhyNode) { - SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pAggLogicNode, - QUERY_NODE_PHYSICAL_PLAN_AGG); + SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pAggLogicNode, QUERY_NODE_PHYSICAL_PLAN_AGG); if (NULL == pAgg) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -806,8 +792,8 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode, SPhysiNode** pPhyNode) { - SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode( - pCxt, getPrecision(pChildren), (SLogicNode*)pProjectLogicNode, QUERY_NODE_PHYSICAL_PLAN_PROJECT); + SProjectPhysiNode* pProject = + (SProjectPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pProjectLogicNode, QUERY_NODE_PHYSICAL_PLAN_PROJECT); if (NULL == pProject) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -838,8 +824,8 @@ static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { - SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode( - pCxt, pExchangeLogicNode->node.precision, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); + SExchangePhysiNode* pExchange = + (SExchangePhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); if (NULL == pExchange) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -852,8 +838,8 @@ static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogic static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { - SScanPhysiNode* pScan = (SScanPhysiNode*)makePhysiNode( - pCxt, pExchangeLogicNode->node.precision, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + SScanPhysiNode* pScan = + (SScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -933,11 +919,28 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* return code; } +static ENodeType getIntervalOperatorType(EIntervalAlgorithm intervalAlgo) { + switch (intervalAlgo) { + case INTERVAL_ALGO_HASH: + return QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; + case INTERVAL_ALGO_SORT_MERGE: + return QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL; + case INTERVAL_ALGO_STREAM_FINAL: + return QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL; + case INTERVAL_ALGO_STREAM_SEMI: + return QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL; + case INTERVAL_ALGO_STREAM_SINGLE: + return QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; + default: + break; + } + return QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; +} + static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)makePhysiNode( - pCxt, getPrecision(pChildren), (SLogicNode*)pWindowLogicNode, - (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL : QUERY_NODE_PHYSICAL_PLAN_INTERVAL)); + pCxt, (SLogicNode*)pWindowLogicNode, getIntervalOperatorType(pWindowLogicNode->intervalAlgo)); if (NULL == pInterval) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -954,7 +957,7 @@ static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChil static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { SSessionWinodwPhysiNode* pSession = (SSessionWinodwPhysiNode*)makePhysiNode( - pCxt, getPrecision(pChildren), (SLogicNode*)pWindowLogicNode, + pCxt, (SLogicNode*)pWindowLogicNode, (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION_WINDOW : QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW)); if (NULL == pSession) { @@ -968,8 +971,10 @@ static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* static int32_t createStateWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { - SStateWinodwPhysiNode* pState = (SStateWinodwPhysiNode*)makePhysiNode( - pCxt, getPrecision(pChildren), (SLogicNode*)pWindowLogicNode, QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW); + SStateWinodwPhysiNode* pState = + (SStateWinodwPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pWindowLogicNode, + (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW + : QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW)); if (NULL == pState) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1019,8 +1024,8 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SSortLogicNode* pSortLogicNode, SPhysiNode** pPhyNode) { - SSortPhysiNode* pSort = (SSortPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pSortLogicNode, - QUERY_NODE_PHYSICAL_PLAN_SORT); + SSortPhysiNode* pSort = + (SSortPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pSortLogicNode, QUERY_NODE_PHYSICAL_PLAN_SORT); if (NULL == pSort) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1060,8 +1065,8 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SPartitionLogicNode* pPartLogicNode, SPhysiNode** pPhyNode) { - SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)makePhysiNode( - pCxt, getPrecision(pChildren), (SLogicNode*)pPartLogicNode, QUERY_NODE_PHYSICAL_PLAN_PARTITION); + SPartitionPhysiNode* pPart = + (SPartitionPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pPartLogicNode, QUERY_NODE_PHYSICAL_PLAN_PARTITION); if (NULL == pPart) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1101,8 +1106,7 @@ static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SFillLogicNode* pFillNode, SPhysiNode** pPhyNode) { - SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode(pCxt, getPrecision(pChildren), (SLogicNode*)pFillNode, - QUERY_NODE_PHYSICAL_PLAN_FILL); + SFillPhysiNode* pFill = (SFillPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pFillNode, QUERY_NODE_PHYSICAL_PLAN_FILL); if (NULL == pFill) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1155,8 +1159,8 @@ static int32_t createExchangePhysiNodeByMerge(SMergePhysiNode* pMerge) { } static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pMergeLogicNode, SPhysiNode** pPhyNode) { - SMergePhysiNode* pMerge = (SMergePhysiNode*)makePhysiNode( - pCxt, pMergeLogicNode->node.precision, (SLogicNode*)pMergeLogicNode, QUERY_NODE_PHYSICAL_PLAN_MERGE); + SMergePhysiNode* pMerge = + (SMergePhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pMergeLogicNode, QUERY_NODE_PHYSICAL_PLAN_MERGE); if (NULL == pMerge) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1295,6 +1299,62 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl return pSubplan; } +static int32_t buildInsertSubplan(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, SSubplan* pSubplan) { + pSubplan->msgType = pModify->msgType; + pSubplan->execNode.epSet = pModify->pVgDataBlocks->vg.epSet; + SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0}; + taosArrayPush(pCxt->pExecNodeList, &node); + return createDataInserter(pCxt, pModify->pVgDataBlocks, &pSubplan->pDataSink); +} + +static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, const SPhysiNode* pRoot, + SDataSinkNode** pSink) { + SDataDeleterNode* pDeleter = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_DELETE); + if (NULL == pDeleter) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pDeleter->tableId = pModify->tableId; + pDeleter->tableType = pModify->tableType; + strcpy(pDeleter->tableFName, pModify->tableFName); + pDeleter->deleteTimeRange = pModify->deleteTimeRange; + + pDeleter->sink.pInputDataBlockDesc = nodesCloneNode(pRoot->pOutputDataBlockDesc); + if (NULL == pDeleter->sink.pInputDataBlockDesc) { + nodesDestroyNode(pDeleter); + return TSDB_CODE_OUT_OF_MEMORY; + } + + *pSink = (SDataSinkNode*)pDeleter; + return TSDB_CODE_SUCCESS; +} + +static int32_t buildDeleteSubplan(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pModify, SSubplan* pSubplan) { + int32_t code = + createPhysiNode(pCxt, (SLogicNode*)nodesListGetNode(pModify->node.pChildren, 0), pSubplan, &pSubplan->pNode); + if (TSDB_CODE_SUCCESS == code) { + code = createDataDeleter(pCxt, pModify, pSubplan->pNode, &pSubplan->pDataSink); + } + return code; +} + +static int32_t buildVnodeModifySubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan* pSubplan) { + int32_t code = TSDB_CODE_SUCCESS; + SVnodeModifyLogicNode* pModify = (SVnodeModifyLogicNode*)pLogicSubplan->pNode; + switch (pModify->modifyType) { + case MODIFY_TABLE_TYPE_INSERT: + code = buildInsertSubplan(pCxt, pModify, pSubplan); + break; + case MODIFY_TABLE_TYPE_DELETE: + code = buildDeleteSubplan(pCxt, pModify, pSubplan); + break; + default: + code = TSDB_CODE_FAILED; + break; + } + return code; +} + static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SSubplan** pPhysiSubplan) { SSubplan* pSubplan = makeSubplan(pCxt, pLogicSubplan); if (NULL == pSubplan) { @@ -1304,12 +1364,7 @@ static int32_t createPhysiSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogic int32_t code = TSDB_CODE_SUCCESS; if (SUBPLAN_TYPE_MODIFY == pLogicSubplan->subplanType) { - SVnodeModifLogicNode* pModif = (SVnodeModifLogicNode*)pLogicSubplan->pNode; - pSubplan->msgType = pModif->msgType; - pSubplan->execNode.epSet = pModif->pVgDataBlocks->vg.epSet; - SQueryNodeLoad node = {.addr = pSubplan->execNode, .load = 0}; - taosArrayPush(pCxt->pExecNodeList, &node); - code = createDataInserter(pCxt, pModif->pVgDataBlocks, &pSubplan->pDataSink); + code = buildVnodeModifySubplan(pCxt, pLogicSubplan, pSubplan); } else { pSubplan->msgType = TDMT_VND_QUERY; code = createPhysiNode(pCxt, pLogicSubplan->pNode, pSubplan, &pSubplan->pNode); diff --git a/source/libs/planner/src/planScaleOut.c b/source/libs/planner/src/planScaleOut.c index 73c8666673ca23168d5dc2bf495cb9553bee55b8..5dd016afb8acff5b0759f986a88feb3a71746b17 100644 --- a/source/libs/planner/src/planScaleOut.c +++ b/source/libs/planner/src/planScaleOut.c @@ -38,26 +38,6 @@ static SLogicSubplan* singleCloneSubLogicPlan(SScaleOutContext* pCxt, SLogicSubp return pDst; } -static int32_t scaleOutForModify(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { - SVnodeModifLogicNode* pNode = (SVnodeModifLogicNode*)pSubplan->pNode; - size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks); - for (int32_t i = 0; i < numOfVgroups; ++i) { - SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); - if (NULL == pNewSubplan) { - return TSDB_CODE_OUT_OF_MEMORY; - } - ((SVnodeModifLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i); - if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pGroup, pNewSubplan)) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } - return TSDB_CODE_SUCCESS; -} - -static int32_t scaleOutForMerge(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { - return nodesListStrictAppend(pGroup, singleCloneSubLogicPlan(pCxt, pSubplan, level)); -} - static int32_t doSetScanVgroup(SLogicNode* pNode, const SVgroupInfo* pVgroup, bool* pFound) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { SScanLogicNode* pScan = (SScanLogicNode*)pNode; @@ -84,23 +64,52 @@ static int32_t setScanVgroup(SLogicNode* pNode, const SVgroupInfo* pVgroup) { return doSetScanVgroup(pNode, pVgroup, &found); } -static int32_t scaleOutForScan(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { - if (pSubplan->pVgroupList && !pCxt->pPlanCxt->streamQuery) { - int32_t code = TSDB_CODE_SUCCESS; - for (int32_t i = 0; i < pSubplan->pVgroupList->numOfVgroups; ++i) { +static int32_t scaleOutByVgroups(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + int32_t code = TSDB_CODE_SUCCESS; + for (int32_t i = 0; i < pSubplan->pVgroupList->numOfVgroups; ++i) { + SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); + if (NULL == pNewSubplan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + code = setScanVgroup(pNewSubplan->pNode, pSubplan->pVgroupList->vgroups + i); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pGroup, pNewSubplan); + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; +} + +static int32_t scaleOutForModify(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + SVnodeModifyLogicNode* pNode = (SVnodeModifyLogicNode*)pSubplan->pNode; + if (MODIFY_TABLE_TYPE_DELETE == pNode->modifyType) { + return scaleOutByVgroups(pCxt, pSubplan, level, pGroup); + } else { + size_t numOfVgroups = taosArrayGetSize(pNode->pDataBlocks); + for (int32_t i = 0; i < numOfVgroups; ++i) { SLogicSubplan* pNewSubplan = singleCloneSubLogicPlan(pCxt, pSubplan, level); if (NULL == pNewSubplan) { return TSDB_CODE_OUT_OF_MEMORY; } - code = setScanVgroup(pNewSubplan->pNode, pSubplan->pVgroupList->vgroups + i); - if (TSDB_CODE_SUCCESS == code) { - code = nodesListStrictAppend(pGroup, pNewSubplan); - } - if (TSDB_CODE_SUCCESS != code) { - break; + ((SVnodeModifyLogicNode*)pNewSubplan->pNode)->pVgDataBlocks = + (SVgDataBlocks*)taosArrayGetP(pNode->pDataBlocks, i); + if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pGroup, pNewSubplan)) { + return TSDB_CODE_OUT_OF_MEMORY; } } - return code; + return TSDB_CODE_SUCCESS; + } +} + +static int32_t scaleOutForMerge(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + return nodesListStrictAppend(pGroup, singleCloneSubLogicPlan(pCxt, pSubplan, level)); +} + +static int32_t scaleOutForScan(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pGroup) { + if (pSubplan->pVgroupList && !pCxt->pPlanCxt->streamQuery) { + return scaleOutByVgroups(pCxt, pSubplan, level, pGroup); } else { return scaleOutForMerge(pCxt, pSubplan, level, pGroup); } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 2ae749bef5e65d7c18ec942d008c5499494d06db..9e6c25ce78abb1cd16b32c8afcdc7769e67ca086 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -24,9 +24,10 @@ #define SPLIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0) typedef struct SSplitContext { - uint64_t queryId; - int32_t groupId; - bool split; + SPlanContext* pPlanCxt; + uint64_t queryId; + int32_t groupId; + bool split; } SSplitContext; typedef int32_t (*FSplit)(SSplitContext* pCxt, SLogicSubplan* pSubplan); @@ -36,7 +37,7 @@ typedef struct SSplitRule { FSplit splitFunc; } SSplitRule; -typedef bool (*FSplFindSplitNode)(SLogicSubplan* pSubplan, void* pInfo); +typedef bool (*FSplFindSplitNode)(SSplitContext* pCxt, SLogicSubplan* pSubplan, void* pInfo); static void splSetSubplanVgroups(SLogicSubplan* pSubplan, SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { @@ -63,19 +64,29 @@ static SLogicSubplan* splCreateScanSubplan(SSplitContext* pCxt, SLogicNode* pNod return pSubplan; } -static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode, - ESubplanType subplanType) { +static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pChild, SExchangeLogicNode** pOutput) { SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); if (NULL == pExchange) { return TSDB_CODE_OUT_OF_MEMORY; } pExchange->srcGroupId = pCxt->groupId; - pExchange->node.precision = pSplitNode->precision; - pExchange->node.pTargets = nodesCloneList(pSplitNode->pTargets); + pExchange->node.precision = pChild->precision; + pExchange->node.pTargets = nodesCloneList(pChild->pTargets); if (NULL == pExchange->node.pTargets) { return TSDB_CODE_OUT_OF_MEMORY; } + *pOutput = pExchange; + return TSDB_CODE_SUCCESS; +} + +static int32_t splCreateExchangeNodeForSubplan(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode, + ESubplanType subplanType) { + SExchangeLogicNode* pExchange = NULL; + if (TSDB_CODE_SUCCESS != splCreateExchangeNode(pCxt, pSplitNode, &pExchange)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pSubplan->subplanType = subplanType; if (NULL == pSplitNode->pParent) { @@ -97,7 +108,7 @@ static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla static bool splMatch(SSplitContext* pCxt, SLogicSubplan* pSubplan, int32_t flag, FSplFindSplitNode func, void* pInfo) { if (!SPLIT_FLAG_TEST_MASK(pSubplan->splitFlag, flag)) { - if (func(pSubplan, pInfo)) { + if (func(pCxt, pSubplan, pInfo)) { return true; } } @@ -125,41 +136,47 @@ static bool stbSplHasGatherExecFunc(const SNodeList* pFuncs) { return false; } -static bool stbSplIsMultiTbScan(SScanLogicNode* pScan) { - return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1); +static bool stbSplIsMultiTbScan(bool streamQuery, SScanLogicNode* pScan) { + return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1) || + (streamQuery && TSDB_SUPER_TABLE == pScan->tableType); } -static bool stbSplHasMultiTbScan(SLogicNode* pNode) { +static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) { if (1 != LIST_LENGTH(pNode->pChildren)) { return false; } SNode* pChild = nodesListGetNode(pNode->pChildren, 0); - return (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan((SScanLogicNode*)pChild)); + return (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pChild)); } -static bool stbSplNeedSplit(SLogicNode* pNode) { +static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { switch (nodeType(pNode)) { - // case QUERY_NODE_LOGIC_PLAN_AGG: - // return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(pNode); - case QUERY_NODE_LOGIC_PLAN_WINDOW: - return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(pNode); + case QUERY_NODE_LOGIC_PLAN_AGG: + return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); + case QUERY_NODE_LOGIC_PLAN_WINDOW: { + SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode; + if (WINDOW_TYPE_INTERVAL != pWindow->winType) { + return false; + } + return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); + } // case QUERY_NODE_LOGIC_PLAN_SORT: - // return stbSplHasMultiTbScan(pNode); + // return stbSplHasMultiTbScan(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_SCAN: - return stbSplIsMultiTbScan((SScanLogicNode*)pNode); + return stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pNode); default: break; } return false; } -static SLogicNode* stbSplMatchByNode(SLogicNode* pNode) { - if (stbSplNeedSplit(pNode)) { +static SLogicNode* stbSplMatchByNode(bool streamQuery, SLogicNode* pNode) { + if (stbSplNeedSplit(streamQuery, pNode)) { return pNode; } SNode* pChild; FOREACH(pChild, pNode->pChildren) { - SLogicNode* pSplitNode = stbSplMatchByNode((SLogicNode*)pChild); + SLogicNode* pSplitNode = stbSplMatchByNode(streamQuery, (SLogicNode*)pChild); if (NULL != pSplitNode) { return pSplitNode; } @@ -167,8 +184,8 @@ static SLogicNode* stbSplMatchByNode(SLogicNode* pNode) { return NULL; } -static bool stbSplFindSplitNode(SLogicSubplan* pSubplan, SStableSplitInfo* pInfo) { - SLogicNode* pSplitNode = stbSplMatchByNode(pSubplan->pNode); +static bool stbSplFindSplitNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SStableSplitInfo* pInfo) { + SLogicNode* pSplitNode = stbSplMatchByNode(pCxt->pPlanCxt->streamQuery, pSubplan->pNode); if (NULL != pSplitNode) { pInfo->pSplitNode = pSplitNode; pInfo->pSubplan = pSubplan; @@ -278,7 +295,8 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic return code; } -static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicNode* pParent, SLogicNode* pPartChild) { +static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicNode* pParent, SNodeList* pMergeKeys, + SLogicNode* pPartChild) { SMergeLogicNode* pMerge = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE); if (NULL == pMerge) { return TSDB_CODE_OUT_OF_MEMORY; @@ -287,25 +305,48 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicNode* pParent, S pMerge->srcGroupId = pCxt->groupId; pMerge->node.pParent = pParent; pMerge->node.precision = pPartChild->precision; - int32_t code = nodesListMakeStrictAppend(&pMerge->pMergeKeys, nodesCloneNode(((SWindowLogicNode*)pParent)->pTspk)); - if (TSDB_CODE_SUCCESS == code) { - pMerge->node.pTargets = nodesCloneList(pPartChild->pTargets); - if (NULL == pMerge->node.pTargets) { - code = TSDB_CODE_OUT_OF_MEMORY; - } + pMerge->pMergeKeys = pMergeKeys; + pMerge->node.pTargets = nodesCloneList(pPartChild->pTargets); + if (NULL == pMerge->node.pTargets) { + nodesDestroyNode(pMerge); + return TSDB_CODE_OUT_OF_MEMORY; } + + return nodesListMakeAppend(&pParent->pChildren, pMerge); +} + +static int32_t stbSplCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pParent, SLogicNode* pPartChild) { + SExchangeLogicNode* pExchange = NULL; + int32_t code = splCreateExchangeNode(pCxt, pPartChild, &pExchange); if (TSDB_CODE_SUCCESS == code) { - code = nodesListMakeAppend(&pParent->pChildren, pMerge); + code = nodesListMakeAppend(&pParent->pChildren, pExchange); } + return code; +} +static int32_t stbSplSplitWindowNodeForBatch(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + SLogicNode* pPartWindow = NULL; + int32_t code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow); + if (TSDB_CODE_SUCCESS == code) { + ((SWindowLogicNode*)pPartWindow)->intervalAlgo = INTERVAL_ALGO_HASH; + ((SWindowLogicNode*)pInfo->pSplitNode)->intervalAlgo = INTERVAL_ALGO_SORT_MERGE; + code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, + splCreateScanSubplan(pCxt, pPartWindow, SPLIT_FLAG_STABLE_SPLIT)); + } + pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; return code; } -static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { +static int32_t stbSplSplitWindowNodeForStream(SSplitContext* pCxt, SStableSplitInfo* pInfo) { SLogicNode* pPartWindow = NULL; int32_t code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, pInfo->pSplitNode, pPartWindow); + ((SWindowLogicNode*)pPartWindow)->intervalAlgo = INTERVAL_ALGO_STREAM_SEMI; + ((SWindowLogicNode*)pInfo->pSplitNode)->intervalAlgo = INTERVAL_ALGO_STREAM_FINAL; + code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow); } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, @@ -315,8 +356,134 @@ static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInf return code; } +static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + if (pCxt->pPlanCxt->streamQuery) { + return stbSplSplitWindowNodeForStream(pCxt, pInfo); + } else { + return stbSplSplitWindowNodeForBatch(pCxt, pInfo); + } +} + +static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pOutput) { + SNodeList* pFunc = pMergeAgg->pAggFuncs; + pMergeAgg->pAggFuncs = NULL; + SNodeList* pGroupKeys = pMergeAgg->pGroupKeys; + pMergeAgg->pGroupKeys = NULL; + SNodeList* pTargets = pMergeAgg->node.pTargets; + pMergeAgg->node.pTargets = NULL; + SNodeList* pChildren = pMergeAgg->node.pChildren; + pMergeAgg->node.pChildren = NULL; + + int32_t code = TSDB_CODE_SUCCESS; + SAggLogicNode* pPartAgg = nodesCloneNode(pMergeAgg); + if (NULL == pPartAgg) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { + pPartAgg->pGroupKeys = pGroupKeys; + code = createColumnByRewriteExps(pPartAgg->pGroupKeys, &pPartAgg->node.pTargets); + } + if (TSDB_CODE_SUCCESS == code && NULL != pGroupKeys) { + pMergeAgg->pGroupKeys = nodesCloneList(pPartAgg->node.pTargets); + if (NULL == pMergeAgg->pGroupKeys) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS == code) { + pMergeAgg->node.pTargets = pTargets; + pPartAgg->node.pChildren = pChildren; + + code = stbSplRewriteFuns(pFunc, &pPartAgg->pAggFuncs, &pMergeAgg->pAggFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pPartAgg->pAggFuncs, &pPartAgg->node.pTargets); + } + + nodesDestroyList(pFunc); + if (TSDB_CODE_SUCCESS == code) { + *pOutput = (SLogicNode*)pPartAgg; + } else { + nodesDestroyNode(pPartAgg); + } + + return code; +} + +static int32_t stbSplSplitAggNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + SLogicNode* pPartAgg = NULL; + int32_t code = stbSplCreatePartAggNode((SAggLogicNode*)pInfo->pSplitNode, &pPartAgg); + if (TSDB_CODE_SUCCESS == code) { + code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartAgg); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, + splCreateScanSubplan(pCxt, pPartAgg, SPLIT_FLAG_STABLE_SPLIT)); + } + pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + return code; +} + +static int32_t stbSplCreatePartSortNode(SSortLogicNode* pMergeSort, SLogicNode** pOutput) { + SNodeList* pSortKeys = pMergeSort->pSortKeys; + pMergeSort->pSortKeys = NULL; + SNodeList* pTargets = pMergeSort->node.pTargets; + pMergeSort->node.pTargets = NULL; + SNodeList* pChildren = pMergeSort->node.pChildren; + pMergeSort->node.pChildren = NULL; + + int32_t code = TSDB_CODE_SUCCESS; + SSortLogicNode* pPartSort = nodesCloneNode(pMergeSort); + if (NULL == pPartSort) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + pMergeSort->node.pTargets = pTargets; + pPartSort->node.pChildren = pChildren; + if (TSDB_CODE_SUCCESS == code) { + pPartSort->pSortKeys = pSortKeys; + code = createColumnByRewriteExps(pPartSort->pSortKeys, &pPartSort->node.pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + pMergeSort->pSortKeys = nodesCloneList(pPartSort->node.pTargets); + if (NULL == pMergeSort->pSortKeys) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pOutput = (SLogicNode*)pPartSort; + } else { + nodesDestroyNode(pPartSort); + } + + return code; +} + +static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + SLogicNode* pPartSort = NULL; + int32_t code = stbSplCreatePartSortNode((SSortLogicNode*)pInfo->pSplitNode, &pPartSort); + if (TSDB_CODE_SUCCESS == code) { + SNodeList* pMergeKeys = nodesCloneList(((SSortLogicNode*)pInfo->pSplitNode)->pSortKeys); + if (NULL != pMergeKeys) { + code = stbSplCreateMergeNode(pCxt, pInfo->pSplitNode, pMergeKeys, pPartSort); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(pMergeKeys); + } + } else { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, + splCreateScanSubplan(pCxt, pPartSort, SPLIT_FLAG_STABLE_SPLIT)); + } + pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + return code; +} + static int32_t stbSplSplitScanNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { - int32_t code = splCreateExchangeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, SUBPLAN_TYPE_MERGE); + int32_t code = splCreateExchangeNodeForSubplan(pCxt, pInfo->pSubplan, pInfo->pSplitNode, SUBPLAN_TYPE_MERGE); if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT)); @@ -325,6 +492,10 @@ static int32_t stbSplSplitScanNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) } static int32_t stableSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { + if (pCxt->pPlanCxt->rSmaQuery) { + return TSDB_CODE_SUCCESS; + } + SStableSplitInfo info = {0}; if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_STABLE_SPLIT, (FSplFindSplitNode)stbSplFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; @@ -332,9 +503,15 @@ static int32_t stableSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(info.pSplitNode)) { + case QUERY_NODE_LOGIC_PLAN_AGG: + code = stbSplSplitAggNode(pCxt, &info); + break; case QUERY_NODE_LOGIC_PLAN_WINDOW: code = stbSplSplitWindowNode(pCxt, &info); break; + case QUERY_NODE_LOGIC_PLAN_SORT: + code = stbSplSplitSortNode(pCxt, &info); + break; case QUERY_NODE_LOGIC_PLAN_SCAN: code = stbSplSplitScanNode(pCxt, &info); break; @@ -375,7 +552,7 @@ static SJoinLogicNode* sigTbJoinSplMatchByNode(SLogicNode* pNode) { return NULL; } -static bool sigTbJoinSplFindSplitNode(SLogicSubplan* pSubplan, SSigTbJoinSplitInfo* pInfo) { +static bool sigTbJoinSplFindSplitNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SSigTbJoinSplitInfo* pInfo) { SJoinLogicNode* pJoin = sigTbJoinSplMatchByNode(pSubplan->pNode); if (NULL != pJoin) { pInfo->pJoin = pJoin; @@ -390,7 +567,7 @@ static int32_t singleTableJoinSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)sigTbJoinSplFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; } - int32_t code = splCreateExchangeNode(pCxt, info.pSubplan, info.pSplitNode, info.pSubplan->subplanType); + int32_t code = splCreateExchangeNodeForSubplan(pCxt, info.pSubplan, info.pSplitNode, info.pSubplan->subplanType); if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, splCreateScanSubplan(pCxt, info.pSplitNode, 0)); } @@ -489,7 +666,7 @@ static SLogicNode* unAllSplMatchByNode(SLogicNode* pNode) { return NULL; } -static bool unAllSplFindSplitNode(SLogicSubplan* pSubplan, SUnionAllSplitInfo* pInfo) { +static bool unAllSplFindSplitNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SUnionAllSplitInfo* pInfo) { SLogicNode* pSplitNode = unAllSplMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { pInfo->pProject = (SProjectLogicNode*)pSplitNode; @@ -581,7 +758,7 @@ static int32_t unDistSplCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* p return nodesListMakeAppend(&pAgg->node.pChildren, pExchange); } -static bool unDistSplFindSplitNode(SLogicSubplan* pSubplan, SUnionDistinctSplitInfo* pInfo) { +static bool unDistSplFindSplitNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SUnionDistinctSplitInfo* pInfo) { SLogicNode* pSplitNode = unDistSplMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { pInfo->pAgg = (SAggLogicNode*)pSplitNode; @@ -623,9 +800,10 @@ static void dumpLogicSubplan(const char* pRuleName, SLogicSubplan* pSubplan) { taosMemoryFree(pStr); } -static int32_t applySplitRule(SLogicSubplan* pSubplan) { - SSplitContext cxt = {.queryId = pSubplan->id.queryId, .groupId = pSubplan->id.groupId + 1, .split = false}; - bool split = false; +static int32_t applySplitRule(SPlanContext* pCxt, SLogicSubplan* pSubplan) { + SSplitContext cxt = { + .pPlanCxt = pCxt, .queryId = pSubplan->id.queryId, .groupId = pSubplan->id.groupId + 1, .split = false}; + bool split = false; do { split = false; for (int32_t i = 0; i < splitRuleNum; ++i) { @@ -651,6 +829,16 @@ static void doSetLogicNodeParent(SLogicNode* pNode, SLogicNode* pParent) { static void setLogicNodeParent(SLogicNode* pNode) { doSetLogicNodeParent(pNode, NULL); } +static void setVgroupsInfo(SLogicNode* pNode, SLogicSubplan* pSubplan) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + TSWAP(((SScanLogicNode*)pNode)->pVgroupList, pSubplan->pVgroupList); + return; + } + + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { setVgroupsInfo((SLogicNode*)pChild, pSubplan); } +} + int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan** pLogicSubplan) { SLogicSubplan* pSubplan = (SLogicSubplan*)nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); if (NULL == pSubplan) { @@ -662,17 +850,21 @@ int32_t splitLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode, SLogicSubplan nodesDestroyNode(pSubplan); return TSDB_CODE_OUT_OF_MEMORY; } - if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIF == nodeType(pLogicNode)) { + + pSubplan->id.queryId = pCxt->queryId; + pSubplan->id.groupId = 1; + setLogicNodeParent(pSubplan->pNode); + + int32_t code = TSDB_CODE_SUCCESS; + if (QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY == nodeType(pLogicNode)) { pSubplan->subplanType = SUBPLAN_TYPE_MODIFY; - TSWAP(((SVnodeModifLogicNode*)pLogicNode)->pDataBlocks, ((SVnodeModifLogicNode*)pSubplan->pNode)->pDataBlocks); + TSWAP(((SVnodeModifyLogicNode*)pLogicNode)->pDataBlocks, ((SVnodeModifyLogicNode*)pSubplan->pNode)->pDataBlocks); + setVgroupsInfo(pSubplan->pNode, pSubplan); } else { pSubplan->subplanType = SUBPLAN_TYPE_SCAN; + code = applySplitRule(pCxt, pSubplan); } - pSubplan->id.queryId = pCxt->queryId; - pSubplan->id.groupId = 1; - setLogicNodeParent(pSubplan->pNode); - int32_t code = applySplitRule(pSubplan); if (TSDB_CODE_SUCCESS == code) { *pLogicSubplan = pSubplan; } else { diff --git a/source/libs/planner/test/CMakeLists.txt b/source/libs/planner/test/CMakeLists.txt index abea60b0c798a055617abf3693be25f365fbc867..b34ffa7bb1017e25c2b08a7b077430fea74bd923 100644 --- a/source/libs/planner/test/CMakeLists.txt +++ b/source/libs/planner/test/CMakeLists.txt @@ -32,9 +32,9 @@ if(${BUILD_WINGETOPT}) target_link_libraries(plannerTest PUBLIC wingetopt) endif() -if(NOT TD_WINDOWS) +# if(NOT TD_WINDOWS) add_test( NAME plannerTest COMMAND plannerTest ) -endif(NOT TD_WINDOWS) +# endif(NOT TD_WINDOWS) diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index cf516034707e53a230d4e2e7af6fb81c3b8aaecf..201df2efde6236c5ae68aaedf04625a2c4acda19 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -67,3 +67,11 @@ TEST_F(PlanGroupByTest, selectFunc) { run("SELECT MAX(c1), c2 FROM t1 GROUP BY c3"); run("SELECT MAX(c1), t1.* FROM t1 GROUP BY c3"); } + +TEST_F(PlanGroupByTest, stable) { + useDb("root", "test"); + + run("SELECT COUNT(*) FROM st1"); + + run("SELECT COUNT(*) FROM st1 GROUP BY c1"); +} diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 4234a1320a433da1184cdca6d5f567fa3a2005c6..84ccea668d55cbcb0edff9920c25faa0ec0b57aa 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -36,6 +36,10 @@ TEST_F(PlanOptimizeTest, ConditionPushDown) { useDb("root", "test"); run("SELECT ts, c1 FROM st1 WHERE tag1 > 4"); + + run("SELECT ts, c1 FROM st1 WHERE tag1 > 4 or tag1 < 2"); + + run("SELECT ts, c1 FROM st1 WHERE tag1 > 4 AND tag2 = 'hello'"); } TEST_F(PlanOptimizeTest, orderByPrimaryKey) { diff --git a/source/libs/planner/test/planOrderByTest.cpp b/source/libs/planner/test/planOrderByTest.cpp index d95d1bdf1d429d9ea6c5e62c5a280dbb5c6de477..ef6f438b55c60f89e96de40c9eeeff71f3f2fcdf 100644 --- a/source/libs/planner/test/planOrderByTest.cpp +++ b/source/libs/planner/test/planOrderByTest.cpp @@ -23,20 +23,27 @@ class PlanOrderByTest : public PlannerTestBase {}; TEST_F(PlanOrderByTest, basic) { useDb("root", "test"); - // order by key is in the projection list - run("select c1 from t1 order by c1"); - // order by key is not in the projection list - run("select c1 from t1 order by c2"); + // ORDER BY key is in the projection list + run("SELECT c1 FROM t1 ORDER BY c1"); + // ORDER BY key is not in the projection list + run("SELECT c1 FROM t1 ORDER BY c2"); } TEST_F(PlanOrderByTest, expr) { useDb("root", "test"); - run("select * from t1 order by c1 + 10, c2"); + run("SELECT * FROM t1 ORDER BY c1 + 10, c2"); } TEST_F(PlanOrderByTest, nullsOrder) { useDb("root", "test"); - run("select * from t1 order by c1 desc nulls first"); + run("SELECT * FROM t1 ORDER BY c1 DESC NULLS FIRST"); +} + +TEST_F(PlanOrderByTest, stable) { + useDb("root", "test"); + + // ORDER BY key is in the projection list + run("SELECT c1 FROM st1 ORDER BY c1"); } diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index 67c09d706e34ea44ab0c4070d9bbb665a15dded1..d5d37fda648f04941628bdfe705deb90a389d623 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -33,6 +33,12 @@ TEST_F(PlanOtherTest, createStream) { "interval(10s)"); } +TEST_F(PlanOtherTest, createStreamUseSTable) { + useDb("root", "test"); + + run("create stream if not exists s1 as select count(*) from st1 interval(10s)"); +} + TEST_F(PlanOtherTest, createSmaIndex) { useDb("root", "test"); @@ -54,3 +60,15 @@ TEST_F(PlanOtherTest, show) { run("SHOW DATABASES"); } + +TEST_F(PlanOtherTest, delete) { + useDb("root", "test"); + + run("DELETE FROM t1"); + + run("DELETE FROM t1 WHERE ts > now - 2d and ts < now - 1d"); + + run("DELETE FROM st1"); + + run("DELETE FROM st1 WHERE ts > now - 2d and ts < now - 1d AND tag1 = 10"); +} diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index 42c8558239b4389e0a0c469fdaa5842b53dc2398..acac3d505306e40e83bf9969273eba7cf61ad2a0 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -16,6 +16,7 @@ #include #include +#include "functionMgt.h" #include "getopt.h" #include "mockCatalog.h" #include "planTestUtil.h" @@ -23,6 +24,7 @@ class PlannerEnv : public testing::Environment { public: virtual void SetUp() { + fmFuncMgtInit(); initMetaDataEnv(); generateMetaData(); initLog(TD_TMP_DIR_PATH "td"); diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 4fe3c1839310be9e264f7241fbc0cce48837a05c..082db6428f79575cf29a635a1965f214893d22a5 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -33,7 +33,8 @@ extern "C" { #define QW_DEFAULT_TASK_NUMBER 10000 #define QW_DEFAULT_SCH_TASK_NUMBER 10000 #define QW_DEFAULT_SHORT_RUN_TIMES 2 -#define QW_DEFAULT_HEARTBEAT_MSEC 3000 +#define QW_DEFAULT_HEARTBEAT_MSEC 5000 +#define QW_SCH_TIMEOUT_MSEC 180000 enum { QW_PHASE_PRE_QUERY = 1, @@ -137,7 +138,7 @@ typedef struct SQWTaskCtx { } SQWTaskCtx; typedef struct SQWSchStatus { - int32_t lastAccessTs; // timestamp in second + int64_t hbBrokenTs; // timestamp in msecond SRWLatch hbConnLock; SRpcHandleInfo hbConnInfo; SQueryNodeEpId hbEpId; @@ -354,6 +355,8 @@ int32_t qwOpenRef(void); void qwSetHbParam(int64_t refId, SQWHbParam **pParam); int32_t qwUpdateTimeInQueue(SQWorker *mgmt, int64_t ts, EQueueType type); int64_t qwGetTimeInQueue(SQWorker *mgmt, EQueueType type); +void qwClearExpiredSch(SArray* pExpiredSch); +int32_t qwAcquireScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch); void qwDbgDumpMgmtInfo(SQWorker *mgmt); int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore); diff --git a/source/libs/qworker/inc/qwMsg.h b/source/libs/qworker/inc/qwMsg.h index ede085b6f912842c85dce8597374613856d80f1f..5a8a45cd5150bc5e2aa009f2f3690e8f98a14c05 100644 --- a/source/libs/qworker/inc/qwMsg.h +++ b/source/libs/qworker/inc/qwMsg.h @@ -23,6 +23,7 @@ extern "C" { #include "qwInt.h" #include "dataSinkMgt.h" +int32_t qwPrerocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain); int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg); int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg); diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 27fe22295d3706eb21a237f8d662e34b4dce9b36..3914541157e86aa67a55c56cd56afb76da2061cf 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -36,7 +36,8 @@ int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, break; case JOB_TASK_STATUS_NOT_START: - if (newStatus != JOB_TASK_STATUS_CANCELLED) { + if (newStatus != JOB_TASK_STATUS_DROPPING && newStatus != JOB_TASK_STATUS_EXECUTING + && newStatus != JOB_TASK_STATUS_FAILED) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index f8205a6bb4b2d004bc1c4f35b67eabc5635c5ca7..2f1ecc51723a4799071292f5d7eb8d3b3d2f65ab 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -248,8 +248,8 @@ int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t sId, SRpcHandleInfo * return TSDB_CODE_SUCCESS; } -int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) { - if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { +int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg) { + if (NULL == qWorkerMgmt || NULL == pMsg) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -257,9 +257,6 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int SSubQueryMsg *msg = pMsg->pCont; SQWorker * mgmt = (SQWorker *)qWorkerMgmt; - qwUpdateTimeInQueue(mgmt, ts, QUERY_QUEUE); - QW_STAT_INC(mgmt->stat.msgStat.queryProcessed, 1); - if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { QW_ELOG("invalid query msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -277,6 +274,44 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int uint64_t tId = msg->taskId; int64_t rId = msg->refId; + SQWMsg qwMsg = {.msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connInfo = pMsg->info}; + + QW_SCH_TASK_DLOG("prerocessQuery start, handle:%p", pMsg->info.handle); + QW_ERR_RET(qwPrerocessQuery(QW_FPARAMS(), &qwMsg)); + QW_SCH_TASK_DLOG("prerocessQuery end, handle:%p", pMsg->info.handle); + + return TSDB_CODE_SUCCESS; +} + +int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) { + if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { + QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + int32_t code = 0; + SSubQueryMsg *msg = pMsg->pCont; + SQWorker * mgmt = (SQWorker *)qWorkerMgmt; + + qwUpdateTimeInQueue(mgmt, ts, QUERY_QUEUE); + QW_STAT_INC(mgmt->stat.msgStat.queryProcessed, 1); + + if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { + QW_ELOG("invalid query msg, msg:%p, msgLen:%d", msg, pMsg->contLen); + QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + msg->sId = msg->sId; + msg->queryId = msg->queryId; + msg->taskId = msg->taskId; + msg->refId = msg->refId; + msg->phyLen = msg->phyLen; + msg->sqlLen = msg->sqlLen; + + uint64_t sId = msg->sId; + uint64_t qId = msg->queryId; + uint64_t tId = msg->taskId; + int64_t rId = msg->refId; + SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connInfo = pMsg->info}; char * sql = strndup(msg->msg, msg->sqlLen); QW_SCH_TASK_DLOG("processQuery start, node:%p, handle:%p, sql:%s", node, pMsg->info.handle, sql); diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 3d0204e355bd228836a2729cc9e52e74981c4e0f..8bfb80f0616e223f96eafcd34e8d89768ffb98ca 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -535,3 +535,9 @@ int64_t qwGetTimeInQueue(SQWorker *mgmt, EQueueType type) { return -1; } + +void qwClearExpiredSch(SArray* pExpiredSch) { + +} + + diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index fd16fa53b7a6d9b4c5e460719c3fc09134ef034b..44a8fdf7f4e29c6ad8ee0e8dc5b26612bc1e665f 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -21,10 +21,12 @@ int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *re SSchedulerHbRsp rsp = {0}; SQWSchStatus *sch = NULL; - QW_ERR_RET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); + QW_ERR_RET(qwAcquireScheduler(mgmt, req->sId, QW_READ, &sch)); QW_LOCK(QW_WRITE, &sch->hbConnLock); + sch->hbBrokenTs = taosGetTimestampMs(); + if (qwMsg->connInfo.handle == sch->hbConnInfo.handle) { tmsgReleaseHandle(&sch->hbConnInfo, TAOS_CONN_SERVER); sch->hbConnInfo.handle = NULL; @@ -248,11 +250,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_TASK_DLOG("start to handle event at phase %s", qwPhaseStr(phase)); - if (QW_PHASE_PRE_QUERY == phase) { - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), &ctx)); - } else { - QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); - } + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -285,7 +283,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu break; } - QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING)); break; } case QW_PHASE_PRE_FETCH: { @@ -437,7 +435,7 @@ _return: QW_RET(code); } -int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain) { +int32_t qwPrerocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; bool queryRsped = false; SSubplan *plan = NULL; @@ -448,6 +446,32 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex QW_ERR_JRET(qwRegisterQueryBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo)); + QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), &ctx)); + + ctx->ctrlConnInfo = qwMsg->connInfo; + + QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_NOT_START)); + +_return: + + if (ctx) { + QW_UPDATE_RSP_CODE(ctx, code); + qwReleaseTaskCtx(mgmt, ctx); + } + + QW_RET(TSDB_CODE_SUCCESS); +} + + +int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain) { + int32_t code = 0; + bool queryRsped = false; + SSubplan *plan = NULL; + SQWPhaseInput input = {0}; + qTaskInfo_t pTaskInfo = NULL; + DataSinkHandle sinkHandle = NULL; + SQWTaskCtx *ctx = NULL; + QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, NULL)); QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); @@ -455,8 +479,6 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex atomic_store_8(&ctx->taskType, taskType); atomic_store_8(&ctx->explain, explain); - ctx->ctrlConnInfo = qwMsg->connInfo; - /*QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg);*/ code = qStringToSubplan(qwMsg->msg, &plan); @@ -663,7 +685,7 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { // TODO : TASK ALREADY REMOVED AND A NEW DROP MSG RECEIVED - QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), &ctx)); + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -774,6 +796,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { SQWSchStatus *sch = NULL; int32_t taskNum = 0; SQWHbInfo *rspList = NULL; + SArray *pExpiredSch = NULL; int32_t code = 0; qwDbgDumpMgmtInfo(mgmt); @@ -789,8 +812,11 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { } rspList = taosMemoryCalloc(schNum, sizeof(SQWHbInfo)); - if (NULL == rspList) { + pExpiredSch = taosArrayInit(schNum, sizeof(uint64_t)); + if (NULL == rspList || NULL == pExpiredSch) { QW_UNLOCK(QW_READ, &mgmt->schLock); + taosMemoryFree(rspList); + taosArrayDestroy(pExpiredSch); QW_ELOG("calloc %d SQWHbInfo failed", schNum); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); qwRelease(refId); @@ -800,6 +826,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { void *key = NULL; size_t keyLen = 0; int32_t i = 0; + int64_t currentMs = taosGetTimestampMs(); void *pIter = taosHashIterate(mgmt->schHash, NULL); while (pIter) { @@ -807,6 +834,11 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { if (NULL == sch->hbConnInfo.handle) { uint64_t *sId = taosHashGetKey(pIter, NULL); QW_TLOG("cancel send hb to sch %" PRIx64 " cause of no connection handle", *sId); + + if (sch->hbBrokenTs > 0 && ((currentMs - sch->hbBrokenTs) > QW_SCH_TIMEOUT_MSEC) && taosHashGetSize(sch->tasksHash) <= 0) { + taosArrayPush(pExpiredSch, sId); + } + pIter = taosHashIterate(mgmt->schHash, pIter); continue; } @@ -832,7 +864,12 @@ _return: tFreeSSchedulerHbRsp(&rspList[j].rsp); } + if (taosArrayGetSize(pExpiredSch) > 0) { + qwClearExpiredSch(pExpiredSch); + } + taosMemoryFreeClear(rspList); + taosArrayDestroy(pExpiredSch); taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); qwRelease(refId); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 16dcd7b6e025dd5761202308d00c20435d9a55f0..5bb6acee6014fb9352b9ac11304500df33e3ffcd 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -172,8 +172,7 @@ int32_t qwtPutReqToFetchQueue(void *node, struct SRpcMsg *pMsg) { return 0; } - -int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { +int32_t qwtPutReqToQueue(void *node, EQueueType qtype, struct SRpcMsg *pMsg) { taosWLockLatch(&qwtTestQueryQueueLock); struct SRpcMsg *newMsg = (struct SRpcMsg *)taosMemoryCalloc(1, sizeof(struct SRpcMsg)); memcpy(newMsg, pMsg, sizeof(struct SRpcMsg)); @@ -181,7 +180,7 @@ int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { if (qwtTestQueryQueueWIdx >= qwtTestQueryQueueSize) { qwtTestQueryQueueWIdx = 0; } - + qwtTestQueryQueueNum++; if (qwtTestQueryQueueWIdx == qwtTestQueryQueueRIdx) { @@ -702,9 +701,9 @@ void *qwtclientThread(void *param) { while (!qwtTestStop) { qwtTestCaseFinished = false; - + qwtBuildQueryReqMsg(&queryRpc); - qwtPutReqToQueue((void *)0x1, &queryRpc); + qwtPutReqToQueue((void *)0x1, QUERY_QUEUE, &queryRpc); while (!qwtTestCaseFinished) { taosUsleep(1); @@ -874,7 +873,7 @@ TEST(seqTest, normalCase) { SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; - msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); @@ -907,10 +906,10 @@ TEST(seqTest, cancelFirst) { stubSetStringToPlan(); stubSetRpcSendResponse(); - + SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; - msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); @@ -944,10 +943,10 @@ TEST(seqTest, randCase) { stubSetCreateExecTask(); taosSeedRand(taosGetTimestampSec()); - + SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; - msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); @@ -1015,10 +1014,10 @@ TEST(seqTest, multithreadRand) { stubSetGetDataBlock(); taosSeedRand(taosGetTimestampSec()); - + SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; - msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); @@ -1081,7 +1080,7 @@ TEST(rcTest, shortExecshortDelay) { SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; - msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); @@ -1165,7 +1164,7 @@ TEST(rcTest, longExecshortDelay) { SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; - msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); @@ -1251,7 +1250,7 @@ TEST(rcTest, shortExeclongDelay) { SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; - msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); @@ -1332,10 +1331,10 @@ TEST(rcTest, dropTest) { stubSetGetDataBlock(); taosSeedRand(taosGetTimestampSec()); - + SMsgCb msgCb = {0}; msgCb.mgmt = (void *)mockPointer; - msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + msgCb.putToQueueFp = (PutToQueueFp)qwtPutReqToQueue; code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index f4cd9dd1c11ce799019a4230ec33489bbbb771f6..54e873065b20c6317d96c6a56240c7920371a663 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -350,7 +350,7 @@ struct SFilterInfo { extern bool filterDoCompare(__compar_fn_t func, uint8_t optr, void *left, void *right); extern __compar_fn_t filterGetCompFunc(int32_t type, int32_t optr); - +extern OptrStr gOptrStr[]; #ifdef __cplusplus } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 195ec8a57791062cbca0e4c1a39ccce1866a5095..5fdbbfe8105953784091216fce49715024ec0112 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -29,8 +29,9 @@ OptrStr gOptrStr[] = { {OP_TYPE_SUB, "-"}, {OP_TYPE_MULTI, "*"}, {OP_TYPE_DIV, "/"}, - {OP_TYPE_MOD, "%"}, - + {OP_TYPE_REM, "%"}, + {OP_TYPE_MINUS, "minus"}, + {OP_TYPE_ASSIGN, "assign"}, // bit operator {OP_TYPE_BIT_AND, "&"}, {OP_TYPE_BIT_OR, "|"}, diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 370ea8fa2ecd6fb1dab661f931199f08e6f8f65a..2b15f5031eaf40d0bad6eeb658be2c4d232866df 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -449,7 +449,7 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu bool hasNull = false; for (int32_t i = 0; i < inputNum; ++i) { if (colDataIsNull_s(pInputData[i], k) || - GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_NULL) { + IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[i]))) { colDataAppendNULL(pOutputData, k); hasNull = true; break; @@ -526,7 +526,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p for (int32_t k = 0; k < numOfRows; ++k) { if (colDataIsNull_s(pInputData[0], k) || - GET_PARAM_TYPE(&pInput[0]) == TSDB_DATA_TYPE_NULL) { + IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[0]))) { colDataAppendNULL(pOutputData, k); continue; } @@ -535,7 +535,7 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p bool hasNull = false; for (int32_t i = 1; i < inputNum; ++i) { if (colDataIsNull_s(pInputData[i], k) || - GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_NULL) { + IS_NULL_TYPE(GET_PARAM_TYPE(&pInput[i]))) { hasNull = true; break; } @@ -718,7 +718,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int16_t outputType = GET_PARAM_TYPE(&pOutput[0]); int64_t outputLen = GET_PARAM_BYTES(&pOutput[0]); - char *outputBuf = taosMemoryCalloc(outputLen * pInput[0].numOfRows, 1); + char *outputBuf = taosMemoryCalloc(outputLen * pInput[0].numOfRows + 1, 1); char *output = outputBuf; for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { @@ -1221,8 +1221,8 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p if (IS_VAR_DATA_TYPE(type)) { /* datetime format strings */ int32_t ret = convertStringToTimestamp(type, input[k], TSDB_TIME_PRECISION_NANO, &timeVal[k]); if (ret != TSDB_CODE_SUCCESS) { - colDataAppendNULL(pOutput->columnData, i); - continue; + hasNull = true; + break; } } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { /* unix timestamp or ts column*/ GET_TYPED_DATA(timeVal[k], int64_t, type, input[k]); @@ -1247,8 +1247,8 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { timeVal[k] = timeVal[k]; } else { - colDataAppendNULL(pOutput->columnData, i); - continue; + hasNull = true; + break; } } } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index e844b3cdb6b29612aab571acfb2739e47044770b..d9f582e66946fee3c897d9c85009f79fe70de1bb 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -29,6 +29,12 @@ #define LEFT_COL ((pLeftCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pLeftCol : pLeftCol->pData)) #define RIGHT_COL ((pRightCol->info.type == TSDB_DATA_TYPE_JSON ? (void*)pRightCol : pRightCol->pData)) +#define IS_NULL colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i) \ + || IS_JSON_NULL(pLeft->columnData->info.type, colDataGetVarData(pLeft->columnData, i)) \ + || IS_JSON_NULL(pRight->columnData->info.type, colDataGetVarData(pRight->columnData, i)) + +#define IS_HELPER_NULL(col,i) colDataIsNull_s(col, i) || IS_JSON_NULL(col->info.type, colDataGetVarData(col, i)) + void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType){ switch (outType) { case TSDB_DATA_TYPE_BOOL: { @@ -387,7 +393,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in pOut->numOfRows = pIn->numOfRows; for (int32_t i = 0; i < pIn->numOfRows; ++i) { - if (colDataIsNull_s(pIn->columnData, i)) { + if (IS_HELPER_NULL(pIn->columnData, i)) { colDataAppendNULL(pOut->columnData, i); continue; } @@ -396,8 +402,7 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in int32_t convertType = inType; if(inType == TSDB_DATA_TYPE_JSON){ if(*data == TSDB_DATA_TYPE_NULL) { - colDataAppendNULL(pOut->columnData, i); - continue; + ASSERT(0); } else if(*data == TSDB_DATA_TYPE_NCHAR) { data += CHAR_BYTES; @@ -447,13 +452,13 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){ return out; } -void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){ +bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){ if(optr == OP_TYPE_JSON_CONTAINS) { - return; + return true; } if(typeLeft != TSDB_DATA_TYPE_JSON && typeRight != TSDB_DATA_TYPE_JSON){ - return; + return true; } if(typeLeft == TSDB_DATA_TYPE_JSON){ @@ -464,15 +469,22 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t typeRight = **pRightData; (*pRightData) ++; } + + if(optr == OP_TYPE_LIKE || optr == OP_TYPE_NOT_LIKE || optr == OP_TYPE_MATCH || optr == OP_TYPE_NMATCH){ + if(typeLeft != TSDB_DATA_TYPE_NCHAR && typeLeft != TSDB_DATA_TYPE_BINARY){ + return false; + } + } + if(typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL){ *isNull = true; - return; + return true; } int8_t type = vectorGetConvertType(typeLeft, typeRight); if(type == 0) { *fp = filterGetCompFunc(typeLeft, optr); - return; + return true; } *fp = filterGetCompFunc(type, optr); @@ -492,6 +504,7 @@ void convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t convertNumberToNumber(*pRightData, pRightOut, typeRight, type); *pRightData = pRightOut; } + return true; } int32_t vectorConvertToVarData(const SScalarParam* pIn, SScalarParam* pOut, int16_t inType, int16_t outType) { @@ -867,11 +880,11 @@ static void vectorMathAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig double *output = (double *)pOutputCol->pData; - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -887,11 +900,11 @@ static void vectorMathBigintAddHelper(SColumnInfoData* pLeftCol, SColumnInfoData int64_t *output = (int64_t *)pOutputCol->pData; - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -982,7 +995,7 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_NULL) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1000,7 +1013,7 @@ void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_NULL){ colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1024,11 +1037,11 @@ static void vectorMathSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRig double *output = (double *)pOutputCol->pData; - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1044,11 +1057,11 @@ static void vectorMathBigintSubHelper(SColumnInfoData* pLeftCol, SColumnInfoData int64_t *output = (int64_t *)pOutputCol->pData; - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1077,7 +1090,7 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_NULL) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1095,7 +1108,7 @@ void vectorMathSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_NULL) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1119,11 +1132,11 @@ static void vectorMathMultiplyHelper(SColumnInfoData* pLeftCol, SColumnInfoData* double *output = (double *)pOutputCol->pData; - if (colDataIsNull_f(pRightCol->nullbitmap, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1149,7 +1162,7 @@ void vectorMathMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam double *output = (double *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_NULL) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1182,7 +1195,7 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p double *output = (double *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { //divide by 0 check + if (IS_NULL || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { //divide by 0 check colDataAppendNULL(pOutputCol, i); continue; } @@ -1190,11 +1203,11 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p / getVectorDoubleValueFnRight(RIGHT_COL, i); } } else if (pLeft->numOfRows == 1) { - if (colDataIsNull_s(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pRightCol, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { // divide by 0 check + if (IS_HELPER_NULL(pRightCol, i) || (getVectorDoubleValueFnRight(RIGHT_COL, i) == 0)) { // divide by 0 check colDataAppendNULL(pOutputCol, i); continue; } @@ -1203,11 +1216,11 @@ void vectorMathDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *p } } } else if (pRight->numOfRows == 1) { - if (colDataIsNull_s(pRightCol, 0) || (getVectorDoubleValueFnRight(RIGHT_COL, 0) == 0)) { // Set pLeft->numOfRows NULL value (divde by 0 check) + if (IS_HELPER_NULL(pRightCol, 0) || (getVectorDoubleValueFnRight(RIGHT_COL, 0) == 0)) { // Set pLeft->numOfRows NULL value (divde by 0 check) colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; } @@ -1236,18 +1249,17 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRightCol->info.type); double *output = (double *)pOutputCol->pData; - double zero = 0.0; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i) || colDataIsNull_s(pRightCol, i)) { + if (IS_NULL) { colDataAppendNULL(pOutputCol, i); continue; } double lx = getVectorDoubleValueFnLeft(LEFT_COL, i); double rx = getVectorDoubleValueFnRight(RIGHT_COL, i); - if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx)) { + if (isnan(lx) || isinf(lx) || isnan(rx) || isinf(rx) || FLT_EQUAL(rx, 0)) { colDataAppendNULL(pOutputCol, i); continue; } @@ -1256,11 +1268,11 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } } else if (pLeft->numOfRows == 1) { double lx = getVectorDoubleValueFnLeft(LEFT_COL, 0); - if (colDataIsNull_s(pLeftCol, 0) || isnan(lx) || isinf(lx)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pLeftCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pRight->numOfRows); } else { for (; i >= 0 && i < pRight->numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pRightCol, i)) { + if (IS_HELPER_NULL(pRightCol, i)) { colDataAppendNULL(pOutputCol, i); continue; } @@ -1276,11 +1288,11 @@ void vectorMathRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam } } else if (pRight->numOfRows == 1) { double rx = getVectorDoubleValueFnRight(RIGHT_COL, 0); - if (colDataIsNull_s(pRightCol, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0) || FLT_EQUAL(rx, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, pLeft->numOfRows); } else { for (; i >= 0 && i < pLeft->numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; } @@ -1315,7 +1327,7 @@ void vectorMathMinus(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO double *output = (double *)pOutputCol->pData; for (; i < pLeft->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; } @@ -1331,7 +1343,7 @@ void vectorAssign(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, pOut->numOfRows = pLeft->numOfRows; - if (colDataIsNull_s(pRight->columnData, 0)) { + if (IS_HELPER_NULL(pRight->columnData, 0)) { for (int32_t i = 0; i < pOut->numOfRows; ++i) { colDataAppend(pOutputCol, i, NULL, true); } @@ -1397,13 +1409,13 @@ static void vectorBitAndHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRigh _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeftCol->info.type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRightCol->info.type); - double *output = (double *)pOutputCol->pData; + int64_t *output = (int64_t *)pOutputCol->pData; - if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { for (; i >= 0 && i < numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1429,7 +1441,7 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int64_t *output = (int64_t *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_NULL) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1451,12 +1463,12 @@ static void vectorBitOrHelper(SColumnInfoData* pLeftCol, SColumnInfoData* pRight int64_t *output = (int64_t *)pOutputCol->pData; - if (colDataIsNull_s(pRightCol, 0)) { // Set pLeft->numOfRows NULL value + if (IS_HELPER_NULL(pRightCol, 0)) { // Set pLeft->numOfRows NULL value colDataAppendNNULL(pOutputCol, 0, numOfRows); } else { int64_t rx = getVectorBigintValueFnRight(RIGHT_COL, 0); for (; i >= 0 && i < numOfRows; i += step, output += 1) { - if (colDataIsNull_s(pLeftCol, i)) { + if (IS_HELPER_NULL(pLeftCol, i)) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1482,7 +1494,7 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int64_t *output = (int64_t *)pOutputCol->pData; if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step, output += 1) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_NULL) { colDataAppendNULL(pOutputCol, i); continue; // TODO set null or ignore } @@ -1507,7 +1519,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * if (pRight->pHashFilter != NULL) { for (; i >= 0 && i < pLeft->numOfRows; i += step) { - if (colDataIsNull_s(pLeft->columnData, i)) { + if (IS_HELPER_NULL(pLeft->columnData, i)) { bool res = false; colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); continue; @@ -1522,7 +1534,7 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * if (pLeft->numOfRows == pRight->numOfRows) { for (; i < pRight->numOfRows && i >= 0; i += step) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, i)) { + if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, i)) { bool res = false; colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); continue; // TODO set null or ignore @@ -1534,18 +1546,21 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * int64_t leftOut = 0; int64_t rightOut = 0; bool isJsonnull = false; - convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); + bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); if(isJsonnull){ - colDataAppendNULL(pOut->columnData, i); - continue; // TODO set null or ignore + ASSERT(0); + } + if(!result){ + colDataAppendInt8(pOut->columnData, i, (int8_t*)&result); + }else{ + bool res = filterDoCompare(fp, optr, pLeftData, pRightData); + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } - bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } } else if (pRight->numOfRows == 1) { ASSERT(pLeft->pHashFilter == NULL); for (; i >= 0 && i < pLeft->numOfRows; i += step) { - if (colDataIsNull_s(pLeft->columnData, i) || colDataIsNull_s(pRight->columnData, 0)) { + if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, 0)) { bool res = false; colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); continue; @@ -1556,17 +1571,20 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * int64_t leftOut = 0; int64_t rightOut = 0; bool isJsonnull = false; - convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); + bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); if(isJsonnull){ - colDataAppendNULL(pOut->columnData, i); - continue; // TODO set null or ignore + ASSERT(0); + } + if(!result){ + colDataAppendInt8(pOut->columnData, i, (int8_t*)&result); + }else{ + bool res = filterDoCompare(fp, optr, pLeftData, pRightData); + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } - bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } } else if (pLeft->numOfRows == 1) { for (; i >= 0 && i < pRight->numOfRows; i += step) { - if (colDataIsNull_s(pRight->columnData, i) || colDataIsNull_s(pLeft->columnData, 0)) { + if (IS_HELPER_NULL(pRight->columnData, i) || IS_HELPER_NULL(pLeft->columnData, 0)) { bool res = false; colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); continue; @@ -1577,13 +1595,16 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * int64_t leftOut = 0; int64_t rightOut = 0; bool isJsonnull = false; - convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); + bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); if(isJsonnull){ - colDataAppendNULL(pOut->columnData, i); - continue; // TODO set null or ignore + ASSERT(0); + } + if(!result){ + colDataAppendInt8(pOut->columnData, i, (int8_t*)&result); + }else{ + bool res = filterDoCompare(fp, optr, pLeftData, pRightData); + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } - bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); } } } @@ -1668,7 +1689,7 @@ void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) { - int8_t v = colDataIsNull_s(pLeft->columnData, i)? 1:0; + int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 1 : 0; colDataAppendInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; @@ -1676,7 +1697,7 @@ void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) { - int8_t v = colDataIsNull_s(pLeft->columnData, i)? 0:1; + int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 0 : 1; colDataAppendInt8(pOut->columnData, i, &v); } pOut->numOfRows = pLeft->numOfRows; @@ -1696,7 +1717,7 @@ _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { return vectorMathMultiply; case OP_TYPE_DIV: return vectorMathDivide; - case OP_TYPE_MOD: + case OP_TYPE_REM: return vectorMathRemainder; case OP_TYPE_MINUS: return vectorMathMinus; diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 8a29462a2bbde2b1d4bf57d5f9ed50c983aed6e6..284ce4b95c8ec279da19860c0380400e3bae99bf 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -42,6 +42,7 @@ #include "nodes.h" #include "tlog.h" #include "parUtil.h" +#include "filterInt.h" #define _DEBUG_PRINT_ 0 @@ -212,6 +213,24 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in *pNode = (SNode *)rnode; } +void scltMakeOpNode2(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight, bool isReverse) { + SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_OPERATOR); + SOperatorNode *onode = (SOperatorNode *)node; + onode->node.resType.type = resType; + onode->node.resType.bytes = tDataTypes[resType].bytes; + + onode->opType = opType; + if(isReverse){ + onode->pLeft = pRight; + onode->pRight = pLeft; + }else{ + onode->pLeft = pLeft; + onode->pRight = pRight; + } + + *pNode = (SNode *)onode; +} + void scltMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode *pLeft, SNode *pRight) { SNode *node = (SNode*)nodesMakeNode(QUERY_NODE_OPERATOR); SOperatorNode *onode = (SOperatorNode *)node; @@ -1039,10 +1058,10 @@ void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){ scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight); } -void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData){ +void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32_t rightType, void *rightData, bool isReverse){ int32_t resType = TSDB_DATA_TYPE_NULL; if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || - opType == OP_TYPE_DIV || opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){ + opType == OP_TYPE_DIV || opType == OP_TYPE_REM || opType == OP_TYPE_MINUS){ resType = TSDB_DATA_TYPE_DOUBLE; }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){ resType = TSDB_DATA_TYPE_BIGINT; @@ -1057,7 +1076,7 @@ void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32 SNode *right = NULL; scltMakeValueNode(&right, rightType, rightData); - scltMakeOpNode(opNode, opType, resType, *opNode, right); + scltMakeOpNode2(opNode, opType, resType, *opNode, right, isReverse); SColumnInfo colInfo = createColumnInfo(1, resType, tDataTypes[resType].bytes); int16_t dataBlockId = 0, slotId = 0; @@ -1065,7 +1084,7 @@ void makeOperator(SNode **opNode, SArray *blockList, EOperatorType opType, int32 scltMakeTargetNode(opNode, dataBlockId, slotId, *opNode); } -void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, double exceptValue, EOperatorType opType){ +void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, double exceptValue, EOperatorType opType, bool isReverse){ SArray *blockList = taosArrayInit(2, POINTER_BYTES); SSDataBlock *src = NULL; SNode *opNode = NULL; @@ -1073,7 +1092,7 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do makeJsonArrow(&src, &opNode, json, (char*)key); taosArrayPush(blockList, &src); - makeOperator(&opNode, blockList, opType, rightType, rightData); + makeOperator(&opNode, blockList, opType, rightType, rightData, isReverse); int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); @@ -1087,17 +1106,17 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do printf("result:NULL\n"); }else if(opType == OP_TYPE_ADD || opType == OP_TYPE_SUB || opType == OP_TYPE_MULTI || opType == OP_TYPE_DIV || - opType == OP_TYPE_MOD || opType == OP_TYPE_MINUS){ - printf("1result:%f,except:%f\n", *((double *)colDataGetData(column, 0)), exceptValue); + opType == OP_TYPE_REM || opType == OP_TYPE_MINUS){ + printf("op:%s,1result:%f,except:%f\n", gOptrStr[opType].str, *((double *)colDataGetData(column, 0)), exceptValue); ASSERT_TRUE(fabs(*((double *)colDataGetData(column, 0)) - exceptValue) < 0.0001); }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){ - printf("2result:%ld,except:%f\n", *((int64_t *)colDataGetData(column, 0)), exceptValue); + printf("op:%s,2result:%ld,except:%f\n", gOptrStr[opType].str, *((int64_t *)colDataGetData(column, 0)), exceptValue); ASSERT_EQ(*((int64_t *)colDataGetData(column, 0)), exceptValue); }else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || opType == OP_TYPE_LOWER_THAN || opType == OP_TYPE_LOWER_EQUAL || opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL || opType == OP_TYPE_IS_NULL || opType == OP_TYPE_IS_NOT_NULL || opType == OP_TYPE_IS_TRUE || opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH || opType == OP_TYPE_NMATCH){ - printf("3result:%d,except:%f\n", *((bool *)colDataGetData(column, 0)), exceptValue); + printf("op:%s,3result:%d,except:%f\n", gOptrStr[opType].str, *((bool *)colDataGetData(column, 0)), exceptValue); ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue); } @@ -1107,7 +1126,7 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do TEST(columnTest, json_column_arith_op) { scltInitLogFile(); - char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}"; + char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":-10,\"k7\":-9.8,\"k8\":false,\"k9\":\"8hel\"}"; char rightv[256] = {0}; memcpy(rightv, rightvTmp, strlen(rightvTmp)); @@ -1117,54 +1136,126 @@ TEST(columnTest, json_column_arith_op) { const int32_t len = 8; EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV, - OP_TYPE_MOD, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR}; + OP_TYPE_REM, OP_TYPE_MINUS, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR}; int32_t input[len] = {1, 8, 2, 2, 3, 0, -4, 9}; - printf("--------------------json int---------------------\n"); + printf("--------------------json int-4 op {1, 8, 2, 2, 3, 0, -4, 9}--------------------\n"); char *key = "k1"; - double eRes[len] = {5.0, -4, 8.0, 2.0, 1.0, -4, 4&-4, 4|9}; + double eRes00[len] = {5.0, -4, 8.0, 2.0, 1.0, -4, 4&-4, 4|9}; + double eRes01[len] = {5.0, 4, 8.0, 0.5, 3, 0, 4&-4, 4|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes00[i], op[i], false); + } for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes01[i], op[i], true); } - printf("--------------------json string---------------------\n"); + printf("--------------------json string- 0 op {1, 8, 2, 2, 3, 0, -4, 9}--------------------\n"); key = "k2"; - double eRes1[len] = {1.0, -8, 0, 0, 0, 0, 0, 9}; + double eRes10[len] = {1.0, -8, 0, 0, 0, 0, 0, 9}; + double eRes11[len] = {1.0, 8, 0, DBL_MAX, DBL_MAX, 0, 0, 9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i], false); + } for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes11[i], op[i], true); } - printf("---------------------json null--------------------\n"); + printf("---------------------json null- null op {1, 8, 2, 2, 3, 0, -4, 9}-------------------\n"); key = "k3"; - double eRes2[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + double eRes20[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + double eRes21[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, 0, DBL_MAX, DBL_MAX}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes20[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes21[i], op[i], true); } - printf("---------------------json bool--------------------\n"); + printf("---------------------json bool- true op {1, 8, 2, 2, 3, 0, -4, 9}-------------------\n"); key = "k4"; - double eRes3[len] = {2.0, -7, 2, 0.5, 1, -1, 1&-4, 1|9}; + double eRes30[len] = {2.0, -7, 2, 0.5, 1, -1, 1&-4, 1|9}; + double eRes31[len] = {2.0, 7, 2, 2, 0, 0, 1&-4, 1|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes30[i], op[i], false); + } for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes31[i], op[i], true); } - printf("----------------------json double-------------------\n"); + printf("----------------------json double-- 5.44 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); key = "k5"; - double eRes4[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9}; + double eRes40[len] = {6.44, -2.56, 10.88, 2.72, 2.44, -5.44, 5&-4, 5|9}; + double eRes41[len] = {6.44, 2.56, 10.88, 0.3676470588235294, 3, 0, 5&-4, 5|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes40[i], op[i], false); + } for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes41[i], op[i], true); } - printf("---------------------json not exist--------------------\n"); + printf("----------------------json int-- -10 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); + + key = "k6"; + double eRes50[len] = {-9, -18, -20, -5, -10%3, 10, -10&-4, -10|9}; + double eRes51[len] = {-9, 18, -20, -0.2, 3%-10, 0, -10&-4, -10|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes50[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes51[i], op[i], true); + } + + printf("----------------------json double-- -9.8 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); + + key = "k7"; + double eRes60[len] = {-8.8, -17.8, -19.6, -4.9, -0.8, 9.8, -9&-4, -9|9}; + double eRes61[len] = {-8.8, 17.8, -19.6, -0.2040816326530612, 3, 0, -9&-4, -9|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes60[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes61[i], op[i], true); + } + + printf("----------------------json bool-- 0 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); + + key = "k8"; + double eRes70[len] = {1.0, -8, 0, 0, 0, 0, 0, 9}; + double eRes71[len] = {1.0, 8, 0, DBL_MAX, DBL_MAX, 0, 0, 9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes70[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes71[i], op[i], true); + } + + printf("----------------------json string-- 8 op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); + + key = "k9"; + double eRes80[len] = {9, 0, 16, 4, 8%3, -8, 8&-4, 8|9}; + double eRes81[len] = {9, 0, 16, 0.25, 3%8, 0, 8&-4, 8|9}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes80[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes81[i], op[i], true); + } + + printf("---------------------json not exist-- NULL op {1, 8, 2, 2, 3, 0, -4, 9}------------------\n"); key = "k10"; - double eRes5[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + double eRes90[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX}; + double eRes91[len] = {DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, DBL_MAX, 0, DBL_MAX, DBL_MAX}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes90[i], op[i], false); + } + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes91[i], op[i], true); } taosArrayDestroy(tags); @@ -1183,7 +1274,7 @@ void *prepareNchar(char* rightData){ TEST(columnTest, json_column_logic_op) { scltInitLogFile(); - char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}"; + char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":-10,\"k7\":-9.8,\"k8\":false,\"k9\":\"6.6hello\"}"; char rightv[256] = {0}; memcpy(rightv, rightvTmp, strlen(rightvTmp)); @@ -1191,6 +1282,7 @@ TEST(columnTest, json_column_logic_op) { STag* row = NULL; parseJsontoTagData(rightv, tags, &row, NULL); + const int32_t len0 = 6; const int32_t len = 9; const int32_t len1 = 4; EOperatorType op[len+len1] = {OP_TYPE_GREATER_THAN, OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, OP_TYPE_EQUAL, OP_TYPE_NOT_EQUAL, @@ -1199,93 +1291,183 @@ TEST(columnTest, json_column_logic_op) { int32_t input[len] = {1, 8, 2, 2, 3, 0, 0, 0, 0}; char *inputNchar[len1] = {"hell_", "hel%", "hell", "llll"}; - printf("--------------------json int---------------------\n"); + printf("--------------------json int---4 {1, 8, 2, 2, 3, 0, 0, 0, 0}------------------\n"); char *key = "k1"; bool eRes[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes[i], op[i], false); + } + bool eRes_0[len0] = {false, true, true, true, false, true}; + for(int i = 0; i < len0; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_0[i], op[i], true); } for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes[i], op[i], false); taosMemoryFree(rightData); } - printf("--------------------json string---------------------\n"); + printf("--------------------json string--0 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k2"; bool eRes1[len+len1] = {false, false, true, true, false, false, false, true, false, true, false, true, true}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i], false); + } + bool eRes_1[len0] = {true, true, false, false, false, false}; + for(int i = 0; i < len0; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_1[i], op[i], true); } + for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes1[i], op[i], false); taosMemoryFree(rightData); } - printf("--------------------json null---------------------\n"); + printf("--------------------json null---null {1, 8, 2, 2, 3, 0, 0, 0, 0}------------------\n"); key = "k3"; // (null is true) return NULL, so use DBL_MAX represent NULL double eRes2[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes2[i], op[i], false); + } + bool eRes_2[len0] = {false, false, false, false, false, false}; + for(int i = 0; i < len0; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_2[i], op[i], true); } + for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes2[i], op[i], false); taosMemoryFree(rightData); } - printf("--------------------json bool---------------------\n"); + printf("--------------------json bool--1 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k4"; bool eRes3[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i], false); } + bool eRes_3[len0] = {false, true, false, false, false, true}; + for(int i = 0; i < len0; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_3[i], op[i], true); + } + for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes3[i], op[i], false); taosMemoryFree(rightData); } - printf("--------------------json double---------------------\n"); + printf("--------------------json double--5.44 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k5"; bool eRes4[len+len1] = {true, false, false, false, false, true, false, true, true, false, false, false, false}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes4[i], op[i], false); + } + bool eRes_4[len0] = {false, true, true, true, false, true}; + for(int i = 0; i < len0; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_4[i], op[i], true); } + for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes4[i], op[i], false); taosMemoryFree(rightData); } - printf("--------------------json double---------------------\n"); + printf("--------------------json int-- -10 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k6"; - bool eRes5[len+len1] = {true, false, false, false, false, true, false, true, true, false, true, false, true}; + bool eRes5[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i], false); + } + bool eRes_5[len0] = {true, true, false, false, false, true}; + for(int i = 0; i < len0; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_5[i], op[i], true); + } + + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i], false); + taosMemoryFree(rightData); + } + + printf("--------------------json double-- -9.8 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); + + key = "k7"; + bool eRes6[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes6[i], op[i], false); + } + bool eRes_6[len0] = {true, true, false, false, false, true}; + for(int i = 0; i < len0; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_6[i], op[i], true); + } + + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes6[i], op[i], false); + taosMemoryFree(rightData); + } + + + printf("--------------------json bool-- 0 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); + + key = "k8"; + bool eRes7[len+len1] = {false, false, true, true, false, false, false, true, false, false, false, false, false}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes7[i], op[i], false); + } + bool eRes_7[len0] = {true, true, false, false, false, false}; + for(int i = 0; i < len0; i++) { + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_7[i], op[i], true); } + for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes5[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes7[i], op[i], false); taosMemoryFree(rightData); } - printf("---------------------json not exist--------------------\n"); + + printf("--------------------json string-- 6.6hello {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); + + key = "k9"; + bool eRes8[len+len1] = {true, false, false, false, false, true, false, true, true, false, true, false, true}; + for(int i = 0; i < len; i++){ + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes8[i], op[i], false); + } + bool eRes_8[len0] = {false, true, true, true, false, true}; + for(int i = 0; i < len0; i++) { + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_8[i], op[i], true); + } + + for(int i = len; i < len + len1; i++){ + void* rightData = prepareNchar(inputNchar[i-len]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes8[i], op[i], false); + taosMemoryFree(rightData); + } + + printf("---------------------json not exist-- NULL {1, 8, 2, 2, 3, 0, 0, 0, 0}------------------\n"); key = "k10"; // (NULL is true) return NULL, so use DBL_MAX represent NULL - double eRes10[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false}; + double eRes9[len+len1] = {false, false, false, false, false, false, true, false, DBL_MAX, false, false, false, false}; for(int i = 0; i < len; i++){ - makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes10[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes9[i], op[i], false); } + bool eRes_9[len0] = {false, false, false, false, false, false}; + for(int i = 0; i < len0; i++) { + makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_9[i], op[i], true); + } + for(int i = len; i < len + len1; i++){ void* rightData = prepareNchar(inputNchar[i-len]); - makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes10[i], op[i]); + makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes9[i], op[i], false); taosMemoryFree(rightData); } diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 44b3e6d396b8750a78ba9e414243f45686a121ec..d961780ae515c7c1aefbe4879957ce7d7607b3fa 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -32,6 +32,10 @@ extern "C" { #define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000 #define SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM 200 // unit is TSDB_TABLE_NUM_UNIT +#define SCH_DEFAULT_TASK_TIMEOUT_USEC 10000000 +#define SCH_MAX_TASK_TIMEOUT_USEC 60000000 + +#define SCH_TASK_MAX_EXEC_TIMES 5 #define SCH_MAX_CANDIDATE_EP_NUM TSDB_MAX_REPLICA enum { @@ -51,6 +55,7 @@ typedef struct SSchTrans { typedef struct SSchHbTrans { SRWLatch lock; + int64_t taskNum; SRpcCtx rpcCtx; SSchTrans trans; } SSchHbTrans; @@ -97,11 +102,11 @@ typedef struct SSchedulerMgmt { uint64_t taskId; // sequential taksId uint64_t sId; // schedulerId SSchedulerCfg cfg; - SRWLatch lock; bool exit; int32_t jobRef; int32_t jobNum; SSchStat stat; + SRWLatch hbLock; SHashObj *hbConnections; } SSchedulerMgmt; @@ -114,7 +119,8 @@ typedef struct SSchTaskCallbackParam { uint64_t queryId; int64_t refId; uint64_t taskId; - void *transport; + int32_t execIdx; + void *pTrans; } SSchTaskCallbackParam; typedef struct SSchHbCallbackParam { @@ -148,25 +154,36 @@ typedef struct SSchLevel { SArray *subTasks; // Element is SQueryTask } SSchLevel; +typedef struct SSchTaskProfile { + int64_t startTs; + int64_t execUseTime[SCH_TASK_MAX_EXEC_TIMES]; + int64_t waitTime; + int64_t endTs; +} SSchTaskProfile; + typedef struct SSchTask { uint64_t taskId; // task id SRWLatch lock; // task lock + int32_t maxExecTimes; // task may exec times + int32_t execIdx; // task current execute try index SSchLevel *level; // level + SRWLatch planLock; // task update plan lock SSubplan *plan; // subplan char *msg; // operator tree int32_t msgLen; // msg length int8_t status; // task status int32_t lastMsgType; // last sent msg type - int32_t tryTimes; // task already tried times + int64_t timeoutUsec; // taks timeout useconds before reschedule SQueryNodeAddr succeedAddr; // task executed success node address int8_t candidateIdx; // current try condidation index SArray *candidateAddrs; // condidate node addresses, element is SQueryNodeAddr - SArray *execNodes; // all tried node for current task, element is SSchNodeInfo - SQueryProfileSummary summary; // task execution summary + SHashObj *execNodes; // all tried node for current task, element is SSchNodeInfo + SSchTaskProfile profile; // task execution profile int32_t childReady; // child task ready number SArray *children; // the datasource tasks,from which to fetch the result, element is SQueryTask* SArray *parents; // the data destination tasks, get data from current task, element is SQueryTask* - void* handle; // task send handle + void* handle; // task send handle + bool registerdHb; // registered in hb } SSchTask; typedef struct SSchJobAttr { @@ -215,8 +232,39 @@ typedef struct SSchJob { extern SSchedulerMgmt schMgmt; +#define SCH_LOG_TASK_START_TS(_task) \ + do { \ + int64_t us = taosGetTimestampUs(); \ + int32_t idx = (_task)->execIdx % SCH_TASK_MAX_EXEC_TIMES; \ + (_task)->profile.execUseTime[idx] = us; \ + if (0 == (_task)->execIdx) { \ + (_task)->profile.startTs = us; \ + } \ + } while (0) + +#define SCH_LOG_TASK_WAIT_TS(_task) \ + do { \ + int64_t us = taosGetTimestampUs(); \ + int32_t idx = (_task)->execIdx % SCH_TASK_MAX_EXEC_TIMES; \ + (_task)->profile.waitTime += us - (_task)->profile.execUseTime[idx]; \ + } while (0) + + +#define SCH_LOG_TASK_END_TS(_task) \ + do { \ + int64_t us = taosGetTimestampUs(); \ + int32_t idx = (_task)->execIdx % SCH_TASK_MAX_EXEC_TIMES; \ + (_task)->profile.execUseTime[idx] = us - (_task)->profile.execUseTime[idx]; \ + (_task)->profile.endTs = us; \ + } while (0) + +#define SCH_TASK_TIMEOUT(_task) ((taosGetTimestampUs() - (_task)->profile.execUseTime[(_task)->execIdx % SCH_TASK_MAX_EXEC_TIMES]) > (_task)->timeoutUsec) + #define SCH_TASK_READY_FOR_LAUNCH(readyNum, task) ((readyNum) >= taosArrayGetSize((task)->children)) +#define SCH_LOCK_TASK(_task) SCH_LOCK(SCH_WRITE, &(_task)->lock) +#define SCH_UNLOCK_TASK(_task) SCH_UNLOCK(SCH_WRITE, &(_task)->lock) + #define SCH_TASK_ID(_task) ((_task) ? (_task)->taskId : -1) #define SCH_SET_TASK_LASTMSG_TYPE(_task, _type) do { if(_task) { atomic_store_32(&(_task)->lastMsgType, _type); } } while (0) #define SCH_GET_TASK_LASTMSG_TYPE(_task) ((_task) ? atomic_load_32(&(_task)->lastMsgType) : -1) @@ -272,6 +320,8 @@ extern SSchedulerMgmt schMgmt; #define SCH_UNLOCK(type, _lock) (SCH_READ == (type) ? taosRUnLockLatch(_lock) : taosWUnLockLatch(_lock)) +void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask); +void schCleanClusterHb(void* pTrans); int32_t schLaunchTask(SSchJob *job, SSchTask *task); int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); SSchJob *schAcquireJob(int64_t refId); @@ -284,7 +334,7 @@ int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask); int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask); int32_t schFetchFromRemote(SSchJob *pJob); int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode); -int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId); +int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray* taskAction); int32_t schCloneSMsgSendInfo(void *src, void **dst); int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob); void schFreeJobImpl(void *job); @@ -301,10 +351,9 @@ int32_t schSaveJobQueryRes(SSchJob *pJob, SQueryTableRsp *rsp); int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp); void schProcessOnDataFetched(SSchJob *job); int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask); -int32_t schUpdateTaskExecNodeHandle(SSchTask *pTask, void *handle, int32_t rspCode); void schFreeRpcCtxVal(const void *arg); int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb); -int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, void *handle); +int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t execIdx); int32_t schExecStaticExplainJob(void *pTrans, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, SSchResInfo *pRes, bool sync); int32_t schExecJobImpl(void *pTrans, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, @@ -318,7 +367,8 @@ int32_t schExecJob(void *pTrans, SArray *pNodeList, SQueryPlan *pDag, int64_t *p int32_t schAsyncExecJob(void *pTrans, SArray *pNodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, SSchResInfo *pRes); int32_t schFetchRows(SSchJob *pJob); int32_t schAsyncFetchRows(SSchJob *pJob); -int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, int32_t msgType, void *handle, int32_t rspCode); +int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execIdx); +int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId* pEpId, SArray* pStatusList); #ifdef __cplusplus diff --git a/source/libs/scheduler/src/schDbg.c b/source/libs/scheduler/src/schDbg.c index 4b5f74114d2ae7d4ec47b09f8a48da2f3f61de8d..5c0c6fbb76974f87e06b5ba1cbcb761b864fd163 100644 --- a/source/libs/scheduler/src/schDbg.c +++ b/source/libs/scheduler/src/schDbg.c @@ -30,13 +30,4 @@ void schdExecCallback(SQueryResult* pResult, void* param, int32_t code) { tsem_post(&schdRspSem); } -void schdFetchCallback(void* pResult, void* param, int32_t code) { - SSchdFetchParam* fParam = (SSchdFetchParam*)param; - - *fParam->pData = pResult; - *fParam->code = code; - - tsem_post(&schdRspSem); -} - diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index dbad053c65ba3572a6d4740e4b991f3760e9631b..ca90d2fe34a74a61652d5fe61054b34b5b4412b2 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -21,18 +21,21 @@ #include "tref.h" #include "trpc.h" -FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); } +FORCE_INLINE SSchJob *schAcquireJob(int64_t refId) { qDebug("acquire jobId:0x%"PRIx64, refId); return (SSchJob *)taosAcquireRef(schMgmt.jobRef, refId); } -FORCE_INLINE int32_t schReleaseJob(int64_t refId) { return taosReleaseRef(schMgmt.jobRef, refId); } +FORCE_INLINE int32_t schReleaseJob(int64_t refId) { qDebug("release jobId:0x%"PRIx64, refId); return taosReleaseRef(schMgmt.jobRef, refId); } int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel) { pTask->plan = pPlan; pTask->level = pLevel; + pTask->execIdx = -1; + pTask->maxExecTimes = SCH_TASK_MAX_EXEC_TIMES; + pTask->timeoutUsec = SCH_DEFAULT_TASK_TIMEOUT_USEC; SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_NOT_START); pTask->taskId = schGenTaskId(); - pTask->execNodes = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SSchNodeInfo)); + pTask->execNodes = taosHashInit(SCH_MAX_CANDIDATE_EP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); if (NULL == pTask->execNodes) { - SCH_TASK_ELOG("taosArrayInit %d execNodes failed", SCH_MAX_CANDIDATE_EP_NUM); + SCH_TASK_ELOG("taosHashInit %d execNodes failed", SCH_MAX_CANDIDATE_EP_NUM); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -123,7 +126,10 @@ _return: SCH_RET(code); } -void schFreeTask(SSchTask *pTask) { + +void schFreeTask(SSchJob *pJob, SSchTask *pTask) { + schDeregisterTaskHb(pJob, pTask); + if (pTask->candidateAddrs) { taosArrayDestroy(pTask->candidateAddrs); } @@ -139,7 +145,7 @@ void schFreeTask(SSchTask *pTask) { } if (pTask->execNodes) { - taosArrayDestroy(pTask->execNodes); + taosHashCleanup(pTask->execNodes); } } @@ -329,45 +335,60 @@ int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } -int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, void *handle) { - SSchNodeInfo nodeInfo = {.addr = *addr, .handle = handle}; +int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t execIdx) { + SSchNodeInfo nodeInfo = {.addr = *addr, .handle = NULL}; - if (NULL == taosArrayPush(pTask->execNodes, &nodeInfo)) { - SCH_TASK_ELOG("taosArrayPush nodeInfo to execNodes list failed, errno:%d", errno); + if (taosHashPut(pTask->execNodes, &execIdx, sizeof(execIdx), &nodeInfo, sizeof(nodeInfo))) { + SCH_TASK_ELOG("taosHashPut nodeInfo to execNodes failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("task execNode recorded, handle:%p", handle); + SCH_TASK_DLOG("task execNode added, execIdx:%d", execIdx); return TSDB_CODE_SUCCESS; } -int32_t schDropTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle) { +int32_t schDropTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int32_t execIdx) { if (NULL == pTask->execNodes) { return TSDB_CODE_SUCCESS; } - int32_t num = taosArrayGetSize(pTask->execNodes); - for (int32_t i = 0; i < num; ++i) { - SSchNodeInfo* pNode = taosArrayGet(pTask->execNodes, i); - if (pNode->handle == handle) { - taosArrayRemove(pTask->execNodes, i); - break; - } + if (taosHashRemove(pTask->execNodes, &execIdx, sizeof(execIdx))) { + SCH_TASK_ELOG("fail to remove execIdx %d from execNodeList", execIdx); + } else { + SCH_TASK_DLOG("execIdx %d removed from execNodeList", execIdx); + } + + if (execIdx != pTask->execIdx) { // ignore it + SCH_TASK_DLOG("execIdx %d is not current execIdx %d", execIdx, pTask->execIdx); + SCH_RET(TSDB_CODE_SCH_IGNORE_ERROR); } return TSDB_CODE_SUCCESS; } -int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, int32_t msgType, void *handle, int32_t rspCode) { - SCH_SET_TASK_HANDLE(pTask, handle); +int32_t schUpdateTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int32_t execIdx) { + if (taosHashGetSize(pTask->execNodes) <= 0) { + return TSDB_CODE_SUCCESS; + } + + SSchNodeInfo *nodeInfo = taosHashGet(pTask->execNodes, &execIdx, sizeof(execIdx)); + nodeInfo->handle = handle; - schUpdateTaskExecNodeHandle(pTask, handle, rspCode); + SCH_TASK_DLOG("handle updated to %p for execIdx %d", handle, execIdx); - if (msgType == TDMT_SCH_LINK_BROKEN) { - schDropTaskExecNode(pJob, pTask, handle); + return TSDB_CODE_SUCCESS; +} + +int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execIdx) { + if (dropExecNode) { + SCH_RET(schDropTaskExecNode(pJob, pTask, handle, execIdx)); } + SCH_SET_TASK_HANDLE(pTask, handle); + + schUpdateTaskExecNode(pJob, pTask, handle, execIdx); + return TSDB_CODE_SUCCESS; } @@ -514,6 +535,8 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + SCH_TASK_DLOG("set %dth condidate addr, id %d, fqdn:%s, port:%d", i, naddr->nodeId, SCH_GET_CUR_EP(naddr)->fqdn, SCH_GET_CUR_EP(naddr)->port); + ++addNum; } } @@ -672,7 +695,6 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) { int8_t status = 0; - ++pTask->tryTimes; if (schJobNeedToStop(pJob, &status)) { *needRetry = false; @@ -680,9 +702,19 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo return TSDB_CODE_SUCCESS; } - if (pTask->tryTimes >= REQUEST_MAX_TRY_TIMES) { + if (TSDB_CODE_SCH_TIMEOUT_ERROR == errCode) { + pTask->maxExecTimes++; + if (pTask->timeoutUsec < SCH_MAX_TASK_TIMEOUT_USEC) { + pTask->timeoutUsec *= 2; + if (pTask->timeoutUsec > SCH_MAX_TASK_TIMEOUT_USEC) { + pTask->timeoutUsec = SCH_MAX_TASK_TIMEOUT_USEC; + } + } + } + + if ((pTask->execIdx + 1) >= pTask->maxExecTimes) { *needRetry = false; - SCH_TASK_DLOG("task no more retry since reach max try times, tryTimes:%d", pTask->tryTimes); + SCH_TASK_DLOG("task no more retry since reach max try times, execIdx:%d", pTask->execIdx); return TSDB_CODE_SUCCESS; } @@ -694,16 +726,16 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo // TODO CHECK epList/condidateList if (SCH_IS_DATA_SRC_TASK(pTask)) { - if (pTask->tryTimes >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) { + if ((pTask->execIdx + 1) >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) { *needRetry = false; - SCH_TASK_DLOG("task no more retry since all ep tried, tryTimes:%d, epNum:%d", pTask->tryTimes, + SCH_TASK_DLOG("task no more retry since all ep tried, execIdx:%d, epNum:%d", pTask->execIdx, SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)); return TSDB_CODE_SUCCESS; } } else { int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs); - if ((pTask->candidateIdx + 1) >= candidateNum) { + if ((pTask->candidateIdx + 1) >= candidateNum && (TSDB_CODE_SCH_TIMEOUT_ERROR != errCode)) { *needRetry = false; SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d", pTask->candidateIdx, candidateNum); @@ -712,7 +744,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo } *needRetry = true; - SCH_TASK_DLOG("task need the %dth retry, errCode:%x - %s", pTask->tryTimes, errCode, tstrerror(errCode)); + SCH_TASK_DLOG("task need the %dth retry, errCode:%x - %s", pTask->execIdx + 1, errCode, tstrerror(errCode)); return TSDB_CODE_SUCCESS; } @@ -728,10 +760,15 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(schLaunchTasksInFlowCtrlList(pJob, pTask)); } + schDeregisterTaskHb(pJob, pTask); + if (SCH_IS_DATA_SRC_TASK(pTask)) { SCH_SWITCH_EPSET(&pTask->plan->execNode); } else { - ++pTask->candidateIdx; + int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs); + if (++pTask->candidateIdx >= candidateNum) { + pTask->candidateIdx = 0; + } } SCH_ERR_RET(schLaunchTask(pJob, pTask)); @@ -906,6 +943,12 @@ void schProcessOnDataFetched(SSchJob *job) { int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) { int8_t status = 0; + if (errCode == TSDB_CODE_SCH_TIMEOUT_ERROR) { + SCH_LOG_TASK_WAIT_TS(pTask); + } else { + SCH_LOG_TASK_END_TS(pTask); + } + if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("task failed not processed cause of job status, job status:%s", jobTaskStatusStr(status)); SCH_RET(atomic_load_32(&pJob->errCode)); @@ -989,6 +1032,8 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_DLOG("taskOnSuccess, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); + SCH_LOG_TASK_END_TS(pTask); + SCH_ERR_JRET(schMoveTaskToSuccList(pJob, pTask, &moved)); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED); @@ -1051,6 +1096,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_UNLOCK(SCH_WRITE, &parent->lock); if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) { + SCH_TASK_DLOG("all %d children task done, start to launch parent task %" PRIx64, readyNum, parent->taskId); SCH_ERR_RET(schLaunchTask(pJob, parent)); } } @@ -1105,6 +1151,93 @@ int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRs return TSDB_CODE_SUCCESS; } +void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { + if (NULL == pTask->execNodes) { + SCH_TASK_DLOG("no exec address, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); + return; + } + + int32_t size = (int32_t)taosHashGetSize(pTask->execNodes); + + if (size <= 0) { + SCH_TASK_DLOG("task has no execNodes, no need to drop it, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); + return; + } + + SSchNodeInfo *nodeInfo = taosHashIterate(pTask->execNodes, NULL); + while (nodeInfo) { + SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle); + + schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_VND_DROP_TASK); + + nodeInfo = taosHashIterate(pTask->execNodes, nodeInfo); + } + + SCH_TASK_DLOG("task has been dropped on %d exec nodes", size); +} + + +int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) { + if (SCH_IS_DATA_SRC_QRY_TASK(pTask)) { + return TSDB_CODE_SUCCESS; + } + + SCH_LOCK_TASK(pTask); + if (JOB_TASK_STATUS_EXECUTING == pTask->status && pJob->fetchTask != pTask && taosArrayGetSize(pTask->candidateAddrs) > 1) { + SCH_TASK_DLOG("task execIdx %d will be rescheduled now", pTask->execIdx); + schDropTaskOnExecNode(pJob, pTask); + taosHashClear(pTask->execNodes); + schProcessOnTaskFailure(pJob, pTask, TSDB_CODE_SCH_TIMEOUT_ERROR); + } + SCH_UNLOCK_TASK(pTask); + + return TSDB_CODE_SUCCESS; +} + +int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId* pEpId, SArray* pStatusList) { + int32_t taskNum = (int32_t)taosArrayGetSize(pStatusList); + SSchTask *pTask = NULL; + + qDebug("%d task status in hb rsp from nodeId:%d, fqdn:%s, port:%d", taskNum, pEpId->nodeId, pEpId->ep.fqdn, pEpId->ep.port); + + for (int32_t i = 0; i < taskNum; ++i) { + STaskStatus *taskStatus = taosArrayGet(pStatusList, i); + + SSchJob *pJob = schAcquireJob(taskStatus->refId); + if (NULL == pJob) { + qWarn("job not found, refId:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64, taskStatus->refId, + taskStatus->queryId, taskStatus->taskId); + // TODO DROP TASK FROM SERVER!!!! + continue; + } + + SCH_JOB_DLOG("TID:0x%" PRIx64 " task status in server: %s", taskStatus->taskId, jobTaskStatusStr(taskStatus->status)); + + pTask = NULL; + schGetTaskInJob(pJob, taskStatus->taskId, &pTask); + if (NULL == pTask) { + // TODO DROP TASK FROM SERVER!!!! + schReleaseJob(taskStatus->refId); + continue; + } + + if (taskStatus->status == JOB_TASK_STATUS_FAILED) { + // RECORD AND HANDLE ERROR!!!! + schReleaseJob(taskStatus->refId); + continue; + } + + if (taskStatus->status == JOB_TASK_STATUS_NOT_START && SCH_TASK_TIMEOUT(pTask)) { + schRescheduleTask(pJob, pTask); + } + + schReleaseJob(taskStatus->refId); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t schSaveJobQueryRes(SSchJob *pJob, SQueryTableRsp *rsp) { if (rsp->tbFName[0]) { if (NULL == pJob->execRes.res) { @@ -1156,24 +1289,16 @@ int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask) { return TSDB_CODE_SUCCESS; } - -int32_t schUpdateTaskExecNodeHandle(SSchTask *pTask, void *handle, int32_t rspCode) { - if (rspCode || NULL == pTask->execNodes || taosArrayGetSize(pTask->execNodes) > 1 || - taosArrayGetSize(pTask->execNodes) <= 0) { - return TSDB_CODE_SUCCESS; - } - - SSchNodeInfo *nodeInfo = taosArrayGet(pTask->execNodes, 0); - nodeInfo->handle = handle; - - return TSDB_CODE_SUCCESS; -} - int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { int8_t status = 0; int32_t code = 0; atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); + pTask->execIdx++; + + SCH_TASK_DLOG("start to launch task's %dth exec", pTask->execIdx); + + SCH_LOG_TASK_START_TS(pTask); if (schJobNeedToStop(pJob, &status)) { SCH_TASK_DLOG("no need to launch task cause of job status, job status:%s", jobTaskStatusStr(status)); @@ -1257,29 +1382,6 @@ int32_t schLaunchJob(SSchJob *pJob) { return TSDB_CODE_SUCCESS; } -void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { - if (NULL == pTask->execNodes) { - SCH_TASK_DLOG("no exec address, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); - return; - } - - int32_t size = (int32_t)taosArrayGetSize(pTask->execNodes); - - if (size <= 0) { - SCH_TASK_DLOG("task has no execNodes, no need to drop it, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); - return; - } - - SSchNodeInfo *nodeInfo = NULL; - for (int32_t i = 0; i < size; ++i) { - nodeInfo = (SSchNodeInfo *)taosArrayGet(pTask->execNodes, i); - SCH_SET_TASK_HANDLE(pTask, nodeInfo->handle); - - schBuildAndSendMsg(pJob, pTask, &nodeInfo->addr, TDMT_VND_DROP_TASK); - } - - SCH_TASK_DLOG("task has %d exec address", size); -} void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { if (!SCH_IS_NEED_DROP_JOB(pJob)) { @@ -1332,7 +1434,7 @@ void schFreeJobImpl(void *job) { int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks); for (int32_t j = 0; j < numOfTasks; ++j) { SSchTask *pTask = taosArrayGet(pLevel->subTasks, j); - schFreeTask(pTask); + schFreeTask(pJob, pTask); } taosArrayDestroy(pLevel->subTasks); @@ -1358,15 +1460,14 @@ void schFreeJobImpl(void *job) { qDebug("QID:0x%" PRIx64 " job freed, refId:%" PRIx64 ", pointer:%p", queryId, refId, pJob); - atomic_sub_fetch_32(&schMgmt.jobNum, 1); - - schCloseJobRef(); + int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); + if (jobNum == 0) { + schCloseJobRef(); + } } int32_t schExecJobImpl(void *pTrans, SArray *pNodeList, SQueryPlan *pDag, int64_t *job, const char *sql, SSchResInfo *pRes, int64_t startTs, bool sync) { - qDebug("QID:0x%" PRIx64 " job started", pDag->queryId); - if (pNodeList == NULL || taosArrayGetSize(pNodeList) <= 0) { qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pDag->queryId); } @@ -1375,6 +1476,7 @@ int32_t schExecJobImpl(void *pTrans, SArray *pNodeList, SQueryPlan *pDag, int64_ SSchJob *pJob = NULL; SCH_ERR_RET(schInitJob(&pJob, pDag, pTrans, pNodeList, sql, pRes, startTs, sync)); + qDebug("QID:0x%" PRIx64 " jobId:0x%"PRIx64 " started", pDag->queryId, pJob->refId); *job = pJob->refId; SCH_ERR_JRET(schLaunchJob(pJob)); @@ -1386,7 +1488,7 @@ int32_t schExecJobImpl(void *pTrans, SArray *pNodeList, SQueryPlan *pDag, int64_ pJob->userCb = SCH_EXEC_CB; } - SCH_JOB_DLOG("job exec done, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); + SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%"PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId); _return: diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 0ba91a1c85649463225bd5f6d82edad4b56714c9..bf51d8d631bfff7501ba40fea396b570e14c5c90 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -92,8 +92,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch int8_t status = 0; if (schJobNeedToStop(pJob, &status)) { - SCH_TASK_ELOG("rsp not processed cause of job status, job status:%s, rspCode:0x%x", jobTaskStatusStr(status), - rspCode); + SCH_TASK_ELOG("rsp not processed cause of job status, job status:%s, rspCode:0x%x", jobTaskStatusStr(status), rspCode); taosMemoryFreeClear(msg); SCH_RET(atomic_load_32(&pJob->errCode)); } @@ -344,7 +343,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch _return: - taosMemoryFreeClear(msg); + taosMemoryFreeClear(msg); SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } @@ -364,13 +363,25 @@ int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, in SCH_ERR_JRET(schGetTaskInJob(pJob, pParam->taskId, &pTask)); + SCH_LOCK_TASK(pTask); + SCH_TASK_DLOG("rsp msg received, type:%s, handle:%p, code:%s", TMSG_INFO(msgType), pMsg->handle, tstrerror(rspCode)); - SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, msgType, pMsg->handle, rspCode)); + if (pParam->execIdx != pTask->execIdx) { + SCH_TASK_DLOG("execIdx %d mis-match current execIdx %d", pParam->execIdx, pTask->execIdx); + goto _return; + } + + bool dropExecNode = (msgType == TDMT_SCH_LINK_BROKEN || rspCode == TSDB_CODE_RPC_NETWORK_UNAVAIL); + SCH_ERR_JRET(schUpdateTaskHandle(pJob, pTask, dropExecNode, pMsg->handle, pParam->execIdx)); SCH_ERR_JRET(schHandleResponseMsg(pJob, pTask, msgType, pMsg->pData, pMsg->len, rspCode)); _return: + + if (pTask) { + SCH_UNLOCK_TASK(pTask); + } if (pJob) { schReleaseJob(pParam->refId); @@ -426,7 +437,7 @@ int32_t schHandleLinkBrokenCallback(void *param, const SDataBuf *pMsg, int32_t c SSchTrans trans = {.pTrans = hbParam->pTrans, .pHandle = NULL}; SCH_ERR_RET(schUpdateHbConnection(&hbParam->nodeEpId, &trans)); - SCH_ERR_RET(schBuildAndSendHbMsg(&hbParam->nodeEpId)); + SCH_ERR_RET(schBuildAndSendHbMsg(&hbParam->nodeEpId, NULL)); } else { SCH_ERR_RET(schHandleCallback(param, pMsg, TDMT_SCH_LINK_BROKEN, code)); } @@ -454,7 +465,8 @@ int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, int32_t msgType, param->queryId = pJob->queryId; param->refId = pJob->refId; param->taskId = SCH_TASK_ID(pTask); - param->transport = pJob->pTrans; + param->pTrans = pJob->pTrans; + param->execIdx = pTask->execIdx; msgSendInfo->param = param; msgSendInfo->fp = fp; @@ -636,32 +648,7 @@ _return: SCH_RET(code); } -int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId, bool *exist) { - int32_t code = 0; - SSchHbTrans hb = {0}; - - hb.trans.pTrans = pJob->pTrans; - - SCH_ERR_RET(schMakeHbRpcCtx(pJob, pTask, &hb.rpcCtx)); - - code = taosHashPut(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId), &hb, sizeof(SSchHbTrans)); - if (code) { - schFreeRpcCtx(&hb.rpcCtx); - - if (HASH_NODE_EXIST(code)) { - *exist = true; - return TSDB_CODE_SUCCESS; - } - - qError("taosHashPut hb trans failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); - SCH_ERR_RET(code); - } - - return TSDB_CODE_SUCCESS; -} - - -int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) { +int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray* taskAction) { SSchedulerHbReq req = {0}; int32_t code = 0; SRpcCtx rpcCtx = {0}; @@ -672,17 +659,20 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) { req.sId = schMgmt.sId; memcpy(&req.epId, nodeEpId, sizeof(SQueryNodeEpId)); + SCH_LOCK(SCH_READ, &schMgmt.hbLock); SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, nodeEpId, sizeof(SQueryNodeEpId)); if (NULL == hb) { - qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", nodeEpId->nodeId, nodeEpId->ep.fqdn, + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + qError("hb connection no longer exist, nodeId:%d, fqdn:%s, port:%d", nodeEpId->nodeId, nodeEpId->ep.fqdn, nodeEpId->ep.port); - SCH_ERR_RET(code); + return TSDB_CODE_SUCCESS; } SCH_LOCK(SCH_WRITE, &hb->lock); code = schCloneHbRpcCtx(&hb->rpcCtx, &rpcCtx); memcpy(&trans, &hb->trans, sizeof(trans)); SCH_UNLOCK(SCH_WRITE, &hb->lock); + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); SCH_ERR_RET(code); @@ -717,7 +707,7 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId) { __async_send_cb_fn_t fp = NULL; SCH_ERR_JRET(schGetCallbackFp(msgType, &fp)); - param->transport = trans.pTrans; + param->pTrans = trans.pTrans; pMsgSendInfo->param = param; pMsgSendInfo->msgInfo.pData = msg; @@ -752,49 +742,6 @@ _return: SCH_RET(code); } - -int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { - SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); - SQueryNodeEpId epId = {0}; - - epId.nodeId = addr->nodeId; - - SEp* pEp = SCH_GET_CUR_EP(addr); - strcpy(epId.ep.fqdn, pEp->fqdn); - epId.ep.port = pEp->port; - - SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, &epId, sizeof(SQueryNodeEpId)); - if (NULL == hb) { - bool exist = false; - SCH_ERR_RET(schRegisterHbConnection(pJob, pTask, &epId, &exist)); - if (!exist) { - SCH_ERR_RET(schBuildAndSendHbMsg(&epId)); - } - } - - return TSDB_CODE_SUCCESS; -} - -int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { - int32_t code = 0; - SSchHbTrans *hb = NULL; - - hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); - if (NULL == hb) { - qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - - SCH_LOCK(SCH_WRITE, &hb->lock); - memcpy(&hb->trans, trans, sizeof(*trans)); - SCH_UNLOCK(SCH_WRITE, &hb->lock); - - qDebug("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, pTrans:%p, pHandle:%p", schMgmt.sId, - epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->pTrans, trans->pHandle); - - return TSDB_CODE_SUCCESS; -} - int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { SSchedulerHbRsp rsp = {0}; SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; @@ -810,33 +757,12 @@ int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code) { } SSchTrans trans = {0}; - trans.pTrans = pParam->transport; + trans.pTrans = pParam->pTrans; trans.pHandle = pMsg->handle; SCH_ERR_JRET(schUpdateHbConnection(&rsp.epId, &trans)); - int32_t taskNum = (int32_t)taosArrayGetSize(rsp.taskStatus); - qDebug("%d task status in hb rsp, nodeId:%d, fqdn:%s, port:%d", taskNum, rsp.epId.nodeId, rsp.epId.ep.fqdn, - rsp.epId.ep.port); - - for (int32_t i = 0; i < taskNum; ++i) { - STaskStatus *taskStatus = taosArrayGet(rsp.taskStatus, i); - - SSchJob *pJob = schAcquireJob(taskStatus->refId); - if (NULL == pJob) { - qWarn("job not found, refId:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64, taskStatus->refId, - taskStatus->queryId, taskStatus->taskId); - // TODO DROP TASK FROM SERVER!!!! - continue; - } - - // TODO - - SCH_JOB_DLOG("TID:0x%" PRIx64 " task status in server: %s", taskStatus->taskId, - jobTaskStatusStr(taskStatus->status)); - - schReleaseJob(taskStatus->refId); - } + SCH_ERR_JRET(schProcessOnTaskStatusRsp(&rsp.epId, rsp.taskStatus)); _return: @@ -856,7 +782,8 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { param->queryId = pJob->queryId; param->refId = pJob->refId; param->taskId = SCH_TASK_ID(pTask); - param->transport = pJob->pTrans; + param->pTrans = pJob->pTrans; + param->taskId = pTask->taskId; *pParam = param; @@ -1034,6 +961,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); isCandidateAddr = true; + SCH_TASK_DLOG("target candidateIdx %d", pTask->candidateIdx); } SEpSet epSet = addr->epSet; @@ -1158,7 +1086,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, (rpcCtx.args ? &rpcCtx : NULL))); if (msgType == TDMT_VND_QUERY) { - SCH_ERR_RET(schRecordTaskExecNode(pJob, pTask, addr, trans.pHandle)); + SCH_ERR_RET(schAppendTaskExecNode(pJob, pTask, addr, pTask->execIdx)); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 81c95ea976e0c685fa1585df6dbb42bed75fd0c8..18398802dbd2ed0c251bd93a1fcafc56363d797e 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -21,17 +21,189 @@ #include "tref.h" #include "trpc.h" + +void schCleanClusterHb(void* pTrans) { + SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); + + SSchHbTrans *hb = taosHashIterate(schMgmt.hbConnections, NULL); + while (hb) { + if (hb->trans.pTrans == pTrans) { + SQueryNodeEpId* pEpId = taosHashGetKey(hb, NULL); + rpcReleaseHandle(hb->trans.pHandle, TAOS_CONN_CLIENT); + taosHashRemove(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); + } + + hb = taosHashIterate(schMgmt.hbConnections, hb); + } + + SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); +} + +int32_t schRemoveHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId) { + return TSDB_CODE_SUCCESS; // TODO ENABLE IT WHEN RPC IS READY + + int32_t code = 0; + + SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); + SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); + if (NULL == hb) { + SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); + SCH_TASK_ELOG("nodeId %d fqdn %s port %d not in hb connections", epId->nodeId, epId->ep.fqdn, epId->ep.port); + return TSDB_CODE_SUCCESS; + } + + int64_t taskNum = atomic_load_64(&hb->taskNum); + if (taskNum <= 0) { + rpcReleaseHandle(hb->trans.pHandle, TAOS_CONN_CLIENT); + taosHashRemove(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); + } + SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); + + return TSDB_CODE_SUCCESS; +} + + +int32_t schAddHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *epId, bool *exist) { + int32_t code = 0; + SSchHbTrans hb = {0}; + + hb.trans.pTrans = pJob->pTrans; + hb.taskNum = 1; + + SCH_ERR_RET(schMakeHbRpcCtx(pJob, pTask, &hb.rpcCtx)); + + SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); + code = taosHashPut(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId), &hb, sizeof(SSchHbTrans)); + if (code) { + SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); + schFreeRpcCtx(&hb.rpcCtx); + + if (HASH_NODE_EXIST(code)) { + *exist = true; + return TSDB_CODE_SUCCESS; + } + + qError("taosHashPut hb trans failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); + SCH_ERR_RET(code); + } + + SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); + + return TSDB_CODE_SUCCESS; +} + +int32_t schRegisterHbConnection(SSchJob *pJob, SSchTask *pTask, SQueryNodeEpId *pEpId) { + SSchHbTrans *hb = NULL; + + while (true) { + SCH_LOCK(SCH_READ, &schMgmt.hbLock); + hb = taosHashGet(schMgmt.hbConnections, pEpId, sizeof(SQueryNodeEpId)); + if (NULL == hb) { + bool exist = false; + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + SCH_ERR_RET(schAddHbConnection(pJob, pTask, pEpId, &exist)); + if (!exist) { + SCH_RET(schBuildAndSendHbMsg(pEpId, NULL)); + } + + continue; + } + + break; + } + + atomic_add_fetch_64(&hb->taskNum, 1); + + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + + return TSDB_CODE_SUCCESS; +} + +void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask) { + if (!pTask->registerdHb) { + return; + } + + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SQueryNodeEpId epId = {0}; + + epId.nodeId = addr->nodeId; + + SEp* pEp = SCH_GET_CUR_EP(addr); + strcpy(epId.ep.fqdn, pEp->fqdn); + epId.ep.port = pEp->port; + + SCH_LOCK(SCH_READ, &schMgmt.hbLock); + SSchHbTrans *hb = taosHashGet(schMgmt.hbConnections, &epId, sizeof(SQueryNodeEpId)); + if (NULL == hb) { + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + SCH_TASK_WLOG("nodeId %d fqdn %s port %d not in hb connections", epId.nodeId, epId.ep.fqdn, epId.ep.port); + return; + } + + int64_t taskNum = atomic_sub_fetch_64(&hb->taskNum, 1); + if (0 == taskNum) { + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + schRemoveHbConnection(pJob, pTask, &epId); + } else { + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + } + + pTask->registerdHb = false; +} + + + +int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask) { + SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SQueryNodeEpId epId = {0}; + + epId.nodeId = addr->nodeId; + + SEp* pEp = SCH_GET_CUR_EP(addr); + strcpy(epId.ep.fqdn, pEp->fqdn); + epId.ep.port = pEp->port; + + SCH_ERR_RET(schRegisterHbConnection(pJob, pTask, &epId)); + + pTask->registerdHb = true; + + return TSDB_CODE_SUCCESS; +} + +int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { + int32_t code = 0; + SSchHbTrans *hb = NULL; + + SCH_LOCK(SCH_READ, &schMgmt.hbLock); + hb = taosHashGet(schMgmt.hbConnections, epId, sizeof(SQueryNodeEpId)); + if (NULL == hb) { + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); + SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + } + + SCH_LOCK(SCH_WRITE, &hb->lock); + memcpy(&hb->trans, trans, sizeof(*trans)); + SCH_UNLOCK(SCH_WRITE, &hb->lock); + SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); + + qDebug("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, pTrans:%p, pHandle:%p", schMgmt.sId, + epId->nodeId, epId->ep.fqdn, epId->ep.port, trans->pTrans, trans->pHandle); + + return TSDB_CODE_SUCCESS; +} + + void schCloseJobRef(void) { if (!atomic_load_8((int8_t *)&schMgmt.exit)) { return; } - SCH_LOCK(SCH_WRITE, &schMgmt.lock); - if (atomic_load_32(&schMgmt.jobNum) <= 0 && schMgmt.jobRef >= 0) { + if (schMgmt.jobRef >= 0) { taosCloseRef(schMgmt.jobRef); schMgmt.jobRef = -1; } - SCH_UNLOCK(SCH_WRITE, &schMgmt.lock); } uint64_t schGenTaskId(void) { return atomic_add_fetch_64(&schMgmt.taskId, 1); } @@ -88,4 +260,3 @@ void schFreeRpcCtx(SRpcCtx *pCtx) { (*pCtx->freeFunc)(pCtx->brokenVal.val); } - diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 522bd8044d43709f9139092dd705ab123c7799ea..b4dc067f4c08b7eb765cfa3365b551f0053671a2 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -79,12 +79,19 @@ int32_t schedulerExecJob(void *pTrans, SArray *pNodeList, SQueryPlan *pDag, int6 int32_t schedulerAsyncExecJob(void *pTrans, SArray *pNodeList, SQueryPlan *pDag, int64_t *pJob, const char *sql, int64_t startTs, schedulerExecCallback fp, void* param) { - if (NULL == pTrans || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == fp || NULL == param) { - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } - - SSchResInfo resInfo = {.execFp = fp, .userParam = param}; - SCH_RET(schAsyncExecJob(pTrans, pNodeList, pDag, pJob, sql, startTs, &resInfo)); + int32_t code = 0; + if (NULL == pTrans || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == fp) { + code = TSDB_CODE_QRY_INVALID_INPUT; + } else { + SSchResInfo resInfo = {.execFp = fp, .userParam = param}; + code = schAsyncExecJob(pTrans, pNodeList, pDag, pJob, sql, startTs, &resInfo); + } + + if (code != TSDB_CODE_SUCCESS) { + fp(NULL, param, code); + } + + return code; } int32_t schedulerFetchRows(int64_t job, void **pData) { @@ -95,7 +102,7 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { int32_t code = 0; SSchJob *pJob = schAcquireJob(job); if (NULL == pJob) { - qError("acquire job from jobRef list failed, may be dropped, refId:%" PRIx64, job); + qError("acquire job from jobRef list failed, may be dropped, jobId:0x%" PRIx64, job); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -108,27 +115,26 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { SCH_RET(code); } -int32_t schedulerAsyncFetchRows(int64_t job, schedulerFetchCallback fp, void* param) { +void schedulerAsyncFetchRows(int64_t job, schedulerFetchCallback fp, void* param) { if (NULL == fp || NULL == param) { - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + fp(NULL, param, TSDB_CODE_QRY_INVALID_INPUT); + return; } - int32_t code = 0; SSchJob *pJob = schAcquireJob(job); if (NULL == pJob) { - qError("acquire job from jobRef list failed, may be dropped, refId:%" PRIx64, job); - SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + qError("acquire job from jobRef list failed, may be dropped, jobId:0x%" PRIx64, job); + fp(NULL, param, TSDB_CODE_SCH_STATUS_ERROR); + return; } pJob->attr.syncSchedule = false; pJob->userRes.fetchFp = fp; pJob->userRes.userParam = param; - code = schAsyncFetchRows(pJob); + /*code = */schAsyncFetchRows(pJob); schReleaseJob(job); - - SCH_RET(code); } int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub) { @@ -165,7 +171,7 @@ _return: int32_t scheduleCancelJob(int64_t job) { SSchJob *pJob = schAcquireJob(job); if (NULL == pJob) { - qError("acquire job from jobRef list failed, may be dropped, refId:%" PRIx64, job); + qError("acquire job from jobRef list failed, may be dropped, jobId:0x%" PRIx64, job); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -176,10 +182,18 @@ int32_t scheduleCancelJob(int64_t job) { SCH_RET(code); } +void schedulerStopQueryHb(void *pTrans) { + if (NULL == pTrans) { + return; + } + + schCleanClusterHb(pTrans); +} + void schedulerFreeJob(int64_t job) { SSchJob *pJob = schAcquireJob(job); if (NULL == pJob) { - qDebug("acquire job from jobRef list failed, may be dropped, refId:%" PRIx64, job); + qError("acquire job from jobRef list failed, may be dropped, jobId:0x%" PRIx64, job); return; } @@ -214,6 +228,7 @@ void schedulerDestroy(void) { } } + SCH_LOCK(SCH_WRITE, &schMgmt.hbLock); if (schMgmt.hbConnections) { void *pIter = taosHashIterate(schMgmt.hbConnections, NULL); while (pIter != NULL) { @@ -224,4 +239,5 @@ void schedulerDestroy(void) { taosHashCleanup(schMgmt.hbConnections); schMgmt.hbConnections = NULL; } + SCH_UNLOCK(SCH_WRITE, &schMgmt.hbLock); } diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 3afe7b15e213c0da3760c7a8ef1f313d145cd31f..89dcd8a476b7c88397b47cdb56618fac1c5863b1 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -200,7 +200,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SSyncRaftEntry* pRollBackEntry = ths->pLogStore->getEntry(ths->pLogStore, index); assert(pRollBackEntry != NULL); - // if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) { + // if (pRollBackEntry->msgType != TDMT_SYNC_NOOP) { if (syncUtilUserRollback(pRollBackEntry->msgType)) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); @@ -229,7 +229,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { - // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_SYNC_NOOP) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pAppendEntry->index; @@ -261,7 +261,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SRpcMsg rpcMsg; syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); if (ths->pFsm != NULL) { - // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_SYNC_NOOP) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pAppendEntry->index; @@ -346,7 +346,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { } // config change - if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) { + if (pEntry->originalRpcType == TDMT_SYNC_CONFIG_CHANGE) { SSyncCfg oldSyncCfg = ths->pRaftCfg->cfg; SSyncCfg newSyncCfg; diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 4a1a40a2d7ddd47d9d6ec30a683f284dacc70fa7..c6376495a4775f5f4264f3d7960e3995f610cc87 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -124,7 +124,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { } // config change - if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) { + if (pEntry->originalRpcType == TDMT_SYNC_CONFIG_CHANGE) { SSyncCfg oldSyncCfg = pSyncNode->pRaftCfg->cfg; SSyncCfg newSyncCfg; diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index e30a39e6342c4b7df77ee9cfdbe4f29333e36c16..aa8484de998ff327ab34293d793a974b5e92a05a 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -256,7 +256,7 @@ static void *syncIOConsumerFunc(void *param) { syncRpcMsgLog2((char *)"==syncIOConsumerFunc==", pRpcMsg); // use switch case instead of if else - if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) { + if (pRpcMsg->msgType == TDMT_SYNC_PING) { if (io->FpOnSyncPing != NULL) { SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -264,7 +264,7 @@ static void *syncIOConsumerFunc(void *param) { syncPingDestroy(pSyncMsg); } - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) { + } else if (pRpcMsg->msgType == TDMT_SYNC_PING_REPLY) { if (io->FpOnSyncPingReply != NULL) { SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -272,7 +272,7 @@ static void *syncIOConsumerFunc(void *param) { syncPingReplyDestroy(pSyncMsg); } - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) { + } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { if (io->FpOnSyncClientRequest != NULL) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -280,7 +280,7 @@ static void *syncIOConsumerFunc(void *param) { syncClientRequestDestroy(pSyncMsg); } - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) { + } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { if (io->FpOnSyncRequestVote != NULL) { SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -288,7 +288,7 @@ static void *syncIOConsumerFunc(void *param) { syncRequestVoteDestroy(pSyncMsg); } - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) { + } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { if (io->FpOnSyncRequestVoteReply != NULL) { SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -296,7 +296,7 @@ static void *syncIOConsumerFunc(void *param) { syncRequestVoteReplyDestroy(pSyncMsg); } - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) { + } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { if (io->FpOnSyncAppendEntries != NULL) { SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -304,7 +304,7 @@ static void *syncIOConsumerFunc(void *param) { syncAppendEntriesDestroy(pSyncMsg); } - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) { + } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { if (io->FpOnSyncAppendEntriesReply != NULL) { SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -312,7 +312,7 @@ static void *syncIOConsumerFunc(void *param) { syncAppendEntriesReplyDestroy(pSyncMsg); } - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_TIMEOUT) { + } else if (pRpcMsg->msgType == TDMT_SYNC_TIMEOUT) { if (io->FpOnSyncTimeout != NULL) { SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg); assert(pSyncMsg != NULL); @@ -365,7 +365,7 @@ static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { } static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { - if (pMsg->msgType == TDMT_VND_SYNC_COMMON_RESPONSE) { + if (pMsg->msgType == TDMT_SYNC_COMMON_RESPONSE) { sTrace("==syncIOProcessReply=="); } else { syncRpcMsgLog2((char *)"==syncIOProcessReply==", pMsg); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 620fc514c6960754762187445b167098ce4382d3..d522b829dd8620c856316626c8c62c7e75ba83a0 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -174,7 +174,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { sInfo("==syncReconfig== newconfig:%s", configChange); SRpcMsg rpcMsg = {0}; - rpcMsg.msgType = TDMT_VND_SYNC_CONFIG_CHANGE; + rpcMsg.msgType = TDMT_SYNC_CONFIG_CHANGE; rpcMsg.info.noResp = 1; rpcMsg.contLen = strlen(configChange) + 1; rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); @@ -1399,7 +1399,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_SYNC_NOOP) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; @@ -1421,7 +1421,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) { + // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_SYNC_NOOP) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta; cbMeta.index = pEntry->index; @@ -1450,12 +1450,12 @@ static void syncFreeNode(void* param) { const char* syncStr(ESyncState state) { switch (state) { case TAOS_SYNC_STATE_FOLLOWER: - return "FOLLOWER"; + return "follower"; case TAOS_SYNC_STATE_CANDIDATE: - return "CANDIDATE"; + return "candidate"; case TAOS_SYNC_STATE_LEADER: - return "LEADER"; + return "leader"; default: - return "ERROR"; + return "error"; } } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index fae069f2e6b13c0073c6309f889dc7f8f92c8c6e..6871e6b3ed3d37c45833b6133e6430923ca5d727 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -22,50 +22,50 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { cJSON* pRoot; // in compiler optimization, switch case = if else constants - if (pRpcMsg->msgType == TDMT_VND_SYNC_TIMEOUT) { + if (pRpcMsg->msgType == TDMT_SYNC_TIMEOUT) { SyncTimeout* pSyncMsg = syncTimeoutDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncTimeout2Json(pSyncMsg); syncTimeoutDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) { + } else if (pRpcMsg->msgType == TDMT_SYNC_PING) { SyncPing* pSyncMsg = syncPingDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncPing2Json(pSyncMsg); syncPingDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) { + } else if (pRpcMsg->msgType == TDMT_SYNC_PING_REPLY) { SyncPingReply* pSyncMsg = syncPingReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncPingReply2Json(pSyncMsg); syncPingReplyDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) { + } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { SyncClientRequest* pSyncMsg = syncClientRequestDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncClientRequest2Json(pSyncMsg); syncClientRequestDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST_REPLY) { + } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST_REPLY) { pRoot = syncRpcUnknownMsg2Json(); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) { + } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { SyncRequestVote* pSyncMsg = syncRequestVoteDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncRequestVote2Json(pSyncMsg); syncRequestVoteDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) { + } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { SyncRequestVoteReply* pSyncMsg = syncRequestVoteReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncRequestVoteReply2Json(pSyncMsg); syncRequestVoteReplyDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) { + } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { SyncAppendEntries* pSyncMsg = syncAppendEntriesDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncAppendEntries2Json(pSyncMsg); syncAppendEntriesDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) { + } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { SyncAppendEntriesReply* pSyncMsg = syncAppendEntriesReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); pRoot = syncAppendEntriesReply2Json(pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); - } else if (pRpcMsg->msgType == TDMT_VND_SYNC_COMMON_RESPONSE) { + } else if (pRpcMsg->msgType == TDMT_SYNC_COMMON_RESPONSE) { pRoot = cJSON_CreateObject(); char* s; s = syncUtilprintBin((char*)(pRpcMsg->pCont), pRpcMsg->contLen); @@ -98,7 +98,7 @@ cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg) { cJSON* syncRpcUnknownMsg2Json() { cJSON* pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "msgType", TDMT_VND_SYNC_UNKNOWN); + cJSON_AddNumberToObject(pRoot, "msgType", TDMT_SYNC_UNKNOWN); cJSON_AddStringToObject(pRoot, "data", "unknown message"); cJSON* pJson = cJSON_CreateObject(); @@ -146,7 +146,7 @@ SyncTimeout* syncTimeoutBuild() { SyncTimeout* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; - pMsg->msgType = TDMT_VND_SYNC_TIMEOUT; + pMsg->msgType = TDMT_SYNC_TIMEOUT; return pMsg; } @@ -275,7 +275,7 @@ SyncPing* syncPingBuild(uint32_t dataLen) { SyncPing* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; - pMsg->msgType = TDMT_VND_SYNC_PING; + pMsg->msgType = TDMT_SYNC_PING; pMsg->dataLen = dataLen; return pMsg; } @@ -535,7 +535,7 @@ SyncPingReply* syncPingReplyBuild(uint32_t dataLen) { SyncPingReply* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; - pMsg->msgType = TDMT_VND_SYNC_PING_REPLY; + pMsg->msgType = TDMT_SYNC_PING_REPLY; pMsg->dataLen = dataLen; return pMsg; } @@ -795,7 +795,7 @@ SyncClientRequest* syncClientRequestBuild(uint32_t dataLen) { SyncClientRequest* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; - pMsg->msgType = TDMT_VND_SYNC_CLIENT_REQUEST; + pMsg->msgType = TDMT_SYNC_CLIENT_REQUEST; pMsg->seqNum = 0; pMsg->isWeak = false; pMsg->dataLen = dataLen; @@ -937,7 +937,7 @@ SyncRequestVote* syncRequestVoteBuild(int32_t vgId) { memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->vgId = vgId; - pMsg->msgType = TDMT_VND_SYNC_REQUEST_VOTE; + pMsg->msgType = TDMT_SYNC_REQUEST_VOTE; return pMsg; } @@ -1086,7 +1086,7 @@ SyncRequestVoteReply* syncRequestVoteReplyBuild(int32_t vgId) { memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->vgId = vgId; - pMsg->msgType = TDMT_VND_SYNC_REQUEST_VOTE_REPLY; + pMsg->msgType = TDMT_SYNC_REQUEST_VOTE_REPLY; return pMsg; } @@ -1232,7 +1232,7 @@ SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen, int32_t vgId) { memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->vgId = vgId; - pMsg->msgType = TDMT_VND_SYNC_APPEND_ENTRIES; + pMsg->msgType = TDMT_SYNC_APPEND_ENTRIES; pMsg->dataLen = dataLen; return pMsg; } @@ -1398,7 +1398,7 @@ SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId) { memset(pMsg, 0, bytes); pMsg->bytes = bytes; pMsg->vgId = vgId; - pMsg->msgType = TDMT_VND_SYNC_APPEND_ENTRIES_REPLY; + pMsg->msgType = TDMT_SYNC_APPEND_ENTRIES_REPLY; return pMsg; } @@ -1546,7 +1546,7 @@ SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen) { SyncApplyMsg* pMsg = taosMemoryMalloc(bytes); memset(pMsg, 0, bytes); pMsg->bytes = bytes; - pMsg->msgType = TDMT_VND_SYNC_APPLY_MSG; + pMsg->msgType = TDMT_SYNC_APPLY_MSG; pMsg->dataLen = dataLen; return pMsg; } diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 8755f71654382f3913a3c81b6ee1e9b6e91dbb69..05a2dbaa3f36b00cd9202844d65f97f022ba22ae 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -59,14 +59,14 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) memset(&rpcMsg, 0, sizeof(SRpcMsg)); rpcMsg.contLen = head.contLen; rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - rpcMsg.msgType = TDMT_VND_SYNC_NOOP; + rpcMsg.msgType = TDMT_SYNC_NOOP; memcpy(rpcMsg.pCont, &head, sizeof(head)); SSyncRaftEntry* pEntry = syncEntryBuild(rpcMsg.contLen); assert(pEntry != NULL); - pEntry->msgType = TDMT_VND_SYNC_CLIENT_REQUEST; - pEntry->originalRpcType = TDMT_VND_SYNC_NOOP; + pEntry->msgType = TDMT_SYNC_CLIENT_REQUEST; + pEntry->originalRpcType = TDMT_SYNC_NOOP; pEntry->seqNum = 0; pEntry->isWeak = 0; pEntry->term = term; diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index a6397f8cba24694d6f36847af5e877c72bd1a920..b353ed85db27888483f4de249f86206d4b195c0e 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -104,7 +104,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); assert(pEntry != NULL); - pEntry->msgType = TDMT_VND_SYNC_CLIENT_REQUEST; + pEntry->msgType = TDMT_SYNC_CLIENT_REQUEST; pEntry->originalRpcType = pWalHandle->pHead->head.msgType; pEntry->seqNum = pWalHandle->pHead->head.syncMeta.seqNum; pEntry->isWeak = pWalHandle->pHead->head.syncMeta.isWeek; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index d754acd9f831ac18ce7e28b5ef2fda4b2d8650db..48567b75c27b8c13aaefc6d029c4c7a957b23c1a 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -215,28 +215,28 @@ void syncUtilMsgNtoH(void* msg) { } bool syncUtilIsData(tmsg_t msgType) { - if (msgType == TDMT_VND_SYNC_NOOP || msgType == TDMT_VND_SYNC_CONFIG_CHANGE) { + if (msgType == TDMT_SYNC_NOOP || msgType == TDMT_SYNC_CONFIG_CHANGE) { return false; } return true; } bool syncUtilUserPreCommit(tmsg_t msgType) { - if (msgType != TDMT_VND_SYNC_NOOP && msgType != TDMT_VND_SYNC_CONFIG_CHANGE) { + if (msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_CONFIG_CHANGE) { return true; } return false; } bool syncUtilUserCommit(tmsg_t msgType) { - if (msgType != TDMT_VND_SYNC_NOOP && msgType != TDMT_VND_SYNC_CONFIG_CHANGE) { + if (msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_CONFIG_CHANGE) { return true; } return false; } bool syncUtilUserRollback(tmsg_t msgType) { - if (msgType != TDMT_VND_SYNC_NOOP && msgType != TDMT_VND_SYNC_CONFIG_CHANGE) { + if (msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_CONFIG_CHANGE) { return true; } return false; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 580ab30b7800c05d6f8463700cc208cd850b9a8e..0090701ba567378c4e2aab0869a757663b9d289e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1151,7 +1151,6 @@ void transSetDefaultAddr(void* ahandle, const char* ip, const char* fqdn) { SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i]; tDebug("update epset at thread:%d, threadID:%" PRId64 "", i, thrd->thread); - tsem_t* pSem = pCtx->pSem; transSendAsync(thrd->asyncPool, &(cliMsg->q)); } } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 608fd00b2cda7c9508275cd4487496295b9e0711..50f99128b2fea235aaee1ef1d337149d98efaac6 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -180,6 +180,12 @@ static bool addHandleToAcceptloop(void* arg); if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ return; \ } \ + if (conn->regArg.init) { \ + tTrace("server conn %p release, notify server app", conn); \ + STrans* pTransInst = conn->pTransInst; \ + (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); \ + memset(&conn->regArg, 0, sizeof(conn->regArg)); \ + } \ uvStartSendRespInternal(srvMsg); \ return; \ } \ @@ -1154,6 +1160,10 @@ int transGetConnInfo(void* thandle, STransHandleInfo* pInfo) { } SExHandle* ex = thandle; SSvrConn* pConn = ex->handle; + if (pConn == NULL) { + tTrace("invalid handle %p, failed to Get Conn info", thandle); + return -1; + } struct sockaddr_in addr = pConn->addr; pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 4e2270c37bc00e9055bf340dab898d842feb5ce4..a9a8f8a1f40fbfd6fda3ec43d4fd1bfdf61025dc 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -19,7 +19,7 @@ #include "walInt.h" void walRestoreFromSnapshot(SWal *pWal, int64_t ver) { - pWal->vers.firstVer = -1; + /*pWal->vers.firstVer = -1;*/ pWal->vers.lastVer = ver; pWal->vers.commitVer = ver - 1; pWal->vers.snapshotVer = ver - 1; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index c75cca79f6b82e2989b7199068db297c7b91a1eb..32fed66cd4f2c7319d4a74b7ff19306f17076597 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -399,6 +399,9 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) } int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { + if (pFile == NULL) { + return 0; + } #if FILE_WITH_LOCK taosThreadRwlockWrlock(&(pFile->rwlock)); #endif @@ -430,6 +433,9 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { } int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { + if (pFile == NULL) { + return 0; + } #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 4d7b15401ce60012f4afcdde7cc6bf7bca225081..ace870853f9a0614eaf5a8af1f9672054db21935 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -276,7 +276,8 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { #ifdef WINDOWS - assert(0); + snprintf(releaseName, maxLen, "Windows"); + return 0; #elif defined(_TD_DARWIN_64) char *line = NULL; size_t size = 0; @@ -332,7 +333,15 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { #ifdef WINDOWS - assert(0); + char value[100]; + DWORD bufferSize = sizeof(value); + RegGetValue(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", "ProcessorNameString", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + tstrncpy(cpuModel, value, maxLen); + SYSTEM_INFO si; + memset(&si,0,sizeof(SYSTEM_INFO)); + GetSystemInfo(&si); + *numOfCores = si.dwNumberOfProcessors; + return 0; #elif defined(_TD_DARWIN_64) char *line = NULL; size_t size = 0; diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index dc9527c2f2a052cf2d87b6b6c374019139390beb..724f81c66ce2ace2015c32bdb1372815fc4407c5 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -767,32 +767,36 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8 keyValue[4] = (keyValue[4] == '+' ? '-' : '+'); keyValue[10] = 0; sprintf(winStr, "TZ=%s:00", &(keyValue[1])); + *tsTimezone = taosStr2Int32(&keyValue[4], NULL, 10); } break; } } - char *p = strchr(inTimezoneStr, '+'); - if (p == NULL) p = strchr(inTimezoneStr, '-'); - if (p == NULL) { - sprintf(winStr, "TZ=UTC+00:00:00"); - } else { - sprintf(winStr, "TZ=UTC%c%c%c:%c%c:00", (p[0] == '+' ? '-' : '+'), p[1], p[2], p[3], p[4]); + if (winStr[0] == 0) { + char *p = strchr(inTimezoneStr, '+'); + if (p == NULL) p = strchr(inTimezoneStr, '-'); + if (p != NULL) { + char *pp = strchr(inTimezoneStr, '('); + char *ppp = strchr(inTimezoneStr, ','); + int indexStr; + if (pp == NULL || ppp == NULL) { + indexStr = sprintf(winStr, "TZ=UTC"); + } else { + memcpy(winStr, "TZ=", 3); + pp++; + memcpy(&winStr[3], pp, ppp - pp); + indexStr = ppp - pp + 3; + } + sprintf(&winStr[indexStr], "%c%c%c:%c%c:00", (p[0] == '+'? '-' : '+'), p[1], p[2], p[3], p[4]); + *tsTimezone = taosStr2Int32(p, NULL, 10); + } else { + *tsTimezone = 0; + } } _putenv(winStr); _tzset(); -#ifdef _MSC_VER -#if _MSC_VER >= 1900 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char **tzname = _tzname; -#endif -#endif - - int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); - *tsTimezone = tz; - tz += daylight; - sprintf(outTimezoneStr, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); - *outDaylight = daylight; + strcpy(outTimezoneStr, inTimezoneStr); + *outDaylight = 0; #elif defined(_TD_DARWIN_64) @@ -822,34 +826,27 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8 void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { #ifdef WINDOWS char value[100]; + char keyPath[100]; DWORD bufferSize = sizeof(value); - char *buf = getenv("TZ"); - if (buf == NULL || strlen(buf) == 0) { - RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", "TimeZoneKeyName", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); - strcpy(outTimezoneStr, "not configured"); - if (bufferSize > 0) { - for (size_t i = 0; i < 139; i++) { - if (strcmp(win_tz[i][0],value) == 0) { - strcpy(outTimezoneStr, win_tz[i][1]); - break; + RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", "TimeZoneKeyName", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + strcpy(outTimezoneStr, "not configured"); + *tsTimezone = 0; + if (bufferSize > 0) { + for (size_t i = 0; i < 139; i++) { + if (strcmp(win_tz[i][0],value) == 0) { + strcpy(outTimezoneStr, win_tz[i][1]); + bufferSize = sizeof(value); + sprintf(keyPath, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\%s",value); + RegGetValue(HKEY_LOCAL_MACHINE, keyPath, "Display", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + if (bufferSize > 0) { + // value[4] = (value[4] == '+' ? '-' : '+'); + sprintf(outTimezoneStr, "%s (UTC, %c%c%c%c%c)", outTimezoneStr, value[4], value[5], value[6], value[8], value[9]); + *tsTimezone = taosStr2Int32(&value[4], NULL, 10); } + break; } } - } else { - strcpy(outTimezoneStr, buf); } -#ifdef _MSC_VER -#if _MSC_VER >= 1900 - // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char **tzname = _tzname; -#endif -#endif - int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); - *tsTimezone = tz; - tz += daylight; - sprintf(outTimezoneStr, "%s (%s, %s%02d00)", outTimezoneStr, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); #elif defined(_TD_DARWIN_64) char buf[4096] = {0}; char *tz = NULL; diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 7f1fdf33061c7d1a3a2e820346db49de398a1bdb..b6609278992d2e7cd4c9ef87bf913b94ab46735f 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -227,6 +227,33 @@ int32_t compareJsonContainsKey(const void* pLeft, const void* pRight) { return 1; } +// string > number > bool > null +// ref: https://dev.mysql.com/doc/refman/8.0/en/json.html#json-comparison +int32_t compareJsonVal(const void *pLeft, const void *pRight) { + char leftType = *(char*)pLeft; + char rightType = *(char*)pRight; + if(leftType != rightType){ + return leftType > rightType ? 1 : -1; + } + + char* realDataLeft = POINTER_SHIFT(pLeft, CHAR_BYTES); + char* realDataRight = POINTER_SHIFT(pRight, CHAR_BYTES); + if(leftType == TSDB_DATA_TYPE_BOOL) { + DEFAULT_COMP(GET_INT8_VAL(realDataLeft), GET_INT8_VAL(realDataRight)); + }else if(leftType == TSDB_DATA_TYPE_DOUBLE){ + DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(realDataLeft), GET_DOUBLE_VAL(realDataRight)); + }else if(leftType == TSDB_DATA_TYPE_NCHAR){ + return compareLenPrefixedWStr(realDataLeft, realDataRight); + }else if(leftType == TSDB_DATA_TYPE_NULL) { + return 0; + }else{ + assert(0); + } +} + +int32_t compareJsonValDesc(const void *pLeft, const void *pRight) { + return compareJsonVal(pRight, pLeft); +} /* * Compare two strings * TSDB_MATCH: Match @@ -601,6 +628,8 @@ __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { return (order == TSDB_ORDER_ASC) ? compareLenPrefixedStr : compareLenPrefixedStrDesc; case TSDB_DATA_TYPE_NCHAR: return (order == TSDB_ORDER_ASC) ? compareLenPrefixedWStr : compareLenPrefixedWStrDesc; + case TSDB_DATA_TYPE_JSON: + return (order == TSDB_ORDER_ASC) ? compareJsonVal : compareJsonValDesc; default: return (order == TSDB_ORDER_ASC) ? compareInt32Val : compareInt32ValDesc; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 6e74109a7da568ab8421c799015f79f13a2447b8..8d7bc0851b355aad8221cd2c2d97097e234af039 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -190,7 +190,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_ALREADY_EXIST, "Bnode already exists" TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_NOT_EXIST, "Bnode not there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_FEW_MNODES, "The replicas of mnode cannot less than 1") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_MNODES, "The replicas of mnode cannot exceed 3") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_CANT_DROP_MASTER, "Can't drop mnode which is LEADER") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_CANT_DROP_MASTER, "Can't drop mnode which is leader") // mnode-acct TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, "Account already exists") @@ -225,14 +225,14 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, "Vgroup not in dnode") TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist") // mnode-stable -TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "Stable already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_NOT_EXIST, "Stable not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC, "Stable confilct with topic") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "STable already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_NOT_EXIST, "STable not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC, "STable confilct with topic") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_STBS, "Too many stables") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB, "Invalid stable name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_OPTION, "Invalid stable options") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_ALTER_OPTION, "Invalid stable alter options") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_OPTION_UNCHNAGED, "Stable option unchanged") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_OPTION_UNCHNAGED, "STable option unchanged") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ROW_BYTES, "Invalid row bytes") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, "Too many tags") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREADY_EXIST, "Tag already exists") @@ -240,6 +240,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, "Tag does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST, "Column already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC,"Field used by topic") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SINGLE_STB_MODE_DB, "Database is single stable mode") // mnode-infoSchema @@ -444,6 +445,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_CTG_VG_META_MISMATCH, "table meta and vgroup //scheduler TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR, "scheduler status error") TAOS_DEFINE_ERROR(TSDB_CODE_SCH_INTERNAL_ERROR, "scheduler internal error") +TAOS_DEFINE_ERROR(TSDB_CODE_SCH_TIMEOUT_ERROR, "Task timeout") TAOS_DEFINE_ERROR(TSDB_CODE_QW_MSG_ERROR, "Invalid msg order") // parser diff --git a/source/util/src/tlist.c b/source/util/src/tlist.c index 1d17b4a9e17aa7cafdd89ba273770e8751f09066..b1c018805157fe05ef6be97fa7be6df0255d5d5b 100644 --- a/source/util/src/tlist.c +++ b/source/util/src/tlist.c @@ -95,7 +95,7 @@ SListNode *tdListPopTail(SList *list) { SListNode *tdListGetHead(SList *list) { return TD_DLIST_HEAD(list); } -SListNode *tsListGetTail(SList *list) { return TD_DLIST_TAIL(list); } +SListNode *tdListGetTail(SList *list) { return TD_DLIST_TAIL(list); } SListNode *tdListPopNode(SList *list, SListNode *node) { TD_DLIST_POP(list, node); diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 686e0696ec689b48ecff8f27c7db2eb86daa5eb2..68f96c0385b6c25a4736343917e875f84d4e2c9e 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -163,7 +163,7 @@ int32_t tWWorkerInit(SWWorkerPool *pool) { worker->pool = pool; } - uInfo("worker:%s is initialized, max:%d", pool->name, pool->max); + uDebug("worker:%s is initialized, max:%d", pool->name, pool->max); return 0; } @@ -190,7 +190,7 @@ void tWWorkerCleanup(SWWorkerPool *pool) { taosMemoryFreeClear(pool->workers); taosThreadMutexDestroy(&pool->mutex); - uInfo("worker:%s is closed", pool->name); + uDebug("worker:%s is closed", pool->name); } static void *tWWorkerThreadFp(SWWorker *worker) { diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 21d235ee5c3502ff3248681ada0a6f2c99a805ea..b6ffda5c86d0a2799270634e3eee3f013dc8ca57 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -145,6 +145,12 @@ class TDDnode: def init(self, path, remoteIP = ""): self.path = path self.remoteIP = remoteIP + if (not self.remoteIP == ""): + try: + self.config = eval(self.remoteIP) + self.remote_conn = Connection(host=self.config["host"], port=self.config["port"], user=self.config["user"], connect_kwargs={'password':self.config["password"]}) + except Exception as r: + print(r) def setTestCluster(self, value): self.testCluster = value @@ -169,13 +175,6 @@ class TDDnode: self.cfgDict.update({option: value}) def remoteExec(self, updateCfgDict, execCmd): - try: - config = eval(self.remoteIP) - remote_conn = Connection(host=config["host"], port=config["port"], user=config["user"], connect_kwargs={'password':config["password"]}) - remote_top_dir = config["path"] - except Exception as r: - remote_conn = Connection(host=self.remoteIP, port=22, user='root', connect_kwargs={'password':'123456'}) - remote_top_dir = '~/test' valgrindStr = '' if (self.valgrind==1): valgrindStr = '-g' @@ -188,8 +187,8 @@ class TDDnode: del remoteCfgDict["cfgDir"] remoteCfgDictStr = base64.b64encode(json.dumps(remoteCfgDict).encode()).decode() execCmdStr = base64.b64encode(execCmd.encode()).decode() - with remote_conn.cd((remote_top_dir+sys.path[0].replace(self.path, '')).replace('\\','/')): - remote_conn.run("python3 ./test.py %s -d %s -e %s"%(valgrindStr,remoteCfgDictStr,execCmdStr)) + with self.remote_conn.cd((self.config["path"]+sys.path[0].replace(self.path, '')).replace('\\','/')): + self.remote_conn.run("python3 ./test.py %s -d %s -e %s"%(valgrindStr,remoteCfgDictStr,execCmdStr)) def deploy(self, *updatecfgDict): self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) @@ -247,7 +246,7 @@ class TDDnode: if bool(updatecfgDict) and updatecfgDict[0] and updatecfgDict[0][0]: print(updatecfgDict[0][0]) for key, value in updatecfgDict[0][0].items(): - if key == "clientCfg": + if key == "clientCfg" and self.remoteIP == "" and not platform.system().lower() == 'windows': continue if value == 'dataDir': if isFirstDir: @@ -319,7 +318,7 @@ class TDDnode: print(cmd) if (not self.remoteIP == ""): - self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)\ntdDnodes.start(%d)"%(self.index, self.index)) + self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].deployed=1\ntdDnodes.dnodes[%d].logDir=\"%%s/sim/dnode%%d/log\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.dnodes[%d].cfgDir=\"%%s/sim/dnode%%d/cfg\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.start(%d)"%(self.index-1,self.index-1,self.index-1,self.index,self.index-1,self.index-1,self.index,self.index)) self.running = 1 else: if os.system(cmd) != 0: diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index bb446fb248602842b579817dbdbbcc02f0a0680f..32056e8d0ceb2d128b25b86e24ce9d42dfa6ba15 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -58,7 +58,7 @@ ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim ./test.sh -f tsim/mnode/basic3.sim -#./test.sh -f tsim/mnode/basic4.sim +./test.sh -f tsim/mnode/basic4.sim # ---- show ./test.sh -f tsim/show/basic.sim @@ -135,6 +135,7 @@ ./test.sh -f tsim/sync/3Replica5VgElect.sim ./test.sh -f tsim/sync/oneReplica1VgElect.sim ./test.sh -f tsim/sync/oneReplica5VgElect.sim +# ./test.sh -f tsim/sync/3Replica5VgElect3mnode.sim # --- catalog ./test.sh -f tsim/catalog/alterInCurrent.sim diff --git a/tests/script/tsim/bnode/basic1.sim b/tests/script/tsim/bnode/basic1.sim index b1db6efc72afce083d9594987ccee3d10ab83ef4..80608453b8cf1243f27583a719f315462a4412d4 100644 --- a/tests/script/tsim/bnode/basic1.sim +++ b/tests/script/tsim/bnode/basic1.sim @@ -24,7 +24,7 @@ if $data00 != 1 then return -1 endi -if $data02 != LEADER then +if $data02 != leader then return -1 endi @@ -71,7 +71,7 @@ if $data00 != 1 then return -1 endi -if $data02 != LEADER then +if $data02 != leader then return -1 endi diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index aeb04293f2d5df29a07e629d32df3d96cb5d16b1..744451150429ce110f2caf69af5e059765325f26 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -89,7 +89,7 @@ endi if $data4_db != 3 then # replica return -1 endi -if $data5_db != nostrict then # strict +if $data5_db != no_strict then # strict return -1 endi if $data6_db != 345600 then # days @@ -131,43 +131,43 @@ endi sleep 3000 #sql show db.vgroups -#if $data[0][4] == LEADER then -# if $data[0][6] != FOLLOWER then +#if $data[0][4] == leader then +# if $data[0][6] != follower then # return -1 # endi -# if $data[0][8] != FOLLOWER then +# if $data[0][8] != follower then # return -1 # endi #endi -#if $data[0][6] == LEADER then -# if $data[0][4] != FOLLOWER then +#if $data[0][6] == leader then +# if $data[0][4] != follower then # return -1 # endi -# if $data[0][8] != FOLLOWER then +# if $data[0][8] != follower then # return -1 # endi #endi -#if $data[0][8] == LEADER then -# if $data[0][4] != FOLLOWER then +#if $data[0][8] == leader then +# if $data[0][4] != follower then # return -1 # endi -# if $data[0][6] != FOLLOWER then +# if $data[0][6] != follower then # return -1 # endi #endi # -#if $data[0][4] != LEADER then -# if $data[0][4] != FOLLOWER then +#if $data[0][4] != leader then +# if $data[0][4] != follower then # return -1 # endi #endi -#if $data[0][6] != LEADER then -# if $data[0][6] != FOLLOWER then +#if $data[0][6] != leader then +# if $data[0][6] != follower then # return -1 # endi #endi -#if $data[0][8] != LEADER then -# if $data[0][8] != FOLLOWER then +#if $data[0][8] != leader then +# if $data[0][8] != follower then # return -1 # endi #endi diff --git a/tests/script/tsim/db/alter_replica_13.sim b/tests/script/tsim/db/alter_replica_13.sim index 8ab6eb64fd93d3cb9b2b9882392ed2f40bc15f67..4d45b9296709b3f3367e025cca1fc3c8a31faea4 100644 --- a/tests/script/tsim/db/alter_replica_13.sim +++ b/tests/script/tsim/db/alter_replica_13.sim @@ -12,11 +12,11 @@ sql connect print =============== step1: create dnodes sql create dnode $hostname port 7200 -$loop_cnt = 0 +$x = 0 step1: - $loop_cnt = $loop_cnt + 1 + $ = $x + 1 sleep 1000 - if $loop_cnt == 10 then + if $x == 10 then print ====> dnode not ready! return -1 endi @@ -73,11 +73,11 @@ print =============== step3: create dnodes sql create dnode $hostname port 7300 sql create dnode $hostname port 7400 -$loop_cnt = 0 +$x = 0 step3: - $loop_cnt = $loop_cnt + 1 + $x = $x + 1 sleep 1000 - if $loop_cnt == 10 then + if $x == 10 then print ====> dnode not ready! return -1 endi @@ -103,22 +103,38 @@ if $data(4)[4] != ready then goto step3 endi -return print ============= step4: alter database sql alter database db replica 3 -if $rows != 3 then - return -1 + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show db.vgroups +print ===> rows: $rows +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +if $data[0][4] != leader then + goto step4 endi -if $data(db)[4] != 3 then - return -1 +if $data[0][6] != follower then + goto step4 endi +if $data[0][8] != follower then + goto step4 +endi + +print ============= step5: stop dnode 2 sql select * from db.stb if $rows != 1 then return -1 endi - system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/db/alter_replica_31.sim b/tests/script/tsim/db/alter_replica_31.sim new file mode 100644 index 0000000000000000000000000000000000000000..cbd4183b5e7cdb49c1335693c8d49c0d4c82839d --- /dev/null +++ b/tests/script/tsim/db/alter_replica_31.sim @@ -0,0 +1,167 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sql connect + +print =============== step1: create dnodes +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> rows: $rows +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +print ===> $data30 $data31 $data32 $data33 $data24 $data35 +if $rows != 4 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi + +print =============== step2: create database +sql create database db vgroups 1 replica 3 +sql show databases +if $rows != 3 then + return -1 +endi +if $data(db)[4] != 3 then + return -1 +endi + +sql show dnodes +if $data(2)[2] != 1 then + return -1 +endi +if $data(3)[2] != 1 then + return -1 +endi +if $data(4)[2] != 1 then + return -1 +endi + +# vnodes +sql show dnodes +if $data(2)[2] != 1 then + return -1 +endi +if $data(3)[2] != 1 then + return -1 +endi +if $data(4)[2] != 1 then + return -1 +endi + +# v1_dnode +$hasleader = 0 +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 20 then + print ====> dnode not ready! + return -1 + endi +sql show db.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 +if $data(2)[3] != 4 then + return -1 +endi +if $data(2)[5] != 3 then + return -1 +endi +if $data(2)[7] != 2 then + return -1 +endi +if $data(2)[4] == leader then + $hasleader = 1 +endi +if $data(2)[6] == leader then + $hasleader = 1 +endi +if $data(2)[8] == leader then + $hasleader = 1 +endi +if $hasleader != 1 then + goto step2 +endi + +sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 binary(16)) comment "abd" +sql create table db.ctb using db.stb tags(101, "102") +sql insert into db.ctb values(now, 1, "2") + +sql show db.vgroups +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 + +sleep 1000 +sql select * from db.ctb +print ===> $data00 $data01 $data02 $data03 $data04 $data05 + +if $rows != 1 then + return -1 +endi + +sleep 3000 + +print ============= step3: alter database +sql alter database db replica 1 + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show db.vgroups +print ===> rows: $rows +if $rows != 1 then + goto step3 +endi +if $data(2)[3] != 4 then + goto step3 +endi +if $data(2)[5] != NULL then + goto step3 +endi +if $data(2)[7] != NULL then + goto step3 +endi + +print ============= step5: stop dnode 2 +return +sql select * from db.stb +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index d7d72c4a1df1116d563c2f9283994e9807eac9d7..88f0378d618229ee6edd4ccf7879793453305746 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -110,7 +110,7 @@ if $data4_db != 1 then # replica print expect 1, actual: $data4_db return -1 endi -if $data5_db != nostrict then # strict +if $data5_db != no_strict then # strict return -1 endi if $data6_db != 14400 then # days diff --git a/tests/script/tsim/dnode/basic1.sim b/tests/script/tsim/dnode/basic1.sim index d5c791e902aef3404f854287cef6224767080f82..a5b5427e036e1f74a2287a2d4995c5936fd149f5 100644 --- a/tests/script/tsim/dnode/basic1.sim +++ b/tests/script/tsim/dnode/basic1.sim @@ -27,7 +27,7 @@ if $data00 != 1 then return -1 endi -if $data02 != LEADER then +if $data02 != leader then return -1 endi @@ -74,7 +74,7 @@ if $data00 != 1 then return -1 endi -if $data02 != LEADER then +if $data02 != leader then return -1 endi diff --git a/tests/script/tsim/mnode/basic1.sim b/tests/script/tsim/mnode/basic1.sim index e922ebe37605d64d637e63aa176b53af93b06921..d93d4ca53fc2b6c56e0a8eccf7d3ac3ea74ec0ee 100644 --- a/tests/script/tsim/mnode/basic1.sim +++ b/tests/script/tsim/mnode/basic1.sim @@ -15,7 +15,7 @@ if $data00 != 1 then return -1 endi -if $data02 != LEADER then +if $data02 != leader then return -1 endi @@ -59,13 +59,13 @@ endi if $data(1)[0] != 1 then return -1 endi -if $data(1)[2] != LEADER then +if $data(1)[2] != leader then return -1 endi if $data(2)[0] != 2 then return -1 endi -if $data(2)[2] != FOLLOWER then +if $data(2)[2] != follower then goto step2 endi @@ -95,7 +95,7 @@ endi if $data(1)[0] != 1 then return -1 endi -if $data(1)[2] != LEADER then +if $data(1)[2] != leader then return -1 endi if $data(2)[0] != null then @@ -131,13 +131,13 @@ endi if $data(1)[0] != 1 then return -1 endi -if $data(1)[2] != LEADER then +if $data(1)[2] != leader then return -1 endi if $data(2)[0] != 2 then return -1 endi -if $data(2)[2] != FOLLOWER then +if $data(2)[2] != follower then goto step3 endi diff --git a/tests/script/tsim/mnode/basic2.sim b/tests/script/tsim/mnode/basic2.sim index 18aa85cf5bb00a579b8ed7be14b264845a37948a..78558263d619ee3e9cef2e03c51790823c95b6a9 100644 --- a/tests/script/tsim/mnode/basic2.sim +++ b/tests/script/tsim/mnode/basic2.sim @@ -15,7 +15,7 @@ if $data00 != 1 then return -1 endi -if $data02 != LEADER then +if $data02 != leader then return -1 endi @@ -56,13 +56,13 @@ endi if $data(1)[0] != 1 then return -1 endi -if $data(1)[2] != LEADER then +if $data(1)[2] != leader then return -1 endi if $data(2)[0] != 2 then return -1 endi -if $data(2)[2] != FOLLOWER then +if $data(2)[2] != follower then goto step2 endi diff --git a/tests/script/tsim/mnode/basic3.sim b/tests/script/tsim/mnode/basic3.sim index edbf4290750605d2f5d68f4c67320ca30d8bc42a..bc70cd7a85522230a54359b8a4144eb4ce7a4eed 100644 --- a/tests/script/tsim/mnode/basic3.sim +++ b/tests/script/tsim/mnode/basic3.sim @@ -49,13 +49,13 @@ step2: return -1 endi sql show mnodes -x step2 -if $data(1)[2] != LEADER then +if $data(1)[2] != leader then goto step2 endi -if $data(2)[2] != FOLLOWER then +if $data(2)[2] != follower then goto step2 endi -if $data(3)[2] != FOLLOWER then +if $data(3)[2] != follower then goto step2 endi @@ -114,7 +114,7 @@ print $data(1)[0] $data(1)[1] $data(1)[2] print $data(2)[0] $data(2)[1] $data(2)[2] print $data(3)[0] $data(3)[1] $data(3)[2] -if $data(2)[2] != OFFLINE then +if $data(2)[2] != offline then goto step5 endi diff --git a/tests/script/tsim/mnode/basic4.sim b/tests/script/tsim/mnode/basic4.sim index 11a94dbc557670ff8015245fee5ca739caccb60e..88deb5af898fde58d94f5129fb4e2a030795f29b 100644 --- a/tests/script/tsim/mnode/basic4.sim +++ b/tests/script/tsim/mnode/basic4.sim @@ -63,22 +63,22 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] -if $data(1)[2] != LEADER then +if $data(1)[2] != leader then goto step3 endi -if $data(2)[2] != FOLLOWER then +if $data(2)[2] != follower then goto step3 endi -if $data(3)[2] != OFFLINE then +if $data(3)[2] != offline then goto step3 endi -if $data(1)[3] != READY then +if $data(1)[3] != ready then goto step3 endi -if $data(2)[3] != READY then +if $data(2)[3] != ready then goto step3 endi -if $data(3)[3] != CREATING then +if $data(3)[3] != creating then goto step3 endi @@ -97,22 +97,22 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] -if $data(1)[2] != LEADER then +if $data(1)[2] != leader then goto step4 endi -if $data(2)[2] != FOLLOWER then +if $data(2)[2] != follower then goto step4 endi -if $data(3)[2] != FOLLOWER then +if $data(3)[2] != follower then goto step4 endi -if $data(1)[3] != READY then +if $data(1)[3] != ready then goto step4 endi -if $data(2)[3] != READY then +if $data(2)[3] != ready then goto step4 endi -if $data(3)[3] != READY then +if $data(3)[3] != ready then goto step4 endi @@ -132,22 +132,22 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] -if $data(1)[2] != LEADER then +if $data(1)[2] != leader then goto step5 endi -if $data(2)[2] != FOLLOWER then +if $data(2)[2] != follower then goto step5 endi -if $data(3)[2] != OFFLINE then +if $data(3)[2] != offline then goto step5 endi -if $data(1)[3] != READY then +if $data(1)[3] != ready then goto step5 endi -if $data(2)[3] != READY then +if $data(2)[3] != ready then goto step5 endi -if $data(3)[3] != DROPPING then +if $data(3)[3] != dropping then goto step5 endi @@ -169,19 +169,19 @@ print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] if $rows != 2 then goto step6 endi -if $data(1)[2] != LEADER then +if $data(1)[2] != leader then goto step6 endi -if $data(2)[2] != FOLLOWER then +if $data(2)[2] != follower then goto step6 endi if $data(3)[2] != null then goto step6 endi -if $data(1)[3] != READY then +if $data(1)[3] != ready then goto step6 endi -if $data(2)[3] != READY then +if $data(2)[3] != ready then goto step6 endi if $data(3)[3] != null then diff --git a/tests/script/tsim/qnode/basic1.sim b/tests/script/tsim/qnode/basic1.sim index 2351403909e9f641e2ada2789561a095a0e915d4..7108fcaf59ec420a8657dd8e061e5261ec15ce3c 100644 --- a/tests/script/tsim/qnode/basic1.sim +++ b/tests/script/tsim/qnode/basic1.sim @@ -24,7 +24,7 @@ if $data00 != 1 then return -1 endi -if $data02 != LEADER then +if $data02 != leader then return -1 endi @@ -71,7 +71,7 @@ if $data00 != 1 then return -1 endi -if $data02 != LEADER then +if $data02 != leader then return -1 endi diff --git a/tests/script/tsim/query/explain.sim b/tests/script/tsim/query/explain.sim index 21162a99b0928040ae115b13c117864a170ef4e9..2b0d52d25327833221ffe53953f904d74ed1784a 100644 --- a/tests/script/tsim/query/explain.sim +++ b/tests/script/tsim/query/explain.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -print ========= start dnode1 as LEADER +print ========= start dnode1 as leader system sh/exec.sh -n dnode1 -s start sql connect diff --git a/tests/script/tsim/query/scalarNull.sim b/tests/script/tsim/query/scalarNull.sim index c7e7aa9a349a98623033c40cebfa9c499ee0fe2a..b08ac1d3d9abe157915ec25c438d82e2774ced04 100644 --- a/tests/script/tsim/query/scalarNull.sim +++ b/tests/script/tsim/query/scalarNull.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 2 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 -print ========= start dnode1 as LEADER +print ========= start dnode1 as leader system sh/exec.sh -n dnode1 -s start sleep 2000 sql connect diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index 24ddcc1b75c70865f334f553a6b0f1ee176d62ca..93cae4e3912cab0b5c36e60d28743f0c10f1e45a 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c wallevel -v 2 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode1 -c udf -v 1 -print ========= start dnode1 as LEADER +print ========= start dnode1 as leader system sh/exec.sh -n dnode1 -s start sleep 1000 sql connect diff --git a/tests/script/tsim/snode/basic1.sim b/tests/script/tsim/snode/basic1.sim index 660951c591bb9048b592e7be60492925b13b600d..a9d4867354e70a867d23e65ed03dda47b0b2524c 100644 --- a/tests/script/tsim/snode/basic1.sim +++ b/tests/script/tsim/snode/basic1.sim @@ -24,7 +24,7 @@ if $data00 != 1 then return -1 endi -if $data02 != LEADER then +if $data02 != leader then return -1 endi @@ -71,7 +71,7 @@ if $data00 != 1 then return -1 endi -if $data02 != LEADER then +if $data02 != leader then return -1 endi diff --git a/tests/script/tsim/stream/basic0.sim b/tests/script/tsim/stream/basic0.sim index 9edad991dc0ac5c5c960be026c1fd17073d17881..29775a5ef1d1daf90122f053da6c153bac843341 100644 --- a/tests/script/tsim/stream/basic0.sim +++ b/tests/script/tsim/stream/basic0.sim @@ -63,7 +63,8 @@ if $data02 != 234 then return -1 endi -if $data03 != 234 then +if $data03 != 234 then + print expect 234, actual $data03 return -1 endi diff --git a/tests/script/tsim/stream/session0.sim b/tests/script/tsim/stream/session0.sim index 41a8b3371002848dd6909ab1c681bde0628e6324..a5cd73d17e02d09f17ec468c8601ece235e2b544 100644 --- a/tests/script/tsim/stream/session0.sim +++ b/tests/script/tsim/stream/session0.sim @@ -70,42 +70,42 @@ endi # row 1 if $data11 != 3 then - print ======$data01 + print ======$data11 return -1 endi if $data12 != 10 then - print ======$data02 + print ======$data12 return -1 endi if $data13 != 10 then - print ======$data03 + print ======$data13 return -1 endi if $data14 != 1.100000000 then - print ======$data04 + print ======$data14 return -1 endi if $data15 != 0.000000000 then - print ======$data05 + print ======$data15 return -1 endi if $data16 != 10 then - print ======$data05 + print ======$data15 return -1 endi if $data17 != 1.100000000 then - print ======$data05 + print ======$data17 return -1 endi if $data18 != 5 then - print ======$data05 + print ======$data18 return -1 endi @@ -145,17 +145,17 @@ if $data05 != 0.816496581 then endi if $data06 != 3 then - print ======$data05 + print ======$data06 return -1 endi if $data07 != 1.100000000 then - print ======$data05 + print ======$data07 return -1 endi if $data08 != 13 then - print ======$data05 + print ======$data08 return -1 endi diff --git a/tests/script/tsim/stream/session1.sim b/tests/script/tsim/stream/session1.sim index fb31818f98138948ca91758e14de85146b9940d5..347eda9bf6fc8c6a20e3e94184b78ee1309df5f3 100644 --- a/tests/script/tsim/stream/session1.sim +++ b/tests/script/tsim/stream/session1.sim @@ -187,4 +187,14 @@ if $data34 != 19 then return -1 endi +sql insert into t1 values(1648791000000,1,1,1,1.1,23); +sleep 300 +sql select * from streamt order by s desc; + +# row 0 +if $data01 != 1 then + print ======$data01 + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim new file mode 100644 index 0000000000000000000000000000000000000000..3529f836f420d35598309c238cea3bfdf7ea32cc --- /dev/null +++ b/tests/script/tsim/stream/state0.sim @@ -0,0 +1,477 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database test vgroups 1 +sql show databases +if $rows != 3 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use test + +sql create table t1(ts timestamp, a int, b int , c int, d double, id int); +sql create stream streams1 trigger at_once into streamt1 as select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(a) c4, min(c) c5, max(id) c from t1 state_window(a); + +sql insert into t1 values(1648791213000,1,2,3,1.0,1); +sql insert into t1 values(1648791213000,1,2,3,1.0,2); +sql select * from streamt1 order by c desc; +sleep 300 + +if $rows != 1 then + print ======$rows + return -1; +endi + +sql insert into t1 values(1648791214000,1,2,3,1.0,3); +sql select * from streamt1 order by c desc; +sleep 300 + +if $rows != 1 then + print ======$rows + return -1; +endi + +sql insert into t1 values(1648791213010,2,2,3,1.0,4); +sql insert into t1 values(1648791213000,1,2,3,1.0,5); +sql insert into t1 values(1648791214000,1,2,3,1.0,6); +$loop_count = 0 +loop1: +sql select * from streamt1 where c >=4 order by `_wstartts`; +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 3 then + print ======$rows + goto loop1 + return -1 +endi + +# row 0 +if $data01 != 1 then + print ======$data01 + return -1 +endi + +if $data02 != 1 then + print ======$data02 + return -1 +endi + +if $data03 != 1 then + print ======$data03 + return -1 +endi + +if $data04 != 1 then + print ======$data04 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + return -1 +endi + +if $data06 != 5 then + print ======$data06 + return -1 +endi + +# row 1 +if $data11 != 1 then + print ======$data11 + return -1 +endi + +if $data12 != 1 then + print ======$data12 + return -1 +endi + +if $data13 != 2 then + print ======$data13 + return -1 +endi + +if $data14 != 2 then + print ======$data14 + return -1 +endi + +if $data15 != 3 then + print ======$data15 + return -1 +endi + +if $data16 != 4 then + print ======$data16 + return -1 +endi + +# row 2 +if $data21 != 1 then + print ======$data21 + return -1 +endi + +if $data22 != 1 then + print ======$data22 + return -1 +endi + +if $data23 != 1 then + print ======$data23 + return -1 +endi + +if $data24 != 1 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +if $data26 != 6 then + print ======$data26 + return -1 +endi + +sql insert into t1 values(1648791213011,1,2,3,1.0,7); + +loop2: +$loop_count = 0 +sql select * from streamt1 where c in (5,4,7) order by `_wstartts`; +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 2 +if $data21 != 2 then + print ======$data21 + goto loop2 + return -1 +endi + +if $data22 != 2 then + print ======$data22 + goto loop2 + return -1 +endi + +if $data23 != 2 then + print ======$data23 + goto loop2 + return -1 +endi + +if $data24 != 1 then + print ======$data24 + goto loop2 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + goto loop2 + return -1 +endi + +if $data26 != 7 then + print ======$data26 + goto loop2 + return -1 +endi + +sql insert into t1 values(1648791213011,1,2,3,1.0,8); + +loop21: +$loop_count = 0 +sql select * from streamt1 where c in (5,4,8) order by `_wstartts`; +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data26 != 8 then + print ======$data26 + goto loop21 + return -1 +endi + + +sql insert into t1 values(1648791213020,1,2,3,1.0,9); +sql insert into t1 values(1648791213020,3,2,3,1.0,10); +sql insert into t1 values(1648791214000,1,2,3,1.0,11); +sql insert into t1 values(1648791213011,10,20,10,10.0,12); + +loop3: +$loop_count = 0 +sql select * from streamt1 where c in (5,4,10,11,12) order by `_wstartts`; +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 2 +if $data21 != 1 then + print ======$data21 + goto loop3 + return -1 +endi + +if $data22 != 1 then + print ======$data22 + goto loop3 + return -1 +endi + +if $data23 != 10 then + print ======$data23 + goto loop3 + return -1 +endi + +if $data24 != 10 then + print ======$data24 + goto loop3 + return -1 +endi + +if $data25 != 10 then + print ======$data25 + goto loop3 + return -1 +endi + +if $data26 != 12 then + print ======$data26 + goto loop3 + return -1 +endi + +# row 3 +if $data31 != 1 then + print ======$data31 + goto loop3 + return -1 +endi + +if $data32 != 1 then + print ======$data32 + goto loop3 + return -1 +endi + +if $data33 != 3 then + print ======$data33 + goto loop3 + return -1 +endi + +if $data34 != 3 then + print ======$data34 + goto loop3 + return -1 +endi + +if $data35 != 3 then + print ======$data35 + goto loop3 + return -1 +endi + +if $data36 != 10 then + print ======$data36 + goto loop3 + return -1 +endi + +# row 4 +if $data41 != 1 then + print ======$data41 + goto loop3 + return -1 +endi + +if $data42 != 1 then + print ======$data42 + goto loop3 + return -1 +endi + +if $data43 != 1 then + print ======$data43 + goto loop3 + return -1 +endi + +if $data44 != 1 then + print ======$data44 + goto loop3 + return -1 +endi + +if $data45 != 3 then + print ======$data45 + goto loop3 + return -1 +endi + +if $data46 != 11 then + print ======$data46 + goto loop3 + return -1 +endi + +sql insert into t1 values(1648791213030,3,12,12,12.0,13); +sql insert into t1 values(1648791214040,1,13,13,13.0,14); +sql insert into t1 values(1648791213030,3,14,14,14.0,15) (1648791214020,15,15,15,15.0,16); + +loop4: +$loop_count = 0 +sql select * from streamt1 where c in (14,15,16) order by `_wstartts`; +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 3 then + print ======$rows + goto loop4 + return -1; +endi + +# row 0 +if $data01 != 2 then + print ======$data01 + goto loop4 + return -1 +endi + +if $data02 != 2 then + print ======$data02 + goto loop4 + return -1 +endi + +if $data03 != 6 then + print ======$data03 + goto loop4 + return -1 +endi + +if $data04 != 3 then + print ======$data04 + goto loop4 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + goto loop4 + return -1 +endi + +if $data06 != 15 then + print ======$data06 + goto loop4 + return -1 +endi + +# row 1 +if $data11 != 1 then + print ======$data11 + goto loop4 + return -1 +endi + +if $data12 != 1 then + print ======$data12 + goto loop4 + return -1 +endi + +if $data13 != 15 then + print ======$data13 + goto loop4 + return -1 +endi + +if $data14 != 15 then + print ======$data14 + goto loop4 + return -1 +endi + +if $data15 != 15 then + print ======$data15 + goto loop4 + return -1 +endi + +if $data16 != 16 then + print ======$data16 + goto loop4 + return -1 +endi + +# row 2 +if $data21 != 1 then + print ======$data21 + goto loop4 + return -1 +endi + +if $data22 != 1 then + print ======$data22 + goto loop4 + return -1 +endi + +if $data23 != 1 then + print ======$data23 + goto loop4 + return -1 +endi + +if $data24 != 1 then + print ======$data24 + goto loop4 + return -1 +endi + +if $data25 != 13 then + print ======$data25 + goto loop4 + return -1 +endi + +if $data26 != 14 then + print ======$data26 + goto loop4 + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/triggerSession0.sim b/tests/script/tsim/stream/triggerSession0.sim index fb0666fdcfe847dd25a3e4eb3b66acd16ed09f63..69b3dc170459682a1112297a08c4a7a4a5fd5970 100644 --- a/tests/script/tsim/stream/triggerSession0.sim +++ b/tests/script/tsim/stream/triggerSession0.sim @@ -102,4 +102,4 @@ if $data21 != 1 then return -1 endi -#system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/sync/3Replica1VgElect.sim b/tests/script/tsim/sync/3Replica1VgElect.sim index 61b3b09288faecf857c5d33e7a34ac3544c4db67..e531fa82ad3d78d45447b583834c5b8842c171d1 100644 --- a/tests/script/tsim/sync/3Replica1VgElect.sim +++ b/tests/script/tsim/sync/3Replica1VgElect.sim @@ -105,21 +105,21 @@ if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then - if $data[0][6] == FOLLOWER then - if $data[0][8] == FOLLOWER then +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][8] == FOLLOWER then +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][6] == FOLLOWER then +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi @@ -239,21 +239,21 @@ if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then - if $data[0][6] == FOLLOWER then - if $data[0][8] == FOLLOWER then +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][8] == FOLLOWER then +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][6] == FOLLOWER then +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi @@ -447,21 +447,21 @@ if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then - if $data[0][6] == FOLLOWER then - if $data[0][8] == FOLLOWER then +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][8] == FOLLOWER then +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][6] == FOLLOWER then +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi diff --git a/tests/script/tsim/sync/3Replica5VgElect.sim b/tests/script/tsim/sync/3Replica5VgElect.sim index 4041263e55fa06b93ecea9930ba1dbd728579ce7..1243cf81242fe1c8c8887cd9c89b1b2b48ea9033 100644 --- a/tests/script/tsim/sync/3Replica5VgElect.sim +++ b/tests/script/tsim/sync/3Replica5VgElect.sim @@ -108,21 +108,21 @@ if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then - if $data[0][6] == FOLLOWER then - if $data[0][8] == FOLLOWER then +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][8] == FOLLOWER then +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][6] == FOLLOWER then +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi @@ -130,21 +130,21 @@ else goto check_vg_ready endi -if $data[1][4] == LEADER then - if $data[1][6] == FOLLOWER then - if $data[1][8] == FOLLOWER then +if $data[1][4] == leader then + if $data[1][6] == follower then + if $data[1][8] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] endi endi -elif $data[1][6] == LEADER then - if $data[1][4] == FOLLOWER then - if $data[1][8] == FOLLOWER then +elif $data[1][6] == leader then + if $data[1][4] == follower then + if $data[1][8] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] endi endi -elif $data[1][8] == LEADER then - if $data[1][4] == FOLLOWER then - if $data[1][6] == FOLLOWER then +elif $data[1][8] == leader then + if $data[1][4] == follower then + if $data[1][6] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] endi endi @@ -152,21 +152,21 @@ else goto check_vg_ready endi -if $data[2][4] == LEADER then - if $data[2][6] == FOLLOWER then - if $data[2][8] == FOLLOWER then +if $data[2][4] == leader then + if $data[2][6] == follower then + if $data[2][8] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] endi endi -elif $data[2][6] == LEADER then - if $data[2][4] == FOLLOWER then - if $data[2][8] == FOLLOWER then +elif $data[2][6] == leader then + if $data[2][4] == follower then + if $data[2][8] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] endi endi -elif $data[2][8] == LEADER then - if $data[2][4] == FOLLOWER then - if $data[2][6] == FOLLOWER then +elif $data[2][8] == leader then + if $data[2][4] == follower then + if $data[2][6] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] endi endi @@ -174,21 +174,21 @@ else goto check_vg_ready endi -if $data[3][4] == LEADER then - if $data[3][6] == FOLLOWER then - if $data[3][8] == FOLLOWER then +if $data[3][4] == leader then + if $data[3][6] == follower then + if $data[3][8] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] endi endi -elif $data[3][6] == LEADER then - if $data[3][4] == FOLLOWER then - if $data[3][8] == FOLLOWER then +elif $data[3][6] == leader then + if $data[3][4] == follower then + if $data[3][8] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] endi endi -elif $data[3][8] == LEADER then - if $data[3][4] == FOLLOWER then - if $data[3][6] == FOLLOWER then +elif $data[3][8] == leader then + if $data[3][4] == follower then + if $data[3][6] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] endi endi @@ -196,21 +196,21 @@ else goto check_vg_ready endi -if $data[4][4] == LEADER then - if $data[4][6] == FOLLOWER then - if $data[4][8] == FOLLOWER then +if $data[4][4] == leader then + if $data[4][6] == follower then + if $data[4][8] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] endi endi -elif $data[4][6] == LEADER then - if $data[4][4] == FOLLOWER then - if $data[4][8] == FOLLOWER then +elif $data[4][6] == leader then + if $data[4][4] == follower then + if $data[4][8] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] endi endi -elif $data[4][8] == LEADER then - if $data[4][4] == FOLLOWER then - if $data[4][6] == FOLLOWER then +elif $data[4][8] == leader then + if $data[4][4] == follower then + if $data[4][6] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] endi endi @@ -335,21 +335,21 @@ if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then - if $data[0][6] == FOLLOWER then - if $data[0][8] == FOLLOWER then +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][8] == FOLLOWER then +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][6] == FOLLOWER then +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi @@ -357,21 +357,21 @@ else goto check_vg_ready1 endi -if $data[1][4] == LEADER then - if $data[1][6] == FOLLOWER then - if $data[1][8] == FOLLOWER then +if $data[1][4] == leader then + if $data[1][6] == follower then + if $data[1][8] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] endi endi -elif $data[1][6] == LEADER then - if $data[1][4] == FOLLOWER then - if $data[1][8] == FOLLOWER then +elif $data[1][6] == leader then + if $data[1][4] == follower then + if $data[1][8] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] endi endi -elif $data[1][8] == LEADER then - if $data[1][4] == FOLLOWER then - if $data[1][6] == FOLLOWER then +elif $data[1][8] == leader then + if $data[1][4] == follower then + if $data[1][6] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] endi endi @@ -379,21 +379,21 @@ else goto check_vg_ready1 endi -if $data[2][4] == LEADER then - if $data[2][6] == FOLLOWER then - if $data[2][8] == FOLLOWER then +if $data[2][4] == leader then + if $data[2][6] == follower then + if $data[2][8] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] endi endi -elif $data[2][6] == LEADER then - if $data[2][4] == FOLLOWER then - if $data[2][8] == FOLLOWER then +elif $data[2][6] == leader then + if $data[2][4] == follower then + if $data[2][8] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] endi endi -elif $data[2][8] == LEADER then - if $data[2][4] == FOLLOWER then - if $data[2][6] == FOLLOWER then +elif $data[2][8] == leader then + if $data[2][4] == follower then + if $data[2][6] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] endi endi @@ -401,21 +401,21 @@ else goto check_vg_ready1 endi -if $data[3][4] == LEADER then - if $data[3][6] == FOLLOWER then - if $data[3][8] == FOLLOWER then +if $data[3][4] == leader then + if $data[3][6] == follower then + if $data[3][8] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] endi endi -elif $data[3][6] == LEADER then - if $data[3][4] == FOLLOWER then - if $data[3][8] == FOLLOWER then +elif $data[3][6] == leader then + if $data[3][4] == follower then + if $data[3][8] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] endi endi -elif $data[3][8] == LEADER then - if $data[3][4] == FOLLOWER then - if $data[3][6] == FOLLOWER then +elif $data[3][8] == leader then + if $data[3][4] == follower then + if $data[3][6] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] endi endi @@ -423,21 +423,21 @@ else goto check_vg_ready1 endi -if $data[4][4] == LEADER then - if $data[4][6] == FOLLOWER then - if $data[4][8] == FOLLOWER then +if $data[4][4] == leader then + if $data[4][6] == follower then + if $data[4][8] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] endi endi -elif $data[4][6] == LEADER then - if $data[4][4] == FOLLOWER then - if $data[4][8] == FOLLOWER then +elif $data[4][6] == leader then + if $data[4][4] == follower then + if $data[4][8] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] endi endi -elif $data[4][8] == LEADER then - if $data[4][4] == FOLLOWER then - if $data[4][6] == FOLLOWER then +elif $data[4][8] == leader then + if $data[4][4] == follower then + if $data[4][6] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] endi endi @@ -635,21 +635,21 @@ if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then - if $data[0][6] == FOLLOWER then - if $data[0][8] == FOLLOWER then +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][8] == FOLLOWER then +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][6] == FOLLOWER then +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi @@ -657,21 +657,21 @@ else goto check_vg_ready3 endi -if $data[1][4] == LEADER then - if $data[1][6] == FOLLOWER then - if $data[1][8] == FOLLOWER then +if $data[1][4] == leader then + if $data[1][6] == follower then + if $data[1][8] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] endi endi -elif $data[1][6] == LEADER then - if $data[1][4] == FOLLOWER then - if $data[1][8] == FOLLOWER then +elif $data[1][6] == leader then + if $data[1][4] == follower then + if $data[1][8] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] endi endi -elif $data[1][8] == LEADER then - if $data[1][4] == FOLLOWER then - if $data[1][6] == FOLLOWER then +elif $data[1][8] == leader then + if $data[1][4] == follower then + if $data[1][6] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] endi endi @@ -679,21 +679,21 @@ else goto check_vg_ready3 endi -if $data[2][4] == LEADER then - if $data[2][6] == FOLLOWER then - if $data[2][8] == FOLLOWER then +if $data[2][4] == leader then + if $data[2][6] == follower then + if $data[2][8] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] endi endi -elif $data[2][6] == LEADER then - if $data[2][4] == FOLLOWER then - if $data[2][8] == FOLLOWER then +elif $data[2][6] == leader then + if $data[2][4] == follower then + if $data[2][8] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] endi endi -elif $data[2][8] == LEADER then - if $data[2][4] == FOLLOWER then - if $data[2][6] == FOLLOWER then +elif $data[2][8] == leader then + if $data[2][4] == follower then + if $data[2][6] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] endi endi @@ -701,21 +701,21 @@ else goto check_vg_ready3 endi -if $data[3][4] == LEADER then - if $data[3][6] == FOLLOWER then - if $data[3][8] == FOLLOWER then +if $data[3][4] == leader then + if $data[3][6] == follower then + if $data[3][8] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] endi endi -elif $data[3][6] == LEADER then - if $data[3][4] == FOLLOWER then - if $data[3][8] == FOLLOWER then +elif $data[3][6] == leader then + if $data[3][4] == follower then + if $data[3][8] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] endi endi -elif $data[3][8] == LEADER then - if $data[3][4] == FOLLOWER then - if $data[3][6] == FOLLOWER then +elif $data[3][8] == leader then + if $data[3][4] == follower then + if $data[3][6] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] endi endi @@ -723,21 +723,21 @@ else goto check_vg_ready3 endi -if $data[4][4] == LEADER then - if $data[4][6] == FOLLOWER then - if $data[4][8] == FOLLOWER then +if $data[4][4] == leader then + if $data[4][6] == follower then + if $data[4][8] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] endi endi -elif $data[4][6] == LEADER then - if $data[4][4] == FOLLOWER then - if $data[4][8] == FOLLOWER then +elif $data[4][6] == leader then + if $data[4][4] == follower then + if $data[4][8] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] endi endi -elif $data[4][8] == LEADER then - if $data[4][4] == FOLLOWER then - if $data[4][6] == FOLLOWER then +elif $data[4][8] == leader then + if $data[4][4] == follower then + if $data[4][6] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] endi endi @@ -751,5 +751,6 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT diff --git a/tests/script/tsim/sync/3Replica5VgElect3mnode.sim b/tests/script/tsim/sync/3Replica5VgElect3mnode.sim new file mode 100644 index 0000000000000000000000000000000000000000..8df2c2007da9fdb9e940d9a26995fe49dc4da22b --- /dev/null +++ b/tests/script/tsim/sync/3Replica5VgElect3mnode.sim @@ -0,0 +1,906 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +### create clusters using four dnodes; + + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> 1-dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +if $data[0][0] != 1 then + return -1 +endi +if $data[0][4] != ready then + goto check_dnode_ready +endi + +sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$loop_cnt = 0 +check_dnode_ready_1: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> dnodes not ready! + return -1 +endi +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +if $data[0][4] != ready then + goto check_dnode_ready_1 +endi +if $data[1][4] != ready then + goto check_dnode_ready_1 +endi +if $data[2][4] != ready then + goto check_dnode_ready_1 +endi +if $data[3][4] != ready then + goto check_dnode_ready_1 +endi + +$replica = 3 +$vgroups = 5 + +print ============= create database +sql create database db replica $replica vgroups $vgroups + +$loop_cnt = 0 +check_db_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 100 then + print ====> db not ready! + return -1 +endi +sql show databases +print ===> rows: $rows +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][6] $data[2][11] $data[2][12] $data[2][13] $data[2][14] $data[2][15] $data[2][16] $data[2][17] $data[2][18] $data[2][19] +if $rows != 3 then + return -1 +endi +if $data[2][19] != ready then + goto check_db_ready +endi + +sql use db + +$loop_cnt = 0 +check_vg_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 300 then + print ====> vgroups not ready! + return -1 +endi + +sql show vgroups +print ===> rows: $rows +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[0][10] $data[0][11] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] $data[1][7] $data[1][8] $data[1][9] $data[1][10] $data[1][11] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][10] $data[2][11] +print $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] $data[3][7] $data[3][8] $data[3][9] $data[3][10] $data[3][11] +print $data[4][0] $data[4][1] $data[4][2] $data[4][3] $data[4][4] $data[4][5] $data[4][6] $data[4][7] $data[4][8] $data[4][9] $data[4][10] $data[4][11] +if $rows != $vgroups then + return -1 +endi + +if $data[0][4] == LEADER then + if $data[0][6] == FOLLOWER then + if $data[0][8] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + endi + endi +elif $data[0][6] == LEADER then + if $data[0][4] == FOLLOWER then + if $data[0][8] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + endi + endi +elif $data[0][8] == LEADER then + if $data[0][4] == FOLLOWER then + if $data[0][6] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + endi + endi +else + goto check_vg_ready +endi + +if $data[1][4] == LEADER then + if $data[1][6] == FOLLOWER then + if $data[1][8] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] + endi + endi +elif $data[1][6] == LEADER then + if $data[1][4] == FOLLOWER then + if $data[1][8] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] + endi + endi +elif $data[1][8] == LEADER then + if $data[1][4] == FOLLOWER then + if $data[1][6] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] + endi + endi +else + goto check_vg_ready +endi + +if $data[2][4] == LEADER then + if $data[2][6] == FOLLOWER then + if $data[2][8] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] + endi + endi +elif $data[2][6] == LEADER then + if $data[2][4] == FOLLOWER then + if $data[2][8] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] + endi + endi +elif $data[2][8] == LEADER then + if $data[2][4] == FOLLOWER then + if $data[2][6] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] + endi + endi +else + goto check_vg_ready +endi + +if $data[3][4] == LEADER then + if $data[3][6] == FOLLOWER then + if $data[3][8] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] + endi + endi +elif $data[3][6] == LEADER then + if $data[3][4] == FOLLOWER then + if $data[3][8] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] + endi + endi +elif $data[3][8] == LEADER then + if $data[3][4] == FOLLOWER then + if $data[3][6] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] + endi + endi +else + goto check_vg_ready +endi + +if $data[4][4] == LEADER then + if $data[4][6] == FOLLOWER then + if $data[4][8] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] + endi + endi +elif $data[4][6] == LEADER then + if $data[4][4] == FOLLOWER then + if $data[4][8] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] + endi + endi +elif $data[4][8] == LEADER then + if $data[4][4] == FOLLOWER then + if $data[4][6] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] + endi + endi +else + goto check_vg_ready +endi + +vg_ready: +print ====> create stable/child table +sql create table stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +$ctbPrefix = ctb +$ntbPrefix = ntb +$tbNum = 10 +$i = 0 +while $i < $tbNum + $ctb = $ctbPrefix . $i + sql create table $ctb using stb tags( $i ) + $ntb = $ntbPrefix . $i + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + $i = $i + 1 +endw + +$totalTblNum = $tbNum * 2 +sleep 1000 +sql show tables +print ====> expect $totalTblNum and infinsert $rows in fact +if $rows != $totalTblNum then + return -1 +endi + +sql connect +print ================ insert data +$dbNamme = db +$ctbPrefix = ctb +$ntbPrefix = ntb +$tbNum = 10 +$rowNum = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + + +sql use $dbNamme + +$i = 0 +while $i < $tbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + + $x = 0 + while $x < $rowNum + $binary = ' . binary + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , 999 , 999 , 'binary-ntb' ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw + +print ================ create mnode + + +sql create mnode on dnode 2; +sql create mnode on dnode 3; + + +$loop_cnt = 0 +check_mnode_ready_2: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 100 then + print ====> first create three mnode not ready! + return -1 + endi +sql show mnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] +if $data[0][0] != 1 then + return -1 +endi + +if $data[0][2] != LEADER then + goto check_mnode_ready_2 +endi +if $data[1][2] != FOLLOWER then + goto check_mnode_ready_2 +endi +if $data[2][2] != FOLLOWER then + goto check_mnode_ready_2 +endi + + +start_switch_leader: + +$switch_loop_cnt = 0 +sql show vgroups +$dnodeId = $data[0][3] +$dnodeId = dnode . $dnodeId + +switch_leader_to_offine_loop: + +print $dnodeId +print ====> stop $dnodeId +system sh/exec.sh -n $dnodeId -s stop -x SIGINT + + +$loop_cnt = 0 +$loop_cnt = $loop_cnt + 1 +sleep 201 +if $loop_cnt == 300 then + print ====> vgroups switch fail!!! + return -1 +endi +sql show vgroups +print ===> rows: $rows +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[0][10] $data[0][11] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] $data[1][7] $data[1][8] $data[1][9] $data[1][10] $data[1][11] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][10] $data[2][11] +print $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] $data[3][7] $data[3][8] $data[3][9] $data[3][10] $data[3][11] +print $data[4][0] $data[4][1] $data[4][2] $data[4][3] $data[4][4] $data[4][5] $data[4][6] $data[4][7] $data[4][8] $data[4][9] $data[4][10] $data[4][11] +if $rows != $vgroups then + return -1 +endi + + +vg_offline_1: + +print ====> start $dnodeId +system sh/exec.sh -n $dnodeId -s start + +$switch_loop_cnt = $switch_loop_cnt + 1 +print $switch_loop_cnt + +if $switch_loop_cnt == 1 then + sql show vgroups + $dnodeId = $data[1][3] + $dnodeId = dnode . $dnodeId + goto switch_leader_to_offine_loop +elif $switch_loop_cnt == 2 then + sql show vgroups + $dnodeId = $data[2][3] + $dnodeId = dnode . $dnodeId + goto switch_leader_to_offine_loop +elif $switch_loop_cnt == 3 then + sql show vgroups + $dnodeId = $data[3][3] + $dnodeId = dnode . $dnodeId + goto switch_leader_to_offine_loop +elif $switch_loop_cnt == 4 then + sql show vgroups + $dnodeId = $data[4][3] + $dnodeId = dnode . $dnodeId + goto switch_leader_to_offine_loop +else + goto stop_leader_to_offine_loop +endi + +stop_leader_to_offine_loop: + +$loop_cnt = 0 +check_vg_ready1: +$loop_cnt = $loop_cnt + 1 +print $loop_cnt +sleep 202 +if $loop_cnt == 300 then + print ====> vgroups not ready! + return -1 +endi + +sql show vgroups +print ===> rows: $rows +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[0][10] $data[0][11] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] $data[1][7] $data[1][8] $data[1][9] $data[1][10] $data[1][11] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][10] $data[2][11] +print $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] $data[3][7] $data[3][8] $data[3][9] $data[3][10] $data[3][11] +print $data[4][0] $data[4][1] $data[4][2] $data[4][3] $data[4][4] $data[4][5] $data[4][6] $data[4][7] $data[4][8] $data[4][9] $data[4][10] $data[4][11] +if $rows != $vgroups then + return -1 +endi + +if $data[0][4] == LEADER then + if $data[0][6] == FOLLOWER then + if $data[0][8] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + endi + endi +elif $data[0][6] == LEADER then + if $data[0][4] == FOLLOWER then + if $data[0][8] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + endi + endi +elif $data[0][8] == LEADER then + if $data[0][4] == FOLLOWER then + if $data[0][6] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + endi + endi +else + goto check_vg_ready1 +endi + +if $data[1][4] == LEADER then + if $data[1][6] == FOLLOWER then + if $data[1][8] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] + endi + endi +elif $data[1][6] == LEADER then + if $data[1][4] == FOLLOWER then + if $data[1][8] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] + endi + endi +elif $data[1][8] == LEADER then + if $data[1][4] == FOLLOWER then + if $data[1][6] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] + endi + endi +else + goto check_vg_ready1 +endi + +if $data[2][4] == LEADER then + if $data[2][6] == FOLLOWER then + if $data[2][8] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] + endi + endi +elif $data[2][6] == LEADER then + if $data[2][4] == FOLLOWER then + if $data[2][8] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] + endi + endi +elif $data[2][8] == LEADER then + if $data[2][4] == FOLLOWER then + if $data[2][6] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] + endi + endi +else + goto check_vg_ready1 +endi + +if $data[3][4] == LEADER then + if $data[3][6] == FOLLOWER then + if $data[3][8] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] + endi + endi +elif $data[3][6] == LEADER then + if $data[3][4] == FOLLOWER then + if $data[3][8] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] + endi + endi +elif $data[3][8] == LEADER then + if $data[3][4] == FOLLOWER then + if $data[3][6] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] + endi + endi +else + goto check_vg_ready1 +endi + +if $data[4][4] == LEADER then + if $data[4][6] == FOLLOWER then + if $data[4][8] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] + endi + endi +elif $data[4][6] == LEADER then + if $data[4][4] == FOLLOWER then + if $data[4][8] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] + endi + endi +elif $data[4][8] == LEADER then + if $data[4][4] == FOLLOWER then + if $data[4][6] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] + endi + endi +else + goto check_vg_ready1 +endi + + +print ====> final test: create stable/child table +sql create table stb1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + + +sql show stables +if $rows != 2 then + return -1 +endi + +$ctbPrefix = ctb1 +$ntbPrefix = ntb1 +$tbNum = 10 +$i = 0 +while $i < $tbNum + $ctb = $ctbPrefix . $i + sql create table $ctb using stb1 tags( $i ) + $ntb = $ntbPrefix . $i + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + $i = $i + 1 +endw + +sleep 1000 +sql show stables +if $rows != 2 then + return -1 +endi + +sql show tables +if $rows != 40 then + return -1 +endi + + +system sh/deploy.sh -n dnode5 -i 5 +system sh/exec.sh -n dnode5 -s start +system sh/exec.sh -n dnode1 -s stop +sleep 1000 +system sh/exec.sh -n dnode1 -s start + +sql connect +sql create dnode $hostname port 7500 + +$loop_cnt = 0 +check_dnode_ready3: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 100 then + print ====> 5 dnode not ready! + return -1 + endi + +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +print ===> $rows $data[4][0] $data[4][1] $data[4][2] $data[4][3] $data[4][4] $data[4][5] $data[4][6] + +if $rows != 5 then + return -1 +endi + +if $data[4][4] != ready then + goto check_dnode_ready3 +endi + +$loop_cnt = 0 +check_mnode_ready_3: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 100 then + print ====> second mnode not ready! + return -1 + endi +sql show mnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] +if $data[0][0] != 1 then + return -1 +endi + +if $data[0][2] == LEADER then + if $data[1][2] != FOLLOWER then + goto check_mnode_ready_3 + endi + if $data[2][2] != FOLLOWER then + goto check_mnode_ready_3 + endi +endi +if $data[1][2] == LEADER then + if $data[0][2] != FOLLOWER then + goto check_mnode_ready_3 + endi + if $data[2][2] != FOLLOWER then + goto check_mnode_ready_3 + endi +endi +if $data[2][2] == LEADER then + if $data[1][2] != FOLLOWER then + goto check_mnode_ready_3 + endi + if $data[0][2] != FOLLOWER then + goto check_mnode_ready_3 + endi +endi + +# restart clusters + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT + + + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +system sh/exec.sh -n dnode5 -s start + + +$loop_cnt = 0 +check_dnode_ready_2: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> restart and dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +print ===> $rows $data[4][0] $data[4][1] $data[4][2] $data[4][3] $data[4][4] $data[4][5] $data[4][6] + +if $data[0][0] != 1 then + return -1 +endi + +if $data[0][4] != ready then + goto check_dnode_ready_2 +endi +if $data[1][4] != ready then + goto check_dnode_ready_2 +endi +if $data[2][4] != ready then + goto check_dnode_ready_2 +endi +if $data[3][4] != ready then + goto check_dnode_ready_2 +endi + +sql use db; +$ctbPrefix = ctb2 +$ntbPrefix = ntb2 +$tbNum = 10 +$i = 0 +while $i < $tbNum + $ctb = $ctbPrefix . $i + sql create table $ctb using stb1 tags( $i ) + $ntb = $ntbPrefix . $i + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + $i = $i + 1 +endw + +sleep 1000 +sql use db +sql show stables +if $rows != 2 then + return -1 +endi + +sql show tables +print $rows +if $rows != 60 then + return -1 +endi + + + +$replica = 3 +$vgroups = 5 + +print ============= create database +sql create database db1 replica $replica vgroups $vgroups + +$loop_cnt = 0 +check_db_ready1: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 100 then + print ====> db not ready! + return -1 +endi +sql show databases +print ===> rows: $rows +print $data(db1)[0] $data(db1)[1] $data(db1)[2] $data(db1)[3] $data(db1)[4] $data(db1)[5] $data(db1)[6] $data(db1)[7] $data(db1)[8] $data(db1)[9] $data(db1)[6] $data(db1)[11] $data(db1)[12] $data(db1)[13] $data(db1)[14] $data(db1)[15] $data(db1)[16] $data(db1)[17] $data(db1)[18] $data(db1)[19] +if $rows != 4 then + return -1 +endi +if $data(db1)[19] != ready then + goto check_db_ready1 +endi + + + +sql use db1 + +$loop_cnt = 0 +check_vg_ready3: +$loop_cnt = $loop_cnt + 1 +print $loop_cnt +sleep 202 +if $loop_cnt == 300 then + print ====> vgroups not ready! + return -1 +endi + + +sql show vgroups +print ===> rows: $rows +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[0][10] $data[0][11] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] $data[1][7] $data[1][8] $data[1][9] $data[1][10] $data[1][11] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][10] $data[2][11] +print $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] $data[3][7] $data[3][8] $data[3][9] $data[3][10] $data[3][11] +print $data[4][0] $data[4][1] $data[4][2] $data[4][3] $data[4][4] $data[4][5] $data[4][6] $data[4][7] $data[4][8] $data[4][9] $data[4][10] $data[4][11] +if $rows != $vgroups then + return -1 +endi + +if $data[0][4] == LEADER then + if $data[0][6] == FOLLOWER then + if $data[0][8] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + endi + endi +elif $data[0][6] == LEADER then + if $data[0][4] == FOLLOWER then + if $data[0][8] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + endi + endi +elif $data[0][8] == LEADER then + if $data[0][4] == FOLLOWER then + if $data[0][6] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + endi + endi +else + goto check_vg_ready3 +endi + +if $data[1][4] == LEADER then + if $data[1][6] == FOLLOWER then + if $data[1][8] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] + endi + endi +elif $data[1][6] == LEADER then + if $data[1][4] == FOLLOWER then + if $data[1][8] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] + endi + endi +elif $data[1][8] == LEADER then + if $data[1][4] == FOLLOWER then + if $data[1][6] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] + endi + endi +else + goto check_vg_ready3 +endi + +if $data[2][4] == LEADER then + if $data[2][6] == FOLLOWER then + if $data[2][8] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] + endi + endi +elif $data[2][6] == LEADER then + if $data[2][4] == FOLLOWER then + if $data[2][8] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] + endi + endi +elif $data[2][8] == LEADER then + if $data[2][4] == FOLLOWER then + if $data[2][6] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] + endi + endi +else + goto check_vg_ready3 +endi + +if $data[3][4] == LEADER then + if $data[3][6] == FOLLOWER then + if $data[3][8] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] + endi + endi +elif $data[3][6] == LEADER then + if $data[3][4] == FOLLOWER then + if $data[3][8] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] + endi + endi +elif $data[3][8] == LEADER then + if $data[3][4] == FOLLOWER then + if $data[3][6] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] + endi + endi +else + goto check_vg_ready3 +endi + +if $data[4][4] == LEADER then + if $data[4][6] == FOLLOWER then + if $data[4][8] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] + endi + endi +elif $data[4][6] == LEADER then + if $data[4][4] == FOLLOWER then + if $data[4][8] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] + endi + endi +elif $data[4][8] == LEADER then + if $data[4][4] == FOLLOWER then + if $data[4][6] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] + endi + endi +else + goto check_vg_ready3 +endi + + +$loop_cnt = 0 +check_mnode_ready_3: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 100 then + print ====> third: mnode not ready! + return -1 + endi +sql show mnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] +if $data[0][0] != 1 then + return -1 +endi + +if $data[0][2] == LEADER then + if $data[1][2] != FOLLOWER then + goto check_mnode_ready_3 + endi + if $data[2][2] != FOLLOWER then + goto check_mnode_ready_3 + endi +endi +if $data[1][2] == LEADER then + if $data[0][2] != FOLLOWER then + goto check_mnode_ready_3 + endi + if $data[2][2] != FOLLOWER then + goto check_mnode_ready_3 + endi +endi +if $data[2][2] == LEADER then + if $data[1][2] != FOLLOWER then + goto check_mnode_ready_3 + endi + if $data[0][2] != FOLLOWER then + goto check_mnode_ready_3 + endi +endi + + + diff --git a/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim b/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim new file mode 100644 index 0000000000000000000000000000000000000000..ae02a23c9b47d57239dc89ab71337c27f39e572e --- /dev/null +++ b/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim @@ -0,0 +1,627 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +print ===> create clusters using four dnodes; + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> 1-dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +if $data[0][0] != 1 then + return -1 +endi +if $data[0][4] != ready then + goto check_dnode_ready +endi + +sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 + +$loop_cnt = 0 +check_dnode_ready_1: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 10 then + print ====> dnodes not ready! + return -1 +endi +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +if $data[0][4] != ready then + goto check_dnode_ready_1 +endi +if $data[1][4] != ready then + goto check_dnode_ready_1 +endi +if $data[2][4] != ready then + goto check_dnode_ready_1 +endi +if $data[3][4] != ready then + goto check_dnode_ready_1 +endi + +$replica = 3 +$vgroups = 5 + +print ============= create database +sql create database db replica $replica vgroups $vgroups + +$loop_cnt = 0 +check_db_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 100 then + print ====> db not ready! + return -1 +endi +sql show databases +print ===> rows: $rows +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][6] $data[2][11] $data[2][12] $data[2][13] $data[2][14] $data[2][15] $data[2][16] $data[2][17] $data[2][18] $data[2][19] +if $rows != 3 then + return -1 +endi +if $data[2][19] != ready then + goto check_db_ready +endi + +sql use db + +$loop_cnt = 0 +check_vg_ready: +$loop_cnt = $loop_cnt + 1 +sleep 200 +if $loop_cnt == 300 then + print ====> vgroups not ready! + return -1 +endi + +sql show vgroups +print ===> rows: $rows +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[0][10] $data[0][11] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] $data[1][7] $data[1][8] $data[1][9] $data[1][10] $data[1][11] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][10] $data[2][11] +print $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] $data[3][7] $data[3][8] $data[3][9] $data[3][10] $data[3][11] +print $data[4][0] $data[4][1] $data[4][2] $data[4][3] $data[4][4] $data[4][5] $data[4][6] $data[4][7] $data[4][8] $data[4][9] $data[4][10] $data[4][11] +if $rows != $vgroups then + return -1 +endi + +if $data[0][4] == LEADER then + if $data[0][6] == FOLLOWER then + if $data[0][8] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + endi + endi +elif $data[0][6] == LEADER then + if $data[0][4] == FOLLOWER then + if $data[0][8] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + endi + endi +elif $data[0][8] == LEADER then + if $data[0][4] == FOLLOWER then + if $data[0][6] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + endi + endi +else + goto check_vg_ready +endi + +if $data[1][4] == LEADER then + if $data[1][6] == FOLLOWER then + if $data[1][8] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] + endi + endi +elif $data[1][6] == LEADER then + if $data[1][4] == FOLLOWER then + if $data[1][8] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] + endi + endi +elif $data[1][8] == LEADER then + if $data[1][4] == FOLLOWER then + if $data[1][6] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] + endi + endi +else + goto check_vg_ready +endi + +if $data[2][4] == LEADER then + if $data[2][6] == FOLLOWER then + if $data[2][8] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] + endi + endi +elif $data[2][6] == LEADER then + if $data[2][4] == FOLLOWER then + if $data[2][8] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] + endi + endi +elif $data[2][8] == LEADER then + if $data[2][4] == FOLLOWER then + if $data[2][6] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] + endi + endi +else + goto check_vg_ready +endi + +if $data[3][4] == LEADER then + if $data[3][6] == FOLLOWER then + if $data[3][8] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] + endi + endi +elif $data[3][6] == LEADER then + if $data[3][4] == FOLLOWER then + if $data[3][8] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] + endi + endi +elif $data[3][8] == LEADER then + if $data[3][4] == FOLLOWER then + if $data[3][6] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] + endi + endi +else + goto check_vg_ready +endi + +if $data[4][4] == LEADER then + if $data[4][6] == FOLLOWER then + if $data[4][8] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] + endi + endi +elif $data[4][6] == LEADER then + if $data[4][4] == FOLLOWER then + if $data[4][8] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] + endi + endi +elif $data[4][8] == LEADER then + if $data[4][4] == FOLLOWER then + if $data[4][6] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] + endi + endi +else + goto check_vg_ready +endi + +vg_ready: +print ====> create stable/child table +sql create table stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +$ctbPrefix = ctb +$ntbPrefix = ntb +$tbNum = 10 +$i = 0 +while $i < $tbNum + $ctb = $ctbPrefix . $i + sql create table $ctb using stb tags( $i ) + $ntb = $ntbPrefix . $i + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + $i = $i + 1 +endw + +$totalTblNum = $tbNum * 2 +sleep 1000 +sql show tables +print ====> expect $totalTblNum and infinsert $rows in fact +if $rows != $totalTblNum then + return -1 +endi + +sql connect +print ================ insert data +$dbNamme = db +$ctbPrefix = ctb +$ntbPrefix = ntb +$tbNum = 10 +$rowNum = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + + +sql use $dbNamme + +$i = 0 +while $i < $tbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + + $x = 0 + while $x < $rowNum + $binary = ' . binary + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , 999 , 999 , 'binary-ntb' ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw + +print ================ create mnode + + +sql create mnode on dnode 2; +sql create mnode on dnode 3; + + +$loop_cnt = 0 +check_mnode_ready_2: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 100 then + print ====> first create three mnode not ready! + return -1 + endi +sql show mnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] +if $data[0][0] != 1 then + return -1 +endi + +if $data[0][2] != LEADER then + goto check_mnode_ready_2 +endi +if $data[1][2] != FOLLOWER then + goto check_mnode_ready_2 +endi +if $data[2][2] != FOLLOWER then + goto check_mnode_ready_2 +endi + + +$loop_cnt = 0 +check_vg_ready1: +$loop_cnt = $loop_cnt + 1 +print $loop_cnt +sleep 202 +if $loop_cnt == 300 then + print ====> vgroups not ready! + return -1 +endi + +sql show vgroups +print ===> rows: $rows +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[0][10] $data[0][11] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] $data[1][7] $data[1][8] $data[1][9] $data[1][10] $data[1][11] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] $data[2][7] $data[2][8] $data[2][9] $data[2][10] $data[2][11] +print $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] $data[3][7] $data[3][8] $data[3][9] $data[3][10] $data[3][11] +print $data[4][0] $data[4][1] $data[4][2] $data[4][3] $data[4][4] $data[4][5] $data[4][6] $data[4][7] $data[4][8] $data[4][9] $data[4][10] $data[4][11] +if $rows != $vgroups then + return -1 +endi + +if $data[0][4] == LEADER then + if $data[0][6] == FOLLOWER then + if $data[0][8] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] + endi + endi +elif $data[0][6] == LEADER then + if $data[0][4] == FOLLOWER then + if $data[0][8] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] + endi + endi +elif $data[0][8] == LEADER then + if $data[0][4] == FOLLOWER then + if $data[0][6] == FOLLOWER then + print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] + endi + endi +else + goto check_vg_ready1 +endi + +if $data[1][4] == LEADER then + if $data[1][6] == FOLLOWER then + if $data[1][8] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] + endi + endi +elif $data[1][6] == LEADER then + if $data[1][4] == FOLLOWER then + if $data[1][8] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] + endi + endi +elif $data[1][8] == LEADER then + if $data[1][4] == FOLLOWER then + if $data[1][6] == FOLLOWER then + print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] + endi + endi +else + goto check_vg_ready1 +endi + +if $data[2][4] == LEADER then + if $data[2][6] == FOLLOWER then + if $data[2][8] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] + endi + endi +elif $data[2][6] == LEADER then + if $data[2][4] == FOLLOWER then + if $data[2][8] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] + endi + endi +elif $data[2][8] == LEADER then + if $data[2][4] == FOLLOWER then + if $data[2][6] == FOLLOWER then + print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] + endi + endi +else + goto check_vg_ready1 +endi + +if $data[3][4] == LEADER then + if $data[3][6] == FOLLOWER then + if $data[3][8] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] + endi + endi +elif $data[3][6] == LEADER then + if $data[3][4] == FOLLOWER then + if $data[3][8] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] + endi + endi +elif $data[3][8] == LEADER then + if $data[3][4] == FOLLOWER then + if $data[3][6] == FOLLOWER then + print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] + endi + endi +else + goto check_vg_ready1 +endi + +if $data[4][4] == LEADER then + if $data[4][6] == FOLLOWER then + if $data[4][8] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] + endi + endi +elif $data[4][6] == LEADER then + if $data[4][4] == FOLLOWER then + if $data[4][8] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] + endi + endi +elif $data[4][8] == LEADER then + if $data[4][4] == FOLLOWER then + if $data[4][6] == FOLLOWER then + print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] + endi + endi +else + goto check_vg_ready1 +endi + + +print ====> final test: create stable/child table +sql create table stb1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) + + +sql show stables +if $rows != 2 then + return -1 +endi + +$ctbPrefix = ctb1 +$ntbPrefix = ntb1 +$tbNum = 10 +$i = 0 +while $i < $tbNum + $ctb = $ctbPrefix . $i + sql create table $ctb using stb1 tags( $i ) + $ntb = $ntbPrefix . $i + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) + $i = $i + 1 +endw + +sleep 1000 +sql show stables +if $rows != 2 then + return -1 +endi + +sql show tables +if $rows != 40 then + return -1 +endi + + +system sh/deploy.sh -n dnode5 -i 5 +system sh/exec.sh -n dnode5 -s start + +sql connect +sql create dnode $hostname port 7500 + +$loop_cnt = 0 +check_dnode_ready3: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 100 then + print ====> 5 dnode not ready! + return -1 + endi + +sql show dnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] +print ===> $rows $data[3][0] $data[3][1] $data[3][2] $data[3][3] $data[3][4] $data[3][5] $data[3][6] +print ===> $rows $data[4][0] $data[4][1] $data[4][2] $data[4][3] $data[4][4] $data[4][5] $data[4][6] + +if $rows != 5 then + return -1 +endi + +if $data[4][4] != ready then + goto check_dnode_ready3 +endi + +print ===> 1:if create users sucessfully，then drop mnode leader +sql create user chr pass '123' +$loop_cnt = 0 +check_user_ready: +$loop_cnt = $loop_cnt + 1 +print $loop_cnt +sleep 200 +if $loop_cnt == 300 then + print ====> vgroups not ready! + return -1 +endi + +sql show users +print ===> rows: $rows +print ===> $rows $data[0][0] $data[0][1] $data[0][2] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] +if $rows != 2 then + goto check_user_ready +endi + +if $data[0][0] == chr then + goto check_user_ready_suc +elif $data[1][0] == chr + goto check_user_ready_suc +else + print ====> creating user failed + goto check_user_ready +endi + +check_user_ready_suc: + +$loop_cnt = 0 +check_mnode_ready_3: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 100 then + print ====> second mnode not ready! + return -1 + endi +sql show mnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] +if $data[0][0] != 1 then + return -1 +endi + +if $data[0][2] == LEADER then + if $data[1][2] != FOLLOWER then + goto check_mnode_ready_3 + endi + if $data[2][2] != FOLLOWER then + goto check_mnode_ready_3 + endi +endi +if $data[1][2] == LEADER then + if $data[0][2] != FOLLOWER then + goto check_mnode_ready_3 + endi + if $data[2][2] != FOLLOWER then + goto check_mnode_ready_3 + endi +endi +if $data[2][2] == LEADER then + if $data[1][2] != FOLLOWER then + goto check_mnode_ready_3 + endi + if $data[0][2] != FOLLOWER then + goto check_mnode_ready_3 + endi +endi + +sleep 2000 +# stop leader and drop dnode +system sh/exec.sh -n dnode1 -s stop +sleep 2000 + +print ===> 2:if create users sucessfully，then drop mnode leader +sql create user chr2 pass '123' +$loop_cnt = 0 +check_user_ready2: +$loop_cnt = $loop_cnt + 1 +print $loop_cnt +sleep 200 +if $loop_cnt == 300 then + print ====> vgroups not ready! + return -1 +endi +sql show mnodes +print ===> $rows $data[0][0] $data[0][1] $data[0][2] $data[0][3] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] $data[1][3] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] $data[2][3] + +sql show users +print ===> rows: $rows +print ===> $rows $data[0][0] $data[0][1] $data[0][2] +print ===> $rows $data[1][0] $data[1][1] $data[1][2] +print ===> $rows $data[2][0] $data[2][1] $data[2][2] + +if $rows != 3 then + goto check_user_ready2 +endi + +if $data[0][0] == chr2 then + goto check_user_ready2_suc +elif $data[1][0] == chr2 + goto check_user_ready2_suc +else + print ====> creating user failed + goto check_user_ready2 +endi + +check_user_ready2_suc: + diff --git a/tests/script/tsim/sync/oneReplica1VgElect.sim b/tests/script/tsim/sync/oneReplica1VgElect.sim index d98b823192b82556f0327f5107d2d359176e19cb..cf8912e654c04314e96a5fb4a718a3569ddea5f5 100644 --- a/tests/script/tsim/sync/oneReplica1VgElect.sim +++ b/tests/script/tsim/sync/oneReplica1VgElect.sim @@ -103,13 +103,13 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $d if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then +if $data[0][4] == leader then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] goto vg_ready -elif $data[0][6] == LEADER then +elif $data[0][6] == leader then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] goto vg_ready -elif $data[0][8] == LEADER then +elif $data[0][8] == leader then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] goto vg_ready else @@ -154,11 +154,11 @@ switch_leader_to_offine_loop: print ====> finde vnode of leader, and stop the dnode where the vnode is located, and query stb/ntb count(*) sql show vgroups print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] $data[0][7] $data[0][8] $data[0][9] $data[0][6] $data[0][11] $data[0][12] $data[0][13] -if $data[0][4] == LEADER then +if $data[0][4] == leader then $dnodeId = $data[0][3] -elif $data[0][6] == LEADER then +elif $data[0][6] == leader then $dnodeId = $data[0][5] -elif $data[0][8] == LEADER then +elif $data[0][8] == leader then $dnodeId = $data[0][7] else print ====> no leader vnode!!! @@ -186,13 +186,13 @@ if $rows != $vgroups then return -1 endi -if $data[0][4] == OFFLINE then +if $data[0][4] == offline then print ---- vgroup $dnodeId leader switch to offline goto vg_offline_1 -elif $data[0][6] == OFFLINE then +elif $data[0][6] == offline then print ---- vgroup $dnodeId leader switch to offline goto vg_offline_1 -elif $data[0][8] == OFFLINE then +elif $data[0][8] == offline then print ---- vgroup $dnodeId leader switch to offline goto vg_offline_1 else @@ -218,13 +218,13 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $d if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then +if $data[0][4] == leader then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] goto countinu_loop -elif $data[0][6] == LEADER then +elif $data[0][6] == leader then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] goto countinu_loop -elif $data[0][8] == LEADER then +elif $data[0][8] == leader then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] goto countinu_loop else diff --git a/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim b/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim index 7ceeb2806b320014c2b35ea5c640063e44793063..06a67b3c1bfdf183f919fb7ac9c861055f566f42 100644 --- a/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim +++ b/tests/script/tsim/sync/oneReplica1VgElectWithInsert.sim @@ -104,7 +104,7 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $d if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then +if $data[0][4] == leader then if $data[0][6] != NULL then goto check_vg_ready endi @@ -114,7 +114,7 @@ if $data[0][4] == LEADER then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] goto vg_ready endi -if $data[0][6] == LEADER then +if $data[0][6] == leader then if $data[0][4] != NULL then goto check_vg_ready endi @@ -124,7 +124,7 @@ if $data[0][6] == LEADER then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] goto vg_ready endi -if $data[0][8] == LEADER then +if $data[0][8] == leader then if $data[0][4] != NULL then goto check_vg_ready endi @@ -208,11 +208,11 @@ switch_leader_loop: print ====> finde vnode of leader, and stop the dnode where the vnode is located, and query stb/ntb count(*) sql show vgroups print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] -if $data[0][4] == LEADER then +if $data[0][4] == leader then $dnodeId = $data[0][3] -elif $data[0][6] == LEADER then +elif $data[0][6] == leader then $dnodeId = $data[0][5] -elif $data[0][8] == LEADER then +elif $data[0][8] == leader then $dnodeId = $data[0][7] else print ====> no leader vnode!!! @@ -238,7 +238,7 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $d if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then +if $data[0][4] == leader then if $data[0][6] != NULL then goto check_vg_ready_2 endi @@ -248,7 +248,7 @@ if $data[0][4] == LEADER then print ---- vgroup $data[0][0] leader switch to dnode $data[0][3] goto vg_ready_2 endi -if $data[0][6] == LEADER then +if $data[0][6] == leader then if $data[0][4] != NULL then goto check_vg_ready_2 endi @@ -258,7 +258,7 @@ if $data[0][6] == LEADER then print ---- vgroup $data[0][0] leader switch to dnode $data[0][5] goto vg_ready_2 endi -if $data[0][8] == LEADER then +if $data[0][8] == leader then if $data[0][4] != NULL then goto check_vg_ready_2 endi @@ -343,7 +343,7 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $d if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then +if $data[0][4] == leader then if $data[0][6] != NULL then goto check_vg_ready_1 endi @@ -352,7 +352,7 @@ if $data[0][4] == LEADER then endi goto vg_ready_1 endi -if $data[0][6] == LEADER then +if $data[0][6] == leader then if $data[0][4] != NULL then goto check_vg_ready_1 endi @@ -361,7 +361,7 @@ if $data[0][6] == LEADER then endi goto vg_ready_1 endi -if $data[0][8] == LEADER then +if $data[0][8] == leader then if $data[0][4] != NULL then goto check_vg_ready_1 endi diff --git a/tests/script/tsim/sync/oneReplica5VgElect.sim b/tests/script/tsim/sync/oneReplica5VgElect.sim index d6d18093c341dfea8d4d2f1c22fa10cab173d71c..5af48c7491208c8f3a440665fa7bb6919c373a46 100644 --- a/tests/script/tsim/sync/oneReplica5VgElect.sim +++ b/tests/script/tsim/sync/oneReplica5VgElect.sim @@ -104,51 +104,51 @@ if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then +if $data[0][4] == leader then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] -elif $data[0][6] == LEADER then +elif $data[0][6] == leader then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] -elif $data[0][8] == LEADER then +elif $data[0][8] == leader then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] else goto check_vg_ready endi -if $data[1][4] == LEADER then +if $data[1][4] == leader then print ---- vgroup $data[1][0] leader locate on dnode $data[0][3] -elif $data[1][6] == LEADER then +elif $data[1][6] == leader then print ---- vgroup $data[1][0] leader locate on dnode $data[0][5] -elif $data[1][8] == LEADER then +elif $data[1][8] == leader then print ---- vgroup $data[1][0] leader locate on dnode $data[0][7] else goto check_vg_ready endi -if $data[2][4] == LEADER then +if $data[2][4] == leader then print ---- vgroup $data[2][0] leader locate on dnode $data[0][3] -elif $data[2][6] == LEADER then +elif $data[2][6] == leader then print ---- vgroup $data[2][0] leader locate on dnode $data[0][5] -elif $data[2][8] == LEADER then +elif $data[2][8] == leader then print ---- vgroup $data[2][0] leader locate on dnode $data[0][7] else goto check_vg_ready endi -if $data[3][4] == LEADER then +if $data[3][4] == leader then print ---- vgroup $data[3][0] leader locate on dnode $data[0][3] -elif $data[3][6] == LEADER then +elif $data[3][6] == leader then print ---- vgroup $data[3][0] leader locate on dnode $data[0][5] -elif $data[3][8] == LEADER then +elif $data[3][8] == leader then print ---- vgroup $data[3][0] leader locate on dnode $data[0][7] else goto check_vg_ready endi -if $data[4][4] == LEADER then +if $data[4][4] == leader then print ---- vgroup $data[4][0] leader locate on dnode $data[0][3] -elif $data[4][6] == LEADER then +elif $data[4][6] == leader then print ---- vgroup $data[4][0] leader locate on dnode $data[0][5] -elif $data[4][8] == LEADER then +elif $data[4][8] == leader then print ---- vgroup $data[4][0] leader locate on dnode $data[0][7] else goto check_vg_ready @@ -272,51 +272,51 @@ if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then +if $data[0][4] == leader then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] -elif $data[0][6] == LEADER then +elif $data[0][6] == leader then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] -elif $data[0][8] == LEADER then +elif $data[0][8] == leader then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] else goto check_vg_ready1 endi -if $data[1][4] == LEADER then +if $data[1][4] == leader then print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] -elif $data[1][6] == LEADER then +elif $data[1][6] == leader then print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] -elif $data[1][8] == LEADER then +elif $data[1][8] == leader then print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] else goto check_vg_ready1 endi -if $data[2][4] == LEADER then +if $data[2][4] == leader then print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] -elif $data[2][6] == LEADER then +elif $data[2][6] == leader then print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] -elif $data[2][8] == LEADER then +elif $data[2][8] == leader then print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] else goto check_vg_ready endi -if $data[3][4] == LEADER then +if $data[3][4] == leader then print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] -elif $data[3][6] == LEADER then +elif $data[3][6] == leader then print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] -elif $data[3][8] == LEADER then +elif $data[3][8] == leader then print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] else goto check_vg_ready1 endi -if $data[4][4] == LEADER then +if $data[4][4] == leader then print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] -elif $data[4][6] == LEADER then +elif $data[4][6] == leader then print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] -elif $data[4][8] == LEADER then +elif $data[4][8] == leader then print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] else goto check_vg_ready1 diff --git a/tests/script/tsim/sync/threeReplica1VgElect.sim b/tests/script/tsim/sync/threeReplica1VgElect.sim index 1496d7c778b475895479eb3661fae7ad86a121d3..c3e9c13793466ecdd57890d7e48a71f5b04ca190 100644 --- a/tests/script/tsim/sync/threeReplica1VgElect.sim +++ b/tests/script/tsim/sync/threeReplica1VgElect.sim @@ -104,7 +104,7 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $d if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then +if $data[0][4] == leader then if $data[0][6] != FLLOWER then goto check_vg_ready endi @@ -114,7 +114,7 @@ if $data[0][4] == LEADER then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] goto vg_ready endi -if $data[0][6] == LEADER then +if $data[0][6] == leader then if $data[0][4] != FLLOWER then goto check_vg_ready endi @@ -124,7 +124,7 @@ if $data[0][6] == LEADER then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] goto vg_ready endi -if $data[0][8] == LEADER then +if $data[0][8] == leader then if $data[0][4] != FLLOWER then goto check_vg_ready endi @@ -199,11 +199,11 @@ endi print ====> finde vnode of leader, and stop the dnode where the vnode is located, and query stb/ntb count(*) sql show vgroups print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] -if $data[0][4] == LEADER then +if $data[0][4] == leader then $dnodeId = $data[0][3] -elif $data[0][6] == LEADER then +elif $data[0][6] == leader then $dnodeId = $data[0][5] -elif $data[0][8] == LEADER then +elif $data[0][8] == leader then $dnodeId = $data[0][7] else print ====> no leader vnode!!! @@ -216,11 +216,11 @@ system sh/exec.sh -n $dnodeId -s stop -x SIGINT sql show vgroups print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] -if $data[0][4] == LEADER then +if $data[0][4] == leader then print ---- vgroup $data[0][0] leader switch to dnode $data[0][3] -elif $data[0][6] == LEADER then +elif $data[0][6] == leader then print ---- vgroup $data[0][0] leader switch to dnode $data[0][5] -elif $data[0][8] == LEADER then +elif $data[0][8] == leader then print ---- vgroup $data[0][0] leader switch to dnode $data[0][7] else print ====> no leader vnode!!! @@ -264,7 +264,7 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $d if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then +if $data[0][4] == leader then if $data[0][6] != FLLOWER then goto check_vg_ready_1 endi @@ -273,7 +273,7 @@ if $data[0][4] == LEADER then endi goto vg_ready_1 endi -if $data[0][6] == LEADER then +if $data[0][6] == leader then if $data[0][4] != FLLOWER then goto check_vg_ready_1 endi @@ -282,7 +282,7 @@ if $data[0][6] == LEADER then endi goto vg_ready_1 endi -if $data[0][8] == LEADER then +if $data[0][8] == leader then if $data[0][4] != FLLOWER then goto check_vg_ready_1 endi @@ -325,27 +325,27 @@ system sh/exec.sh -n $dnodeId -s stop -x SIGINT check_vg_ready_3: sql show vgroups print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] -if $data[0][4] == LEADER then - if $data[0][6] == LEADER then +if $data[0][4] == leader then + if $data[0][6] == leader then goto check_vg_ready_3 endi - if $data[0][8] == LEADER then + if $data[0][8] == leader then goto check_vg_ready_3 endi print ---- vgroup $data[0][0] leader locating dnode $data[0][5] -elif $data[0][6] == LEADER then - if $data[0][4] == LEADER then +elif $data[0][6] == leader then + if $data[0][4] == leader then goto check_vg_ready_3 endi - if $data[0][8] == LEADER then + if $data[0][8] == leader then goto check_vg_ready_3 endi print ---- vgroup $data[0][0] leader locating dnode $data[0][7] -elif $data[0][8] == LEADER then - if $data[0][4] == LEADER then +elif $data[0][8] == leader then + if $data[0][4] == leader then goto check_vg_ready_3 endi - if $data[0][6] == LEADER then + if $data[0][6] == leader then goto check_vg_ready_3 endi print ---- vgroup $data[0][0] leader locating dnode $data[0][9] diff --git a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim index fc501096e687c0b7681bbf9e7fcad706f7aafced..3c21dff1b65a9737d96f9e0d9ae1b8c173fa3250 100644 --- a/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim +++ b/tests/script/tsim/sync/threeReplica1VgElectWihtInsert.sim @@ -103,29 +103,29 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $d if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then - if $data[0][6] != FOLLOWER then +if $data[0][4] == leader then + if $data[0][6] != follower then goto check_vg_ready endi - if $data[0][8] != FOLLOWER then + if $data[0][8] != follower then goto check_vg_ready endi print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] goto vg_ready -elif $data[0][6] == LEADER then - if $data[0][4] != FOLLOWER then +elif $data[0][6] == leader then + if $data[0][4] != follower then goto check_vg_ready endi - if $data[0][8] != FOLLOWER then + if $data[0][8] != follower then goto check_vg_ready endi print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] goto vg_ready -elif $data[0][8] == LEADER then - if $data[0][4] != FOLLOWER then +elif $data[0][8] == leader then + if $data[0][4] != follower then goto check_vg_ready endi - if $data[0][6] != FOLLOWER then + if $data[0][6] != follower then goto check_vg_ready endi print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] @@ -193,11 +193,11 @@ switch_leader_loop: print ====> finde vnode of leader, and stop the dnode where the vnode is located, and query stb/ntb count(*) sql show vgroups print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] -if $data[0][4] == LEADER then +if $data[0][4] == leader then $dnodeId = $data[0][3] -elif $data[0][6] == LEADER then +elif $data[0][6] == leader then $dnodeId = $data[0][5] -elif $data[0][8] == LEADER then +elif $data[0][8] == leader then $dnodeId = $data[0][7] else print ====> no leader vnode!!! @@ -226,29 +226,29 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $d if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then - if $data[0][6] != FOLLOWER then +if $data[0][4] == leader then + if $data[0][6] != follower then goto check_vg_ready_2 endi - if $data[0][8] != FOLLOWER then + if $data[0][8] != follower then goto check_vg_ready_2 endi print ---- vgroup $dnodeId leader switch to dnode $data[0][3] goto vg_ready_2 -elif $data[0][6] == LEADER then - if $data[0][4] != FOLLOWER then +elif $data[0][6] == leader then + if $data[0][4] != follower then goto check_vg_ready_2 endi - if $data[0][8] != FOLLOWER then + if $data[0][8] != follower then goto check_vg_ready_2 endi print ---- vgroup $dnodeId leader switch to dnode $data[0][5] goto vg_ready_2 -elif $data[0][8] == LEADER then - if $data[0][4] != FOLLOWER then +elif $data[0][8] == leader then + if $data[0][4] != follower then goto check_vg_ready_2 endi - if $data[0][6] != FOLLOWER then + if $data[0][6] != follower then goto check_vg_ready_2 endi print ---- vgroup $dnodeId leader switch to dnode $data[0][7] @@ -329,29 +329,29 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $d if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then - if $data[0][6] != FOLLOWER then +if $data[0][4] == leader then + if $data[0][6] != follower then goto check_vg_ready_1 endi - if $data[0][8] != FOLLOWER then + if $data[0][8] != follower then goto check_vg_ready_1 endi goto vg_ready_1 endi -if $data[0][6] == LEADER then - if $data[0][4] != FOLLOWER then +if $data[0][6] == leader then + if $data[0][4] != follower then goto check_vg_ready_1 endi - if $data[0][8] != FOLLOWER then + if $data[0][8] != follower then goto check_vg_ready_1 endi goto vg_ready_1 endi -if $data[0][8] == LEADER then - if $data[0][4] != FOLLOWER then +if $data[0][8] == leader then + if $data[0][4] != follower then goto check_vg_ready_1 endi - if $data[0][6] != FOLLOWER then + if $data[0][6] != follower then goto check_vg_ready_1 endi goto vg_ready_1 @@ -390,27 +390,27 @@ system sh/exec.sh -n $dnodeId -s stop -x SIGINT check_vg_ready_3: sql show vgroups print $data(2)[0] $data(2)[1] $data(2)[2] $data(2)[3] $data(2)[4] $data(2)[5] $data(2)[6] $data(2)[7] $data(2)[8] $data(2)[9] $data(2)[10] $data(2)[11] $data(2)[12] $data(2)[13] -if $data[0][4] == LEADER then - if $data[0][6] == LEADER then +if $data[0][4] == leader then + if $data[0][6] == leader then goto check_vg_ready_3 endi - if $data[0][8] == LEADER then + if $data[0][8] == leader then goto check_vg_ready_3 endi print ---- vgroup $data[0][0] leader locating dnode $data[0][5] -elif $data[0][6] == LEADER then - if $data[0][4] == LEADER then +elif $data[0][6] == leader then + if $data[0][4] == leader then goto check_vg_ready_3 endi - if $data[0][8] == LEADER then + if $data[0][8] == leader then goto check_vg_ready_3 endi print ---- vgroup $data[0][0] leader locating dnode $data[0][7] -elif $data[0][8] == LEADER then - if $data[0][4] == LEADER then +elif $data[0][8] == leader then + if $data[0][4] == leader then goto check_vg_ready_3 endi - if $data[0][6] == LEADER then + if $data[0][6] == leader then goto check_vg_ready_3 endi print ---- vgroup $data[0][0] leader locating dnode $data[0][9] diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index 0b1f0df04e9db6af2547cc1da49873082b2682b3..c73efde9d5076e21038741c95f3edf470d288f9e 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -1,80 +1,103 @@ -#run user/pass_alter.sim -#run user/basic1.sim -#run user/privilege2.sim -#run user/user_len.sim -#run user/privilege1.sim -#run user/pass_len.sim -#run tstream/basic1.sim -#run tstream/basic0.sim -#run table/basic1.sim -#run trans/create_db.sim -#run stable/alter1.sim -#run stable/vnode3.sim -#run stable/metrics.sim -#run stable/show.sim -#run stable/values.sim -#run stable/dnode3.sim -#run stable/refcount.sim -#run stable/disk.sim -#run db/basic1.sim -#run db/basic3.sim -#run db/basic7.sim -#run db/basic6.sim -#run db/create_all_options.sim -#run db/basic2.sim -#run db/error1.sim -#run db/taosdlog.sim -#run db/alter_option.sim -#run mnode/basic1.sim -#run parser/fourArithmetic-basic.sim -#run parser/groupby-basic.sim -#run snode/basic1.sim -#run query/time_process.sim -#run query/stddev.sim -#run query/interval-offset.sim -#run query/charScalarFunction.sim -#run query/complex_select.sim -#run query/explain.sim -#run query/crash_sql.sim -#run query/diff.sim -#run query/complex_limit.sim -#run query/complex_having.sim -#run query/udf.sim -#run query/complex_group.sim -#run query/interval.sim -#run query/session.sim - -print ========> dead lock failed when 2 rows in outputCapacity -run query/scalarFunction.sim -run query/scalarNull.sim -run query/complex_where.sim -run tmq/basic1.sim -run tmq/basic4.sim -run tmq/basic1Of2Cons.sim -run tmq/prepareBasicEnv-1vgrp.sim -run tmq/topic.sim -run tmq/basic4Of2Cons.sim -run tmq/prepareBasicEnv-4vgrp.sim -run tmq/basic3.sim -run tmq/basic2Of2Cons.sim -run tmq/basic2.sim -run tmq/basic3Of2Cons.sim -run tmq/basic2Of2ConsOverlap.sim -run tmq/clearConsume.sim -run qnode/basic1.sim -run dnode/basic1.sim -run show/basic.sim -run insert/basic1.sim -run insert/basic0.sim -run insert/backquote.sim -run insert/null.sim -run sync/oneReplica1VgElectWithInsert.sim -run sync/threeReplica1VgElect.sim -run sync/oneReplica1VgElect.sim -run sync/insertDataByRunBack.sim -run sync/threeReplica1VgElectWihtInsert.sim -run sma/tsmaCreateInsertData.sim -run sma/rsmaCreateInsertQuery.sim -run valgrind/checkError.sim -run bnode/basic1.sim - +#run tsim/user/pass_alter.sim +#run tsim/user/basic1.sim +#run tsim/user/privilege2.sim +#run tsim/user/user_len.sim +#run tsim/user/privilege1.sim +#run tsim/user/pass_len.sim +#run tsim/table/basic1.sim +#run tsim/trans/lossdata1.sim +#run tsim/trans/create_db.sim +##run tsim/stable/alter_metrics.sim +##run tsim/stable/tag_modify.sim +##run tsim/stable/alter_comment.sim +##run tsim/stable/column_drop.sim +##run tsim/stable/column_modify.sim +##run tsim/stable/tag_rename.sim +#run tsim/stable/vnode3.sim +#run tsim/stable/metrics.sim +##run tsim/stable/alter_insert2.sim +#run tsim/stable/show.sim +#run tsim/stable/alter_import.sim +##run tsim/stable/tag_add.sim +#run tsim/stable/tag_drop.sim +#run tsim/stable/column_add.sim +#run tsim/stable/alter_count.sim +#run tsim/stable/values.sim +#run tsim/stable/dnode3.sim +#run tsim/stable/alter_insert1.sim +#run tsim/stable/refcount.sim +#run tsim/stable/disk.sim +#run tsim/db/basic1.sim +#run tsim/db/basic3.sim +##run tsim/db/basic7.sim +#run tsim/db/basic6.sim +#run tsim/db/create_all_options.sim +#run tsim/db/basic2.sim +#run tsim/db/error1.sim +#run tsim/db/taosdlog.sim +#run tsim/db/alter_option.sim +#run tsim/mnode/basic1.sim +#run tsim/mnode/basic3.sim +#run tsim/mnode/basic2.sim +#run tsim/parser/fourArithmetic-basic.sim +#run tsim/parser/groupby-basic.sim +#run tsim/snode/basic1.sim +#run tsim/query/time_process.sim +#run tsim/query/stddev.sim +#run tsim/query/interval-offset.sim +#run tsim/query/charScalarFunction.sim +#run tsim/query/complex_select.sim +#run tsim/query/explain.sim +#run tsim/query/crash_sql.sim +#run tsim/query/diff.sim +#run tsim/query/complex_limit.sim +#run tsim/query/complex_having.sim +#run tsim/query/udf.sim +#run tsim/query/complex_group.sim +#run tsim/query/interval.sim +#run tsim/query/session.sim +#run tsim/query/scalarFunction.sim +##run tsim/query/scalarNull.sim +run tsim/query/complex_where.sim +run tsim/tmq/basic1.sim +run tsim/tmq/basic4.sim +run tsim/tmq/basic1Of2Cons.sim +run tsim/tmq/prepareBasicEnv-1vgrp.sim +run tsim/tmq/topic.sim +run tsim/tmq/basic4Of2Cons.sim +run tsim/tmq/prepareBasicEnv-4vgrp.sim +run tsim/tmq/basic3.sim +run tsim/tmq/basic2Of2Cons.sim +run tsim/tmq/basic2.sim +run tsim/tmq/basic3Of2Cons.sim +run tsim/tmq/basic2Of2ConsOverlap.sim +run tsim/tmq/clearConsume.sim +run tsim/qnode/basic1.sim +run tsim/dnode/basic1.sim +run tsim/show/basic.sim +run tsim/stream/basic1.sim +run tsim/stream/triggerInterval0.sim +run tsim/stream/triggerSession0.sim +run tsim/stream/basic0.sim +run tsim/stream/session0.sim +run tsim/stream/session1.sim +run tsim/stream/basic2.sim +run tsim/insert/basic1.sim +run tsim/insert/commit-merge0.sim +run tsim/insert/basic0.sim +run tsim/insert/update0.sim +run tsim/insert/backquote.sim +run tsim/insert/null.sim +run tsim/sync/oneReplica1VgElectWithInsert.sim +run tsim/sync/threeReplica1VgElect.sim +run tsim/sync/oneReplica1VgElect.sim +run tsim/sync/3Replica5VgElect.sim +run tsim/sync/insertDataByRunBack.sim +run tsim/sync/oneReplica5VgElect.sim +run tsim/sync/3Replica1VgElect.sim +run tsim/sync/threeReplica1VgElectWihtInsert.sim +run tsim/sma/tsmaCreateInsertData.sim +run tsim/sma/rsmaCreateInsertQuery.sim +run tsim/valgrind/basic.sim +run tsim/valgrind/checkError.sim +run tsim/bnode/basic1.sim \ No newline at end of file diff --git a/tests/script/tsim/trans/create_db.sim b/tests/script/tsim/trans/create_db.sim index 158a6b9f920e8e194f5336f8985bf609d9c7f2a1..e13014f9c02370f65cf1e1700b84efdc4bcdcce2 100644 --- a/tests/script/tsim/trans/create_db.sim +++ b/tests/script/tsim/trans/create_db.sim @@ -26,7 +26,7 @@ if $data00 != 1 then return -1 endi -if $data02 != LEADER then +if $data02 != leader then return -1 endi diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index 657979658e5b048e8d51d0bf8c67bada5f321402..a219c54e5925075a2c687c9cd134ada03c09e57e 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -75,7 +75,7 @@ def telemetryInfoCheck(infoDict=''): if k not in infoDict["cluster_info"]["dnodes"][0] or v != infoDict["cluster_info"]["dnodes"][0][k] : tdLog.exit("dnodes info is null!") - mnodes_info = { "mnode_id":1, "mnode_ep":f"{hostname}:{serverPort}","role": "LEADER" } + mnodes_info = { "mnode_id":1, "mnode_ep":f"{hostname}:{serverPort}","role": "leader" } for k ,v in mnodes_info.items(): if k not in infoDict["cluster_info"]["mnodes"][0] or v != infoDict["cluster_info"]["mnodes"][0][k] : diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index 46d0a6968875a5e6c484c932abb41946f56bc8ee..57ae1c14905b6a6b620c999bb79e3c3b90c9fc73 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -3,6 +3,7 @@ import taos import sys import time import os +import platform from util.log import * from util.sql import * @@ -25,7 +26,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -41,11 +42,19 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] print(projPath) - libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") - libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") - os.system("mkdir /tmp/udf/") - os.system("cp %s /tmp/udf/ "%libudf1.replace("\n" ,"")) - os.system("cp %s /tmp/udf/ "%libudf2.replace("\n" ,"")) + if platform.system().lower() == 'windows': + self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + if (not tdDnodes.dnodes[0].remoteIP == ""): + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") + self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') + self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') + else: + self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1 = self.libudf1.replace('\r','').replace('\n','') + self.libudf2 = self.libudf2.replace('\r','').replace('\n','') def prepare_data(self): @@ -136,11 +145,11 @@ class TDTestCase: for i in range(5): # create scalar functions - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;") + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) # create aggregate functions - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;") + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -161,11 +170,11 @@ class TDTestCase: tdLog.info("drop two udf functions success ") # create scalar functions - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;") + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) # create aggregate functions - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;") + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -533,8 +542,8 @@ class TDTestCase: tdSql.query("drop function udf2 ") # create function without buffer - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int") - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double") + tdSql.execute("create function udf1 as '%s' outputtype int"%self.libudf1) + tdSql.execute("create aggregate function udf2 as '%s' outputtype double"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -549,8 +558,8 @@ class TDTestCase: tdSql.query("drop function udf2 ") # create function without buffer - tdSql.execute("create aggregate function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") - tdSql.execute("create function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8") + tdSql.execute("create aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute("create function udf2 as '%s' outputtype double bufSize 8"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -558,8 +567,8 @@ class TDTestCase: for aggregate_sql in udf2_sqls: tdSql.error(aggregate_sql) - tdSql.execute(" create function db as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") - tdSql.execute(" create aggregate function test as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") + tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1) tdSql.error(" select db(c1) from stb1 ") tdSql.error(" select db(c1,c6), db(c6) from stb1 ") tdSql.error(" select db(num1,num2), db(num1) from tb ") @@ -607,17 +616,17 @@ class TDTestCase: tdLog.info(" create function name is not build_in functions ") tdSql.execute(" drop function udf1 ") tdSql.execute(" drop function udf2 ") - tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8") - tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8") - tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function tbname as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function function as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function stable as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function union as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function 123 as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function 123db as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function mnode as '/tmp/udf/libudf2.so' outputtype double bufSize 8") + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function stable as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function union as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123 as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123db as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function mnode as '%s' outputtype double bufSize 8"%self.libudf2) def restart_taosd_query_udf(self): diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index e2c6e3c10bd1520c58c4400fd58c741d2904a420..88ed66716153b1e3588a2e4ddb8f89427d352399 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -25,7 +25,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -41,11 +41,19 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] print(projPath) - libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") - libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") - os.system("mkdir /tmp/udf/") - os.system("cp %s /tmp/udf/ "%libudf1.replace("\n" ,"")) - os.system("cp %s /tmp/udf/ "%libudf2.replace("\n" ,"")) + if platform.system().lower() == 'windows': + self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + if (not tdDnodes.dnodes[0].remoteIP == ""): + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") + self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') + self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') + else: + self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1 = self.libudf1.replace('\r','').replace('\n','') + self.libudf2 = self.libudf2.replace('\r','').replace('\n','') def prepare_data(self): @@ -136,11 +144,11 @@ class TDTestCase: for i in range(5): # create scalar functions - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;") + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) # create aggregate functions - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;") + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -161,11 +169,11 @@ class TDTestCase: tdLog.info("drop two udf functions success ") # create scalar functions - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;") + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) # create aggregate functions - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;") + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -533,8 +541,8 @@ class TDTestCase: tdSql.query("drop function udf2 ") # create function without buffer - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int") - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double") + tdSql.execute("create function udf1 as '%s' outputtype int"%self.libudf1) + tdSql.execute("create aggregate function udf2 as '%s' outputtype double"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -549,8 +557,8 @@ class TDTestCase: tdSql.query("drop function udf2 ") # create function without buffer - tdSql.execute("create aggregate function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") - tdSql.execute("create function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8") + tdSql.execute("create aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute("create function udf2 as '%s' outputtype double bufSize 8"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -558,8 +566,8 @@ class TDTestCase: for aggregate_sql in udf2_sqls: tdSql.error(aggregate_sql) - tdSql.execute(" create function db as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") - tdSql.execute(" create aggregate function test as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") + tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1) tdSql.error(" select db(c1) from stb1 ") tdSql.error(" select db(c1,c6), db(c6) from stb1 ") tdSql.error(" select db(num1,num2), db(num1) from tb ") @@ -607,17 +615,17 @@ class TDTestCase: tdLog.info(" create function name is not build_in functions ") tdSql.execute(" drop function udf1 ") tdSql.execute(" drop function udf2 ") - tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8") - tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8") - tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function tbname as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function function as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function stable as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function union as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function 123 as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function 123db as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function mnode as '/tmp/udf/libudf2.so' outputtype double bufSize 8") + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function stable as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function union as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123 as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123db as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function mnode as '%s' outputtype double bufSize 8"%self.libudf2) def restart_taosd_query_udf(self): diff --git a/tests/system-test/0-others/udf_restart_taosd.py b/tests/system-test/0-others/udf_restart_taosd.py index 24d3b5a9c3cf702c4839e83ff02794f5bf08fcb5..94b14f01ba92a3e5e2fd7d5f2a9238f05c993abb 100644 --- a/tests/system-test/0-others/udf_restart_taosd.py +++ b/tests/system-test/0-others/udf_restart_taosd.py @@ -25,7 +25,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -41,11 +41,19 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] print(projPath) - libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") - libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") - os.system("mkdir /tmp/udf/") - os.system("cp %s /tmp/udf/ "%libudf1.replace("\n" ,"")) - os.system("cp %s /tmp/udf/ "%libudf2.replace("\n" ,"")) + if platform.system().lower() == 'windows': + self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + if (not tdDnodes.dnodes[0].remoteIP == ""): + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") + self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') + self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') + else: + self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1 = self.libudf1.replace('\r','').replace('\n','') + self.libudf2 = self.libudf2.replace('\r','').replace('\n','') def prepare_data(self): @@ -136,11 +144,11 @@ class TDTestCase: for i in range(5): # create scalar functions - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;") + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) # create aggregate functions - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;") + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -161,11 +169,11 @@ class TDTestCase: tdLog.info("drop two udf functions success ") # create scalar functions - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8;") + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) # create aggregate functions - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8;") + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) functions = tdSql.getResult("show functions") function_nums = len(functions) @@ -533,8 +541,8 @@ class TDTestCase: tdSql.query("drop function udf2 ") # create function without buffer - tdSql.execute("create function udf1 as '/tmp/udf/libudf1.so' outputtype int") - tdSql.execute("create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double") + tdSql.execute("create function udf1 as '%s' outputtype int"%self.libudf1) + tdSql.execute("create aggregate function udf2 as '%s' outputtype double"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -549,8 +557,8 @@ class TDTestCase: tdSql.query("drop function udf2 ") # create function without buffer - tdSql.execute("create aggregate function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") - tdSql.execute("create function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8") + tdSql.execute("create aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute("create function udf2 as '%s' outputtype double bufSize 8"%self.libudf2) udf1_sqls ,udf2_sqls = self.try_query_sql() for scalar_sql in udf1_sqls: @@ -558,8 +566,8 @@ class TDTestCase: for aggregate_sql in udf2_sqls: tdSql.error(aggregate_sql) - tdSql.execute(" create function db as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") - tdSql.execute(" create aggregate function test as '/tmp/udf/libudf1.so' outputtype int bufSize 8 ") + tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1) tdSql.error(" select db(c1) from stb1 ") tdSql.error(" select db(c1,c6), db(c6) from stb1 ") tdSql.error(" select db(num1,num2), db(num1) from tb ") @@ -607,17 +615,17 @@ class TDTestCase: tdLog.info(" create function name is not build_in functions ") tdSql.execute(" drop function udf1 ") tdSql.execute(" drop function udf2 ") - tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8") - tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create function max as '/tmp/udf/libudf1.so' outputtype int bufSize 8") - tdSql.error("create aggregate function sum as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function tbname as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function function as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function stable as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function union as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function 123 as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function 123db as '/tmp/udf/libudf2.so' outputtype double bufSize 8") - tdSql.error("create aggregate function mnode as '/tmp/udf/libudf2.so' outputtype double bufSize 8") + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function stable as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function union as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123 as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123db as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function mnode as '%s' outputtype double bufSize 8"%self.libudf2) def restart_taosd_query_udf(self): diff --git a/tests/system-test/1-insert/alter_stable.py b/tests/system-test/1-insert/alter_stable.py new file mode 100644 index 0000000000000000000000000000000000000000..e8c354415237f890c906931e0e75ac735bde5098 --- /dev/null +++ b/tests/system-test/1-insert/alter_stable.py @@ -0,0 +1,144 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def get_long_name(self, length, mode="mixed"): + """ + generate long name + mode could be numbers/letters/letters_mixed/mixed + """ + if mode == "numbers": + population = string.digits + elif mode == "letters": + population = string.ascii_letters.lower() + elif mode == "letters_mixed": + population = string.ascii_letters.upper() + string.ascii_letters.lower() + else: + population = string.ascii_letters.lower() + string.digits + return "".join(random.choices(population, k=length)) + def alter_stable_column_check(self,dbname,stbname,tbname): + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.execute( + f'create stable {stbname} (ts timestamp, c1 tinyint, c2 smallint, c3 int, \ + c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 bool,c12 binary(20),c13 nchar(20)) tags(t0 int) ') + tdSql.execute(f'create table {tbname} using {stbname} tags(1)') + tdSql.execute(f'insert into {tbname} values (now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') + tdSql.execute(f'alter stable {stbname} add column c14 int') + tdSql.query(f'select c14 from {stbname}') + tdSql.checkRows(1) + tdSql.execute(f'alter stable {stbname} add column `c15` int') + tdSql.query(f'select c15 from {stbname}') + tdSql.checkRows(1) + tdSql.query(f'describe {stbname}') + tdSql.checkRows(17) + tdSql.execute(f'alter stable {stbname} drop column c14') + tdSql.query(f'describe {stbname}') + tdSql.checkRows(16) + tdSql.execute(f'alter stable {stbname} drop column `c15`') + tdSql.query(f'describe {stbname}') + tdSql.checkRows(15) + tdSql.execute(f'alter stable {stbname} modify column c12 binary(30)') + tdSql.query(f'describe {stbname}') + tdSql.checkData(12,2,30) + tdSql.execute(f'alter stable {stbname} modify column `c12` binary(35)') + tdSql.query(f'describe {stbname}') + tdSql.checkData(12,2,35) + tdSql.error(f'alter stable {stbname} modify column `c12` binary(34)') + tdSql.execute(f'alter stable {stbname} modify column c13 nchar(30)') + tdSql.query(f'describe {stbname}') + tdSql.checkData(13,2,30) + tdSql.error(f'alter stable {stbname} modify column c13 nchar(29)') + tdSql.error(f'alter stable {stbname} rename column c1 c21') + tdSql.error(f'alter stable {stbname} modify column c1 int') + tdSql.error(f'alter stable {stbname} modify column c4 int') + tdSql.error(f'alter stable {stbname} modify column c8 int') + tdSql.error(f'alter stable {stbname} modify column c1 unsigned int') + tdSql.error(f'alter stable {stbname} modify column c9 double') + tdSql.error(f'alter stable {stbname} modify column c10 float') + tdSql.error(f'alter stable {stbname} modify column c11 int') + tdSql.execute(f'drop database {dbname}') + + def alter_stable_tag_check(self,dbname,stbname,tbname): + tdSql.execute(f'create database if not exists {dbname}') + tdSql.execute(f'use {dbname}') + tdSql.execute( + f'create stable {stbname} (ts timestamp, c1 int) tags(ts_tag timestamp, t1 tinyint, t2 smallint, t3 int, \ + t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 bool,t12 binary(20),t13 nchar(20)) ') + tdSql.execute(f'create table {tbname} using {stbname} tags(now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') + tdSql.execute(f'insert into {tbname} values(now,1)') + + tdSql.execute(f'alter stable {stbname} add tag t14 int') + tdSql.query(f'select t14 from {stbname}') + tdSql.checkRows(1) + tdSql.execute(f'alter stable {stbname} add tag `t15` int') + tdSql.query(f'select t14 from {stbname}') + tdSql.checkRows(1) + tdSql.query(f'describe {stbname}') + tdSql.checkRows(18) + tdSql.execute(f'alter stable {stbname} drop tag t14') + tdSql.query(f'describe {stbname}') + tdSql.checkRows(17) + tdSql.execute(f'alter stable {stbname} drop tag `t15`') + tdSql.query(f'describe {stbname}') + tdSql.checkRows(16) + tdSql.execute(f'alter stable {stbname} modify tag t12 binary(30)') + tdSql.query(f'describe {stbname}') + tdSql.checkData(14,2,30) + tdSql.execute(f'alter stable {stbname} modify tag `t12` binary(35)') + tdSql.query(f'describe {stbname}') + tdSql.checkData(14,2,35) + tdSql.error(f'alter stable {stbname} modify tag `t12` binary(34)') + tdSql.execute(f'alter stable {stbname} modify tag t13 nchar(30)') + tdSql.query(f'describe {stbname}') + tdSql.checkData(15,2,30) + tdSql.error(f'alter stable {stbname} modify tag t13 nchar(29)') + tdSql.execute(f'alter table {stbname} rename tag t1 t21') + tdSql.query(f'describe {stbname}') + tdSql.checkData(3,0,'t21') + tdSql.execute(f'alter table {stbname} rename tag `t21` t1') + tdSql.query(f'describe {stbname}') + tdSql.checkData(3,0,'t1') + + for i in ['bigint','unsigned int','float','double','binary(10)','nchar(10)']: + for j in [1,2,3]: + tdSql.error(f'alter table {stbname} modify tag t{j} {i}') + for i in ['int','unsigned int','float','binary(10)','nchar(10)']: + tdSql.error(f'alter table {stbname} modify tag t8 {i}') + tdSql.error(f'alter table {stbname} modify tag t4 int') + tdSql.execute(f'drop database {dbname}') + + def run(self): + + dbname = self.get_long_name(length=10, mode="letters") + stbname = self.get_long_name(length=5, mode="letters") + tbname = self.get_long_name(length=5, mode="letters") + self.alter_stable_column_check(dbname,stbname,tbname) + self.alter_stable_tag_check(dbname,stbname,tbname) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/1-insert/alter_table.py b/tests/system-test/1-insert/alter_table.py new file mode 100644 index 0000000000000000000000000000000000000000..53e3793dcf9852a733730a47659b4d089f3391df --- /dev/null +++ b/tests/system-test/1-insert/alter_table.py @@ -0,0 +1,170 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +from util.log import * +from util.cases import * +from util.sql import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def get_long_name(self, length, mode="mixed"): + """ + generate long name + mode could be numbers/letters/letters_mixed/mixed + """ + if mode == "numbers": + population = string.digits + elif mode == "letters": + population = string.ascii_letters.lower() + elif mode == "letters_mixed": + population = string.ascii_letters.upper() + string.ascii_letters.lower() + else: + population = string.ascii_letters.lower() + string.digits + return "".join(random.choices(population, k=length)) + + def alter_tb_tag_check(self): + tag_tinyint = random.randint(-127,129) + tag_int = random.randint(-2147483648,2147483647) + tag_smallint = random.randint(-32768,32768) + tag_bigint = random.randint(-2147483648,2147483647) + tag_untinyint = random.randint(0,256) + tag_unsmallint = random.randint(0,65536) + tag_unint = random.randint(0,4294967296) + tag_unbigint = random.randint(0,2147483647) + tag_binary = self.get_long_name(length=10, mode="letters") + tag_nchar = self.get_long_name(length=10, mode="letters") + dbname = self.get_long_name(length=10, mode="letters") + tdSql.execute(f'create database if not exists {dbname}') + stbname = self.get_long_name(length=3, mode="letters") + tbname = self.get_long_name(length=3, mode="letters") + tdLog.info('--------------------------child table tag check--------------------------------------') + tdLog.info(f'-----------------create stable {stbname} and child table {tbname}-------------------') + tdSql.execute(f'create stable if not exists {dbname}.{stbname} (col_ts timestamp, c1 int) tags (tag_ts timestamp, t1 tinyint, t2 smallint, t3 int, \ + t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 bool,t12 binary(20),t13 nchar(20))') + tdSql.execute(f'create table if not exists {dbname}.{tbname} using {dbname}.{stbname} tags(now, 1, 2, 3, 4, 5, 6, 7, 8, 9.9, 10.1, True,"abc123","涛思数据")') + tdSql.execute(f'insert into {dbname}.{tbname} values(now, 1)') + tdSql.execute(f'alter table {dbname}.{tbname} set tag tag_ts = 1640966400000') + tdSql.execute(f'alter table {dbname}.{tbname} set tag `t1` = 11') + tdSql.query(f'select * from {dbname}.{stbname}') + tdSql.checkData(0,3,11) + tdSql.execute(f'alter table {dbname}.{tbname} set tag t1 = {tag_tinyint}') + tdSql.execute(f'alter table {dbname}.{tbname} set tag t2 = {tag_smallint}') + tdSql.execute(f'alter table {dbname}.{tbname} set tag t3 = {tag_int}') + tdSql.execute(f'alter table {dbname}.{tbname} set tag t4 = {tag_bigint}') + tdSql.execute(f'alter table {dbname}.{tbname} set tag t5 = {tag_untinyint}') + tdSql.execute(f'alter table {dbname}.{tbname} set tag t6 = {tag_unsmallint}') + tdSql.execute(f'alter table {dbname}.{tbname} set tag t7 = {tag_unint}') + tdSql.execute(f'alter table {dbname}.{tbname} set tag t8 = {tag_unbigint}') + tdSql.execute(f'alter table {dbname}.{tbname} set tag t11 = false') + tdSql.execute(f'alter table {dbname}.{tbname} set tag t12 = "{tag_binary}"') + tdSql.execute(f'alter table {dbname}.{tbname} set tag t13 = "{tag_nchar}"') + tdSql.query(f'select * from {dbname}.{stbname}') + # bug TD-15899 + tdSql.checkData(0,2,'2022-01-01 00:00:00.000') + tdSql.checkData(0,3,tag_tinyint) + tdSql.checkData(0,4,tag_smallint) + tdSql.checkData(0,5,tag_int) + tdSql.checkData(0,6,tag_bigint) + tdSql.checkData(0,7,tag_untinyint) + tdSql.checkData(0,8,tag_unsmallint) + tdSql.checkData(0,9,tag_unint) + tdSql.checkData(0,10,tag_unbigint) + + tdSql.checkData(0,13,False) + tdSql.checkData(0,14,tag_binary) + tdSql.checkData(0,15,tag_nchar) + + # bug TD-16211 insert length more than setting binary and nchar + # tag_binary = self.get_long_name(length=21, mode="letters") + # tag_nchar = self.get_long_name(length=21, mode="letters") + # tdSql.error(f'alter table {dbname}.{tbname} set tag t12 = "{tag_binary}"') + # tdSql.error(f'alter table {dbname}.{tbname} set tag t13 = "{tag_nchar}"') + + # bug TD-16210 modify binary to nchar + # tdSql.error(f'alter table {dbname}.{tbname} modify tag t12 nchar(10)') + tdSql.execute(f"drop database {dbname}") + def alter_ntb_column_check(self): + ''' + alter ntb column check + ''' + dbname = self.get_long_name(length=10, mode="letters") + tdSql.execute(f'create database if not exists {dbname}') + tbname = self.get_long_name(length=3, mode="letters") + tdLog.info('------------------normal table column check---------------------') + tdLog.info(f'-----------------create normal table {tbname}-------------------') + tdSql.execute(f'create table if not exists {dbname}.{tbname} (ts timestamp, c1 tinyint, c2 smallint, c3 int, \ + c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 bool,c12 binary(20),c13 nchar(20))') + tdSql.execute(f'insert into {dbname}.{tbname} values (now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') + # bug TD-15757 + tdSql.execute(f'alter table {dbname}.{tbname} add column c14 int') + tdSql.query(f'select c14 from {dbname}.{tbname}') + tdSql.checkRows(1) + tdSql.execute(f'alter table {dbname}.{tbname} add column `c15` int') + tdSql.query(f'select c15 from {dbname}.{tbname}') + tdSql.checkRows(1) + tdSql.query(f'describe {dbname}.{tbname}') + tdSql.checkRows(16) + tdSql.execute(f'alter table {dbname}.{tbname} drop column c14') + tdSql.query(f'describe {dbname}.{tbname}') + tdSql.checkRows(15) + tdSql.execute(f'alter table {dbname}.{tbname} drop column `c15`') + tdSql.query(f'describe {dbname}.{tbname}') + tdSql.checkRows(14) + tdSql.execute(f'alter table {dbname}.{tbname} modify column c12 binary(30)') + tdSql.query(f'describe {dbname}.{tbname}') + tdSql.checkData(12,2,30) + tdSql.execute(f'alter table {dbname}.{tbname} modify column `c12` binary(35)') + tdSql.query(f'describe {dbname}.{tbname}') + tdSql.checkData(12,2,35) + tdSql.error(f'alter table {dbname}.{tbname} modify column c12 binary(34)') + tdSql.error(f'alter table {dbname}.{tbname} modify column c12 nchar(10)') + tdSql.error(f'alter table {dbname}.{tbname} modify column c12 int') + tdSql.execute(f'alter table {dbname}.{tbname} modify column c13 nchar(30)') + tdSql.query(f'describe {dbname}.{tbname}') + tdSql.checkData(13,2,30) + tdSql.execute(f'alter table {dbname}.{tbname} modify column `c13` nchar(35)') + tdSql.query(f'describe {dbname}.{tbname}') + tdSql.checkData(13,2,35) + tdSql.error(f'alter table {dbname}.{tbname} modify column c13 nchar(34)') + tdSql.error(f'alter table {dbname}.{tbname} modify column c13 binary(10)') + tdSql.execute(f'alter table {dbname}.{tbname} rename column c1 c21') + tdSql.query(f'describe {dbname}.{tbname}') + tdSql.checkData(1,0,'c21') + tdSql.execute(f'alter table {dbname}.{tbname} rename column `c21` c1') + tdSql.query(f'describe {dbname}.{tbname}') + tdSql.checkData(1,0,'c1') + + tdSql.error(f'alter table {dbname}.{tbname} modify column c1 bigint') + tdSql.error(f'alter table {dbname}.{tbname} modify column c1 double') + tdSql.error(f'alter table {dbname}.{tbname} modify column c4 int') + tdSql.error(f'alter table {dbname}.{tbname} modify column `c1` double') + tdSql.error(f'alter table {dbname}.{tbname} modify column c9 double') + tdSql.error(f'alter table {dbname}.{tbname} modify column c10 float') + tdSql.error(f'alter table {dbname}.{tbname} modify column c1 bool') + tdSql.error(f'alter table {dbname}.{tbname} modify column c1 binary(10)') + tdSql.execute(f'drop database {dbname}') + def run(self): + self.alter_tb_tag_check() + self.alter_ntb_column_check() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index d8ef8fa363fe1f2911f4ea3fde9d43a82f7a8a88..6a855ebd4bae8c3ae96d3826f91c686b06da6c7d 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -48,8 +48,8 @@ class TDTestCase: # test duplicate key using the first one. elimate empty key tdSql.execute("CREATE TABLE if not exists jsons1_8 using jsons1 tags('{\"tag1\":null, \"tag1\":true, \"tag1\":45, \"1tag$\":2, \" \":90, \"\":32}')") - #tdSql.query("select jtag from jsons1_8") - #tdSql.checkData(0, 0, '{"tag1":null,"1tag$":2," ":90}') + tdSql.query("select jtag from jsons1_8") + tdSql.checkData(0, 0, '{" ":90,"1tag$":2,"tag1":null}') # test empty json string, save as jtag is NULL tdSql.execute("insert into jsons1_9 using jsons1 tags('\t') values (1591062328000, 24, NULL, '你就会', '2sdw')") @@ -95,8 +95,8 @@ class TDTestCase: tdSql.error("ALTER TABLE jsons1 MODIFY TAG jtag nchar(128)") # tdSql.execute("ALTER TABLE jsons1_1 SET TAG jtag='{\"tag1\":\"femail\",\"tag2\":35,\"tag3\":true}'") - # tdSql.query("select jtag from jsons1_1") - # tdSql.checkData(0, 0, '{"tag1":"femail","tag2":35,"tag3":true}') + tdSql.query("select jtag from jsons1_1") + tdSql.checkData(0, 0, '{"tag1":"femail","tag2":35,"tag3":true}') tdSql.execute("ALTER TABLE jsons1 rename TAG jtag jtag_new") tdSql.execute("ALTER TABLE jsons1 rename TAG jtag_new jtag") @@ -106,69 +106,69 @@ class TDTestCase: # # print("============== STEP 3 ===== query table") # # test error syntax - # tdSql.error("select * from jsons1 where jtag->tag1='beijing'") - # tdSql.error("select * from jsons1 where jtag->'location'") - # tdSql.error("select * from jsons1 where jtag->''") - # tdSql.error("select * from jsons1 where jtag->''=9") - # tdSql.error("select -> from jsons1") - # tdSql.error("select * from jsons1 where contains") - # tdSql.error("select * from jsons1 where jtag->") - # tdSql.error("select jtag->location from jsons1") - # tdSql.error("select jtag contains location from jsons1") - # tdSql.error("select * from jsons1 where jtag contains location") - # tdSql.error("select * from jsons1 where jtag contains''") - # tdSql.error("select * from jsons1 where jtag contains 'location'='beijing'") + tdSql.error("select * from jsons1 where jtag->tag1='beijing'") + #tdSql.error("select * from jsons1 where jtag->'location'") + #tdSql.error("select * from jsons1 where jtag->''") + #tdSql.error("select * from jsons1 where jtag->''=9") + tdSql.error("select -> from jsons1") + tdSql.error("select * from jsons1 where contains") + tdSql.error("select * from jsons1 where jtag->") + tdSql.error("select jtag->location from jsons1") + tdSql.error("select jtag contains location from jsons1") + tdSql.error("select * from jsons1 where jtag contains location") + tdSql.error("select * from jsons1 where jtag contains''") + tdSql.error("select * from jsons1 where jtag contains 'location'='beijing'") # # # test function error - # tdSql.error("select avg(jtag->'tag1') from jsons1") - # tdSql.error("select avg(jtag) from jsons1") - # tdSql.error("select min(jtag->'tag1') from jsons1") - # tdSql.error("select min(jtag) from jsons1") - # tdSql.error("select ceil(jtag->'tag1') from jsons1") - # tdSql.error("select ceil(jtag) from jsons1") + tdSql.error("select avg(jtag->'tag1') from jsons1") + tdSql.error("select avg(jtag) from jsons1") + tdSql.error("select min(jtag->'tag1') from jsons1") + tdSql.error("select min(jtag) from jsons1") + tdSql.error("select ceil(jtag->'tag1') from jsons1") + tdSql.error("select ceil(jtag) from jsons1") # # # test select normal column - # tdSql.query("select dataint from jsons1") - # tdSql.checkRows(9) - # tdSql.checkData(1, 0, 1) + tdSql.query("select dataint from jsons1") + tdSql.checkRows(9) + tdSql.checkData(1, 0, 1) # test select json tag - # tdSql.query("select * from jsons1") - # tdSql.checkRows(8) - # tdSql.query("select jtag from jsons1") - # tdSql.checkRows(7) - # tdSql.query("select jtag from jsons1 where jtag is null") + tdSql.query("select * from jsons1") + tdSql.checkRows(9) + tdSql.query("select jtag from jsons1") + tdSql.checkRows(13) + tdSql.query("select jtag from jsons1 where jtag is null") # tdSql.checkRows(5) - # tdSql.query("select jtag from jsons1 where jtag is not null") + tdSql.query("select jtag from jsons1 where jtag is not null") # tdSql.checkRows(8) # test jtag is NULL - #tdSql.query("select jtag from jsons1_9") - #tdSql.checkData(0, 0, None) + tdSql.query("select jtag from jsons1_9") + tdSql.checkData(0, 0, None) # # test select json tag->'key', value is string - # tdSql.query("select jtag->'tag1' from jsons1_1") - # tdSql.checkData(0, 0, '"femail"') - # tdSql.query("select jtag->'tag2' from jsons1_6") - # tdSql.checkData(0, 0, '""') - # # test select json tag->'key', value is int - # tdSql.query("select jtag->'tag2' from jsons1_1") - # tdSql.checkData(0, 0, 35) - # # test select json tag->'key', value is bool - # tdSql.query("select jtag->'tag3' from jsons1_1") - # tdSql.checkData(0, 0, "true") - # # test select json tag->'key', value is null - # tdSql.query("select jtag->'tag1' from jsons1_4") - # tdSql.checkData(0, 0, "null") - # # test select json tag->'key', value is double - # tdSql.query("select jtag->'tag1' from jsons1_5") - # tdSql.checkData(0, 0, "1.232000000") - # # test select json tag->'key', key is not exist - # tdSql.query("select jtag->'tag10' from jsons1_4") - # tdSql.checkData(0, 0, None) - # - # tdSql.query("select jtag->'tag1' from jsons1") - # tdSql.checkRows(13) + tdSql.query("select jtag->'tag1' from jsons1_1") + tdSql.checkData(0, 0, '"femail"') + tdSql.query("select jtag->'tag2' from jsons1_6") + tdSql.checkData(0, 0, '""') + # test select json tag->'key', value is int + tdSql.query("select jtag->'tag2' from jsons1_1") + tdSql.checkData(0, 0, "35.000000000") + # test select json tag->'key', value is bool + tdSql.query("select jtag->'tag3' from jsons1_1") + tdSql.checkData(0, 0, "true") + # test select json tag->'key', value is null + tdSql.query("select jtag->'tag1' from jsons1_4") + tdSql.checkData(0, 0, "null") + # test select json tag->'key', value is double + tdSql.query("select jtag->'tag1' from jsons1_5") + tdSql.checkData(0, 0, "1.232000000") + # test select json tag->'key', key is not exist + tdSql.query("select jtag->'tag10' from jsons1_4") + tdSql.checkData(0, 0, None) + # + tdSql.query("select jtag->'tag1' from jsons1") + tdSql.checkRows(13) # test header name res = tdSql.getColNameList("select jtag->'tag1' from jsons1") cname_list = [] @@ -176,7 +176,6 @@ class TDTestCase: tdSql.checkColNameList(res, cname_list) - # # test where with json tag # tdSql.error("select * from jsons1_1 where jtag is not null") # tdSql.error("select * from jsons1 where jtag='{\"tag1\":11,\"tag2\":\"\"}'") @@ -261,9 +260,9 @@ class TDTestCase: # tdSql.checkRows(1) # # # where json is null - # tdSql.query("select * from jsons1 where jtag is null") + tdSql.query("select * from jsons1 where jtag is null") # tdSql.checkRows(1) - # tdSql.query("select * from jsons1 where jtag is not null") + tdSql.query("select * from jsons1 where jtag is not null") # tdSql.checkRows(8) # # # where json key is null @@ -313,8 +312,8 @@ class TDTestCase: # tdSql.checkRows(2) # # # test with tbname/normal column - # tdSql.query("select * from jsons1 where tbname = 'jsons1_1'") - # tdSql.checkRows(2) + tdSql.query("select * from jsons1 where tbname = 'jsons1_1'") + tdSql.checkRows(2) # tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3'") # tdSql.checkRows(2) # tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3' and dataint=3") @@ -345,14 +344,14 @@ class TDTestCase: # tdSql.checkRows(1) # # # test distinct - # tdSql.execute("insert into jsons1_14 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":null}') values(1591062628000, 2, NULL, '你就会', 'dws')") - # tdSql.query("select distinct jtag->'tag1' from jsons1") - # tdSql.checkRows(8) - # tdSql.query("select distinct jtag from jsons1") - # tdSql.checkRows(9) + tdSql.execute("insert into jsons1_14 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":null}') values(1591062628000, 2, NULL, '你就会', 'dws')") + tdSql.query("select distinct jtag->'tag1' from jsons1") + tdSql.checkRows(8) + tdSql.query("select distinct jtag from jsons1") + tdSql.checkRows(9) # # #test dumplicate key with normal colomn - # tdSql.execute("INSERT INTO jsons1_15 using jsons1 tags('{\"tbname\":\"tt\",\"databool\":true,\"datastr\":\"是是是\"}') values(1591060828000, 4, false, 'jjsf', \"你就会\")") + tdSql.execute("INSERT INTO jsons1_15 using jsons1 tags('{\"tbname\":\"tt\",\"databool\":true,\"datastr\":\"是是是\"}') values(1591060828000, 4, false, 'jjsf', \"你就会\")") # tdSql.query("select *,tbname,jtag from jsons1 where jtag->'datastr' match '是' and datastr match 'js'") # tdSql.checkRows(1) # tdSql.query("select tbname,jtag->'tbname' from jsons1 where jtag->'tbname'='tt' and tbname='jsons1_14'") @@ -377,67 +376,71 @@ class TDTestCase: # cname_list.append("a.jtag->'tag3'") # tdSql.checkColNameList(res, cname_list) # - # # test group by & order by json tag - # tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag2'") - # tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag") - # tdSql.query("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc") - # tdSql.checkRows(8) - # tdSql.checkData(1, 0, 2) - # tdSql.checkData(1, 1, '"femail"') - # tdSql.checkData(2, 0, 1) - # tdSql.checkData(2, 1, 11) - # tdSql.checkData(5, 0, 1) - # tdSql.checkData(5, 1, "false") - # tdSql.checkData(6, 0, 1) - # tdSql.checkData(6, 1, "null") - # tdSql.checkData(7, 0, 2) - # tdSql.checkData(7, 1, None) - # - # tdSql.query("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag1' asc") - # tdSql.checkRows(8) - # tdSql.checkData(0, 0, 2) - # tdSql.checkData(0, 1, None) - # tdSql.checkData(2, 0, 1) - # tdSql.checkData(2, 1, "false") - # tdSql.checkData(5, 0, 1) - # tdSql.checkData(5, 1, 11) - # tdSql.checkData(6, 0, 2) - # tdSql.checkData(6, 1, '"femail"') - # - # # test stddev with group by json tag - # tdSql.query("select stddev(dataint) from jsons1 group by jtag->'tag1'") - # tdSql.checkData(0, 0, 10) - # tdSql.checkData(0, 1, None) - # tdSql.checkData(1, 0, 0) - # tdSql.checkData(1, 1, "null") - # tdSql.checkData(6, 0, 11) - # tdSql.checkData(6, 1, '"femail"') - # - # res = tdSql.getColNameList("select stddev(dataint) from jsons1 group by jsons1.jtag->'tag1'") - # cname_list = [] - # cname_list.append("stddev(dataint)") - # cname_list.append("jsons1.jtag->'tag1'") - # tdSql.checkColNameList(res, cname_list) - # - # # test top/bottom with group by json tag - # tdSql.query("select top(dataint,100) from jsons1 group by jtag->'tag1'") - # tdSql.checkRows(11) - # tdSql.checkData(0, 1, 4) - # tdSql.checkData(1, 1, 24) - # tdSql.checkData(1, 2, None) - # tdSql.checkData(8, 1, 1) - # tdSql.checkData(8, 2, '"femail"') - # - # # test having + # test group by & order by json tag + tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag->'tag2'") + tdSql.error("select count(*) from jsons1 group by jtag->'tag1' order by jtag") + tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' desc") + tdSql.checkRows(8) + tdSql.checkData(0, 0, 2) + tdSql.checkData(0, 1, '"femail"') + tdSql.checkData(1, 0, 2) + tdSql.checkData(1, 1, '"收到货"') + tdSql.checkData(2, 0, 1) + tdSql.checkData(2, 1, "11.000000000") + tdSql.checkData(5, 0, 1) + tdSql.checkData(5, 1, "false") + tdSql.checkData(6, 0, 1) + tdSql.checkData(6, 1, "null") + tdSql.checkData(7, 0, 2) + tdSql.checkData(7, 1, None) + + tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1' asc") + tdSql.checkRows(8) + tdSql.checkData(0, 0, 2) + tdSql.checkData(0, 1, None) + tdSql.checkData(2, 0, 1) + tdSql.checkData(2, 1, "false") + tdSql.checkData(5, 0, 1) + tdSql.checkData(5, 1, "11.000000000") + tdSql.checkData(7, 0, 2) + tdSql.checkData(7, 1, '"femail"') + # + # test stddev with group by json tag + tdSql.query("select stddev(dataint),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") + tdSql.checkRows(8) + tdSql.checkData(0, 0, 10) + tdSql.checkData(0, 1, None) + tdSql.checkData(4, 0, 0) + tdSql.checkData(4, 1, "5.000000000") + tdSql.checkData(7, 0, 11) + tdSql.checkData(7, 1, '"femail"') + + res = tdSql.getColNameList("select stddev(dataint),jsons1.jtag->'tag1' from jsons1 group by jsons1.jtag->'tag1' order by jtag->'tag1'") + cname_list = [] + cname_list.append("stddev(dataint)") + cname_list.append("jsons1.jtag->'tag1'") + tdSql.checkColNameList(res, cname_list) + + # test top/bottom with group by json tag + tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") + tdSql.checkRows(11) + tdSql.checkData(0, 1, None) + tdSql.checkData(2, 0, 4) + tdSql.checkData(3, 0, 3) + tdSql.checkData(3, 1, "false") + tdSql.checkData(10, 0, 23) + tdSql.checkData(10, 1, '"femail"') + + # test having # tdSql.query("select stddev(dataint) from jsons1 group by jtag->'tag1' having stddev(dataint) > 0") # tdSql.checkRows(2) - # - # # subquery with json tag - # tdSql.query("select * from (select jtag, dataint from jsons1)") - # tdSql.checkRows(11) - # tdSql.checkData(1, 1, 1) - # tdSql.checkData(2, 0, '{"tag1":5,"tag2":"beijing"}') - # + + # subquery with json tag + tdSql.query("select * from (select jtag, dataint from jsons1) order by dataint") + tdSql.checkRows(11) + tdSql.checkData(1, 1, 1) + tdSql.checkData(2, 0, '{"tag1":5,"tag2":"beijing"}') + # tdSql.query("select jtag->'tag1' from (select jtag->'tag1', dataint from jsons1)") # tdSql.checkRows(11) # tdSql.checkData(1, 0, '"femail"') @@ -514,17 +517,17 @@ class TDTestCase: # tdSql.checkRows(3) # # #test TD-12077 - # tdSql.execute("insert into jsons1_16 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":-2.111}') values(1591062628000, 2, NULL, '你就会', 'dws')") - # tdSql.query("select jtag->'tag3' from jsons1_16") - # tdSql.checkData(0, 0, '-2.111000000') - # + tdSql.execute("insert into jsons1_16 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":-2.111}') values(1591062628000, 2, NULL, '你就会', 'dws')") + tdSql.query("select jtag->'tag3' from jsons1_16") + tdSql.checkData(0, 0, '-2.111000000') + # # test TD-12452 - # tdSql.execute("ALTER TABLE jsons1_1 SET TAG jtag=NULL") - # tdSql.query("select jtag from jsons1_1") - # tdSql.checkData(0, 0, None) - # tdSql.execute("CREATE TABLE if not exists jsons1_20 using jsons1 tags(NULL)") - # tdSql.query("select jtag from jsons1_20") - # tdSql.checkData(0, 0, None) + tdSql.execute("ALTER TABLE jsons1_1 SET TAG jtag=NULL") + tdSql.query("select jtag from jsons1_1") + tdSql.checkData(0, 0, None) + tdSql.execute("CREATE TABLE if not exists jsons1_20 using jsons1 tags(NULL)") + tdSql.query("select jtag from jsons1_20") + tdSql.checkData(0, 0, None) # tdSql.execute("insert into jsons1_21 using jsons1 tags(NULL) values(1591061628000, 11, false, '你就会','')") # tdSql.query("select jtag from jsons1_21") # tdSql.checkData(0, 0, None) @@ -536,24 +539,24 @@ class TDTestCase: tdSql.checkData(5, 2, 4095) # # #test TD-13918 - # tdSql.execute("drop table if exists jsons_13918_1") - # tdSql.execute("drop table if exists jsons_13918_2") - # tdSql.execute("drop table if exists jsons_13918_3") - # tdSql.execute("drop table if exists jsons_13918_4") - # tdSql.execute("drop table if exists jsons_stb") - # tdSql.execute("create table jsons_stb (ts timestamp, dataInt int) tags (jtag json)") - # tdSql.error("create table jsons_13918_1 using jsons_stb tags ('nullx')") - # tdSql.error("create table jsons_13918_2 using jsons_stb tags (nullx)") - # tdSql.error("insert into jsons_13918_3 using jsons_stb tags('NULLx') values(1591061628001, 11)") - # tdSql.error("insert into jsons_13918_4 using jsons_stb tags(NULLx) values(1591061628002, 11)") - # tdSql.execute("create table jsons_13918_1 using jsons_stb tags ('null')") - # tdSql.execute("create table jsons_13918_2 using jsons_stb tags (null)") - # tdSql.execute("insert into jsons_13918_1 values(1591061628003, 11)") - # tdSql.execute("insert into jsons_13918_2 values(1591061628004, 11)") - # tdSql.execute("insert into jsons_13918_3 using jsons_stb tags('NULL') values(1591061628005, 11)") - # tdSql.execute("insert into jsons_13918_4 using jsons_stb tags(\"NULL\") values(1591061628006, 11)") - # tdSql.query("select * from jsons_stb") - # tdSql.checkRows(4) + tdSql.execute("drop table if exists jsons_13918_1") + tdSql.execute("drop table if exists jsons_13918_2") + tdSql.execute("drop table if exists jsons_13918_3") + tdSql.execute("drop table if exists jsons_13918_4") + tdSql.execute("drop table if exists jsons_stb") + tdSql.execute("create table jsons_stb (ts timestamp, dataInt int) tags (jtag json)") + tdSql.error("create table jsons_13918_1 using jsons_stb tags ('nullx')") + tdSql.error("create table jsons_13918_2 using jsons_stb tags (nullx)") + tdSql.error("insert into jsons_13918_3 using jsons_stb tags('NULLx') values(1591061628001, 11)") + tdSql.error("insert into jsons_13918_4 using jsons_stb tags(NULLx) values(1591061628002, 11)") + tdSql.execute("create table jsons_13918_1 using jsons_stb tags ('null')") + tdSql.execute("create table jsons_13918_2 using jsons_stb tags (null)") + tdSql.execute("insert into jsons_13918_1 values(1591061628003, 11)") + tdSql.execute("insert into jsons_13918_2 values(1591061628004, 11)") + tdSql.execute("insert into jsons_13918_3 using jsons_stb tags('NULL') values(1591061628005, 11)") + tdSql.execute("insert into jsons_13918_4 using jsons_stb tags(\"NULL\") values(1591061628006, 11)") + tdSql.query("select * from jsons_stb") + tdSql.checkRows(4) def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/mavg.py b/tests/system-test/2-query/mavg.py index c8cbd269f9ce2c3ae0e5bd6b0361f0eb4252b1a9..1d929646159d36a14b5ce6e9e95e1b02ad0be43a 100644 --- a/tests/system-test/2-query/mavg.py +++ b/tests/system-test/2-query/mavg.py @@ -288,15 +288,18 @@ class TDTestCase: else: tdSql.query(f"select {col} from {table_expr} {re.sub('limit [0-9]*|offset [0-9]*','',condition)}") offset_val = condition.split("offset")[1].split(" ")[1] if "offset" in condition else 0 - pre_result = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] - pre_mavg = pre_mavg = np.convolve(pre_result, np.ones(k), "valid")[offset_val:]/k - tdSql.query(self.mavg_query_form( - sel=sel, func=func, col=col, m_comm=m_comm, k=k, r_comm=r_comm, alias=alias, fr=fr, - table_expr=table_expr, condition=condition - )) - for i in range(tdSql.queryRows): - print(f"case in {line}: ", end='') - tdSql.checkData(i, 0, pre_mavg[i]) + # print(f"select {col} from {table_expr} {re.sub('limit [0-9]*|offset [0-9]*','',condition)}") + if not tdSql.queryResult: + pre_result = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + + pre_mavg = pre_mavg = np.convolve(pre_result, np.ones(k), "valid")[offset_val:]/k + tdSql.query(self.mavg_query_form( + sel=sel, func=func, col=col, m_comm=m_comm, k=k, r_comm=r_comm, alias=alias, fr=fr, + table_expr=table_expr, condition=condition + )) + for i in range(tdSql.queryRows): + print(f"case in {line}: ", end='') + tdSql.checkData(i, 0, pre_mavg[i]) pass diff --git a/tests/system-test/7-tmq/schema.py b/tests/system-test/7-tmq/schema.py index 633a097db61ed876c9ebd1994c156c4f64c7ceb5..196b9a5323be642f2496df976eb1f61d1f0e6cb5 100644 --- a/tests/system-test/7-tmq/schema.py +++ b/tests/system-test/7-tmq/schema.py @@ -282,7 +282,7 @@ class TDTestCase: tdSql.execute("create topic %s as select ts, c1, c2, t1, t2 from %s.%s" %(columnTopicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(columnTopicFromNtb, parameterDict['dbName'], ntbName)) - tsLog.info("======== super table test:") + tdLog.info("======== super table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -316,7 +316,7 @@ class TDTestCase: tdSql.query("alter table %s.%s add tag t3 int"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s add tag t4 float"%(parameterDict['dbName'], parameterDict['stbName'])) - tsLog.info("======== normal table test:") + tdLog.info("======== normal table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], ntbName)) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], ntbName)) @@ -419,7 +419,7 @@ class TDTestCase: tdSql.execute("create topic %s as select ts, c1, c2, t1, t2 from %s.%s where c3 > 3 and c4 like 'abc' and t3 = 5 and t4 = 'beijing'" %(columnTopicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s where c3 > 3 and c4 like 'abc'" %(columnTopicFromNtb, parameterDict['dbName'], ntbName)) - tsLog.info("======== super table test:") + tdLog.info("======== super table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -457,7 +457,7 @@ class TDTestCase: tdSql.query("alter table %s.%s add column c5 int"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s add tag t5 float"%(parameterDict['dbName'], parameterDict['stbName'])) - tsLog.info("======== normal table test:") + tdLog.info("======== normal table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], ntbName)) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], ntbName)) @@ -566,7 +566,7 @@ class TDTestCase: tdSql.execute("create topic %s as select * from %s.%s" %(columnTopicFromStb, parameterDict['dbName'], parameterDict['stbName'])) tdSql.execute("create topic %s as select * from %s.%s " %(columnTopicFromNtb, parameterDict['dbName'], ntbName)) - tsLog.info("======== super table test:") + tdLog.info("======== super table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], parameterDict['stbName'])) @@ -601,7 +601,7 @@ class TDTestCase: tdSql.query("alter table %s.%s add column c6 int"%(parameterDict['dbName'], parameterDict['stbName'])) tdSql.query("alter table %s.%s add tag t6 float"%(parameterDict['dbName'], parameterDict['stbName'])) - tsLog.info("======== normal table test:") + tdLog.info("======== normal table test:") # alter actions prohibited: drop column/tag, modify column/tag type, rename column/tag included in topic tdSql.error("alter table %s.%s drop column c1"%(parameterDict['dbName'], ntbName)) tdSql.error("alter table %s.%s drop column c2"%(parameterDict['dbName'], ntbName)) @@ -687,8 +687,8 @@ class TDTestCase: tdLog.info("cfgPath: %s" % cfgPath) self.tmqCase1(cfgPath, buildPath) - self.tmqCase2(cfgPath, buildPath) - self.tmqCase3(cfgPath, buildPath) + # self.tmqCase2(cfgPath, buildPath) + # self.tmqCase3(cfgPath, buildPath) def stop(self): tdSql.close() diff --git a/tests/system-test/fulltest.bat b/tests/system-test/fulltest.bat index 871c93c9824333acb6ba05474d9249fb9f8d8ed7..ea57bc7427d90cb826eb925459ceaa3a805415b6 100644 --- a/tests/system-test/fulltest.bat +++ b/tests/system-test/fulltest.bat @@ -1,4 +1,7 @@ python3 .\test.py -f 0-others\taosShell.py python3 .\test.py -f 0-others\taosShellError.py -python3 .\test.py -f 0-others\taosShellNetChk.py \ No newline at end of file +python3 .\test.py -f 0-others\taosShellNetChk.py +python3 .\test.py -f 0-others\udfTest.py +python3 .\test.py -f 0-others\udf_create.py +python3 .\test.py -f 0-others\udf_restart_taosd.py \ No newline at end of file diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh old mode 100644 new mode 100755 index f03a07e8e105ff83bf9673b0667201b82512f4bf..3991654350ee35f34ec1f7325bcf0421d8ab4d1e --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -19,7 +19,8 @@ python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py #python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py - +python3 ./test.py -f 1-insert/alter_stable.py +python3 ./test.py -f 1-insert/alter_table.py python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py @@ -79,6 +80,7 @@ python3 ./test.py -f 2-query/query_cols_tags_and_or.py # TD-15983 subquery output duplicate name column. # Please Xiangyang Guo modify the following script # python3 ./test.py -f 2-query/nestedQuery_str.py + python3 ./test.py -f 2-query/avg.py python3 ./test.py -f 2-query/elapsed.py python3 ./test.py -f 2-query/csum.py @@ -101,3 +103,4 @@ python3 ./test.py -f 7-tmq/subscribeStb1.py python3 ./test.py -f 7-tmq/subscribeStb2.py python3 ./test.py -f 7-tmq/subscribeStb3.py python3 ./test.py -f 7-tmq/subscribeStb4.py +python3 ./test.py -f 7-tmq/db.py