diff --git a/Jenkinsfile b/Jenkinsfile index 2558df777b5698de59093316da0e18407a160d77..5faa02d0c3162c9b65aa1e9d5eac44de5118a582 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,7 +42,6 @@ def pre_test(){ killall -9 gdb || echo "no gdb running" killall -9 python3.8 || echo "no python program running" cd ${WKC} - git reset --hard HEAD~10 >/dev/null ''' script { if (env.CHANGE_TARGET == 'master') { @@ -75,11 +74,9 @@ def pre_test(){ git pull >/dev/null git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD - git clean -dfx - - git clean -dfx export TZ=Asia/Harbin date + rm -rf debug mkdir debug cd debug cmake .. > /dev/null diff --git a/cmake/cmake.options b/cmake/cmake.options index 2384a427e49cf17a7cce6a1066033bb5d004f167..44fa8c7e4b13bed3cf75a07c66c79054a0c253d0 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -44,6 +44,18 @@ option( OFF ) +option( + BUILD_WITH_UV + "If build with libuv" + OFF +) + +option( + BUILD_WITH_CRAFT + "If build with canonical-raft" + OFF +) + option( BUILD_DEPENDENCY_TESTS "If build dependency tests" diff --git a/cmake/craft_CMakeLists.txt.in b/cmake/craft_CMakeLists.txt.in new file mode 100644 index 0000000000000000000000000000000000000000..3a951c1c99abfb04e9bc7bb13aef55e7fb79a395 --- /dev/null +++ b/cmake/craft_CMakeLists.txt.in @@ -0,0 +1,14 @@ + +# canonical-raft +ExternalProject_Add(craft + GIT_REPOSITORY https://github.com/canonical/raft.git + GIT_TAG v0.11.2 + SOURCE_DIR "${CMAKE_CONTRIB_DIR}/craft" + BINARY_DIR "${CMAKE_CONTRIB_DIR}/craft" + #BUILD_IN_SOURCE TRUE + # https://answers.ros.org/question/333125/how-to-include-external-automakeautoconf-projects-into-ament_cmake/ + CONFIGURE_COMMAND COMMAND autoreconf -i COMMAND ./configure --enable-example + BUILD_COMMAND "$(MAKE)" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/cmake/libuv_CMakeLists.txt.in b/cmake/libuv_CMakeLists.txt.in new file mode 100644 index 0000000000000000000000000000000000000000..ed406e089e0f641f0a87c1a80fd5a501acbaf490 --- /dev/null +++ b/cmake/libuv_CMakeLists.txt.in @@ -0,0 +1,12 @@ + +# libuv +ExternalProject_Add(libuv + GIT_REPOSITORY https://github.com/libuv/libuv.git + GIT_TAG v1.42.0 + SOURCE_DIR "${CMAKE_CONTRIB_DIR}/libuv" + BINARY_DIR "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + ) \ No newline at end of file diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index cffe16448875618be68bcc17ce25571d0939b180..dfdde96172d30d7773e5841c1c261e7d52a87028 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -34,10 +34,21 @@ if(${BUILD_WITH_ROCKSDB}) add_definitions(-DUSE_ROCKSDB) endif(${BUILD_WITH_ROCKSDB}) +# canonical-raft +if(${BUILD_WITH_CRAFT}) + cat("${CMAKE_SUPPORT_DIR}/craft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + SET(BUILD_WITH_UV ON CACHE BOOL "craft need libuv" FORCE) +endif(${BUILD_WITH_CRAFT}) + +#libuv +if(${BUILD_WITH_UV}) + cat("${CMAKE_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif(${BUILD_WITH_UV}) + # bdb if(${BUILD_WITH_BDB}) cat("${CMAKE_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif(${BUILD_WITH_DBD}) +endif(${BUILD_WITH_BDB}) # sqlite if(${BUILD_WITH_SQLITE}) @@ -144,6 +155,23 @@ if(${BUILD_WITH_NURAFT}) add_subdirectory(nuraft) endif(${BUILD_WITH_NURAFT}) +# CRAFT +if(${BUILD_WITH_CRAFT}) + add_library(craft STATIC IMPORTED GLOBAL) + set_target_properties(craft PROPERTIES + IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/craft/.libs/libraft.a" + INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/craft/include" + ) + # target_link_libraries(craft + # INTERFACE pthread + # ) +endif(${BUILD_WITH_CRAFT}) + +# LIBUV +if(${BUILD_WITH_UV}) + add_subdirectory(libuv) +endif(${BUILD_WITH_UV}) + # BDB if(${BUILD_WITH_BDB}) add_library(bdb STATIC IMPORTED GLOBAL) diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 0a333f604c1ac20edb166e386fcbf42b55572661..330fe8f70f8d9bfcd9e09e77611ae67794cc86a0 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -15,4 +15,8 @@ if(${BUILD_WITH_SQLITE}) add_subdirectory(sqlite) endif(${BUILD_WITH_SQLITE}) +if(${BUILD_WITH_CRAFT}) + add_subdirectory(craft) +endif(${BUILD_WITH_CRAFT}) + add_subdirectory(tdev) diff --git a/contrib/test/craft/CMakeLists.txt b/contrib/test/craft/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..e0f6ae64bd3e9c998db15c1006cebf1b15702f5b --- /dev/null +++ b/contrib/test/craft/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(simulate_vnode "simulate_vnode.c") +target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) \ No newline at end of file diff --git a/contrib/test/craft/clear.sh b/contrib/test/craft/clear.sh new file mode 100644 index 0000000000000000000000000000000000000000..6412656d7744b5b5ae8a57b42e34d79849d508f9 --- /dev/null +++ b/contrib/test/craft/clear.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +rm -rf 127.0.0.1* diff --git a/contrib/test/craft/help.txt b/contrib/test/craft/help.txt new file mode 100644 index 0000000000000000000000000000000000000000..48ce9de40368abf55d29d31dd2d72bba28639556 --- /dev/null +++ b/contrib/test/craft/help.txt @@ -0,0 +1,15 @@ + + +make craft: + +sudo apt-get install libuv1-dev liblz4-dev +autoreconf -i +./configure --enable-example +make + + +start: + +./simulate_vnode 10000 10001 10002 +./simulate_vnode 10001 10000 10002 +./simulate_vnode 10002 10000 10001 diff --git a/contrib/test/craft/simulate_vnode.c b/contrib/test/craft/simulate_vnode.c new file mode 100644 index 0000000000000000000000000000000000000000..668fe638b719bacfb9f320b40767eac4fe6da71a --- /dev/null +++ b/contrib/test/craft/simulate_vnode.c @@ -0,0 +1,252 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +const char* exe_name; + +// simulate ------------------------ +typedef struct SVnode { + int vid; +} SVnode; + + +#define VNODE_COUNT 10 +SVnode vnodes[VNODE_COUNT]; + +int vnodeApplyWMsg(SVnode *pVnode, char *pMsg, void **pRsp) { + printf("put value to tsdb, vid:%d msg:%s \n", pVnode->vid, pMsg); + return 0; +} + +int applyCB(struct raft_fsm *fsm, + const struct raft_buffer *buf, + void **result) { + char *msg = (char*)buf->base; + //printf("%s \n", msg); + + // parse msg + char* context; + char* token = strtok_r(msg, ":", &context); + int vid = atoi(token); + + token = strtok_r(NULL, ":", &context); + char *value = token; + + SVnode* tmp_vnodes = (SVnode*)(fsm->data); + vnodeApplyWMsg(&tmp_vnodes[vid], value, NULL); + + return 0; +} + +// Config ------------------------ +#define HOST_LEN 32 +#define MAX_PEERS 10 +typedef struct Address { + char host[HOST_LEN]; + uint32_t port; +} Address; + +uint64_t raftId(Address *addr) { + // test in a single machine, port is unique + // if in multi machines, use host and port + return addr->port; +} + +typedef struct Config { + Address me; + Address peers[MAX_PEERS]; + int peer_count; +} Config; + +Config gConf; + +void printConf(Config *c) { + printf("me: %s:%u \n", c->me.host, c->me.port); + for (int i = 0; i < c->peer_count; ++i) { + printf("peer%d: %s:%u \n", i, c->peers[i].host, c->peers[i].port); + } +} + +// RaftServer ------------------------ +typedef struct RaftServer { + struct uv_loop_s loop; + struct raft_uv_transport transport; + struct raft_io io; + struct raft_fsm fsm; + struct raft raft; + struct raft_configuration conf; +} RaftServer; + +RaftServer gRaftServer; + +static void* startRaftServer(void *param) { + //RaftServer* rs = (RaftServer*)param; + RaftServer* rs = &gRaftServer; + raft_start(&rs->raft); + uv_run(&rs->loop, UV_RUN_DEFAULT); +} + +static const char* state2String(unsigned short state) { + if (state == RAFT_UNAVAILABLE) { + return "RAFT_UNAVAILABLE"; + + } else if (state == RAFT_FOLLOWER) { + return "RAFT_FOLLOWER"; + + } else if (state == RAFT_CANDIDATE) { + return "RAFT_CANDIDATE"; + + } else if (state == RAFT_LEADER) { + return "RAFT_LEADER"; + + } + return "UNKNOWN_RAFT_STATE"; +} + +static void printRaftState(struct raft *r) { + printf("\n"); + printf("my_id: %llu \n", r->id); + printf("address: %s \n", r->address); + printf("current_term: %llu \n", r->current_term); + printf("voted_for: %llu \n", r->voted_for); + printf("role: %s \n", state2String(r->state)); + printf("commit_index: %llu \n", r->commit_index); + printf("last_applied: %llu \n", r->last_applied); + printf("last_stored: %llu \n", r->last_stored); + printf("\n"); +} + +// console ----------------------------------------- +#define PROPOSE_VALUE_LEN 128 +static void proposeValue(struct raft *r) { + struct raft_buffer buf; + + // need free + buf.len = PROPOSE_VALUE_LEN; + buf.base = raft_malloc(buf.len); + + // mock ts value + int vid = rand() % VNODE_COUNT; + snprintf(buf.base, buf.len, "%d:value_%ld", vid, time(NULL)); + + printf("propose value: %s \n", (char*)buf.base); + + // need free + struct raft_apply *req = raft_malloc(sizeof(struct raft_apply)); + raft_apply(r, req, &buf, 1, NULL); +} + +static void* console(void *param) { + while (1) { + // notice! memory buffer overflow! + char buf[128]; + memset(buf, 0, sizeof(buf)); + fgets(buf, 128, stdin); + if (strlen(buf) == 1) { + continue; + } + buf[strlen(buf)-1] = '\0'; + + // do not use strcmp + if (strcmp(buf, "state") == 0) { + printRaftState(&gRaftServer.raft); + + } else if (strcmp(buf, "put") == 0) { + proposeValue(&gRaftServer.raft); + + } else { + printf("unknown command: [%s], support command: state, put \n", buf); + } + } +} + +// ----------------------------------------- +void usage() { + printf("\n"); + printf("%s my_port peer1_port peer2_port ... \n", exe_name); + printf("\n"); +} + +int main(int argc, char **argv) { + srand(time(NULL)); + + exe_name = argv[0]; + if (argc < 2) { + usage(); + exit(-1); + } + + // read conf from argv + strncpy(gConf.me.host, "127.0.0.1", HOST_LEN); + sscanf(argv[1], "%u", &gConf.me.port); + + gConf.peer_count = 0; + for (int i = 2; i < argc; ++i) { + strncpy(gConf.peers[gConf.peer_count].host, "127.0.0.1", HOST_LEN); + sscanf(argv[i], "%u", &gConf.peers[gConf.peer_count].port); + gConf.peer_count++; + } + printConf(&gConf); + + // mkdir + char dir[128]; + snprintf(dir, sizeof(dir), "./%s_%u", gConf.me.host, gConf.me.port); + + char cmd[128]; + snprintf(cmd, sizeof(cmd), "rm -rf ./%s", dir); + system(cmd); + snprintf(cmd, sizeof(cmd), "mkdir -p ./%s", dir); + system(cmd); + + // init io + uv_loop_init(&gRaftServer.loop); + raft_uv_tcp_init(&gRaftServer.transport, &gRaftServer.loop); + raft_uv_init(&gRaftServer.io, &gRaftServer.loop, dir, &gRaftServer.transport); + + // init fsm + gRaftServer.fsm.apply = applyCB; + gRaftServer.fsm.data = vnodes; + for (int i = 0; i < VNODE_COUNT; ++i) { + vnodes[i].vid = i; + } + + // init raft instance with io and fsm + char address_buf[128]; + snprintf(address_buf, sizeof(address_buf), "%s:%u", gConf.me.host, gConf.me.port); + + // test in a single machine, port is unique + uint64_t raft_id = raftId(&gConf.me); + raft_init(&gRaftServer.raft, &gRaftServer.io, &gRaftServer.fsm, raft_id, address_buf); + //raft_init(&gRaftServer.raft, &gRaftServer.io, &gRaftServer.fsm, 11, "127.0.0.1:9000"); + + // init cluster configuration + struct raft_configuration conf; + raft_configuration_init(&conf); + raft_configuration_add(&conf, raftId(&gConf.me), address_buf, RAFT_VOTER); + for (int i = 0; i < gConf.peer_count; ++i) { + char address_buf[128]; + snprintf(address_buf, sizeof(address_buf), "%s:%u", gConf.peers[i].host, gConf.peers[i].port); + raft_configuration_add(&conf, raftId(&gConf.peers[i]), address_buf, RAFT_VOTER); + } + raft_bootstrap(&gRaftServer.raft, &conf); + + // start raft server and loop + pthread_t tid; + pthread_create(&tid, NULL, startRaftServer, &gRaftServer); + + // simulate console + pthread_t tid2; + pthread_create(&tid2, NULL, console, NULL); + + while (1) { + sleep(10); + } + + return 0; +} diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 20b028aac25cd24a0a93fb524317c610d3166ec5..aef7d5711fb1b73d2542ebe34b9c77c636827807 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -50,7 +50,8 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONSUME, "mq-consume" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET, "mq-set" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET_CUR, "mq-set-cur" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RSP_READY, "rsp-ready" ) // message from client to mnode TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" ) @@ -223,6 +224,7 @@ typedef struct SBuildUseDBInput { int32_t vgVersion; } SBuildUseDBInput; + #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta @@ -232,9 +234,9 @@ typedef struct { } SEpAddrMsg; typedef struct { - char *fqdn; + char fqdn[TSDB_FQDN_LEN]; uint16_t port; -} SEpAddr1; +} SEpAddr; typedef struct { int32_t numOfVnodes; @@ -341,7 +343,7 @@ typedef struct { typedef struct { char tableFname[TSDB_TABLE_FNAME_LEN]; char db[TSDB_FULL_DB_NAME_LEN]; - int16_t type; /* operation type */ + int16_t type; /* operation type */ int16_t numOfCols; /* number of schema */ int32_t tagValLen; SSchema schema[]; @@ -543,8 +545,8 @@ typedef struct { int32_t sqlstrLen; // sql query string int32_t prevResultLen; // previous result length int32_t numOfOperator; - int32_t tableScanOperator; // table scan operator. -1 means no scan operator - int32_t udfNum; // number of udf function + int32_t tableScanOperator;// table scan operator. -1 means no scan operator + int32_t udfNum; // number of udf function int32_t udfContentOffset; int32_t udfContentLen; SColumnInfo tableCols[]; @@ -1004,27 +1006,35 @@ typedef struct { // mq related typedef struct { + } SMqConnectReq; typedef struct { + } SMqConnectRsp; typedef struct { + } SMqDisconnectReq; typedef struct { + } SMqDisconnectRsp; typedef struct { + } SMqAckReq; typedef struct { + } SMqAckRsp; typedef struct { + } SMqResetReq; typedef struct { + } SMqResetRsp; // mq related end @@ -1080,6 +1090,24 @@ typedef struct { /* data */ } SUpdateTagValRsp; +typedef struct SSchedulerQueryMsg { + uint64_t queryId; + uint64_t taskId; + uint32_t contentLen; + char msg[]; +} SSchedulerQueryMsg; + +typedef struct SSchedulerReadyMsg { + uint64_t queryId; + uint64_t taskId; +} SSchedulerReadyMsg; + +typedef struct SSchedulerFetchMsg { + uint64_t queryId; + uint64_t taskId; +} SSchedulerFetchMsg; + + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgtype.h b/include/common/tmsgtype.h index 1fb10ae15b692d21f8a07937393399fe49664ce3..09765031155679accc6f6d4043e23b3b89409135 100644 --- a/include/common/tmsgtype.h +++ b/include/common/tmsgtype.h @@ -35,7 +35,7 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SELECT, "select" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_FETCH, "fetch" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_INSERT, "insert" ) - TSDB_DEFINE_SQL_TYPE( TSDB_SQL_UPDATE_TAGS_VAL, "update-tag-val" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_UPDATE_TAG_VAL, "update-tag-val" ) // the SQL below is for mgmt node TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" ) @@ -54,7 +54,7 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_TABLE, "alter-table" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_DB, "alter-db" ) - TSDB_DEFINE_SQL_TYPE(TSDB_SQL_SYNC_DB_REPLICA, "sync db-replica") + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SYNC_DB_REPLICA, "sync db-replica") TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_MNODE, "create-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_MNODE, "drop-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DNODE, "create-dnode" ) diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index 7993a8f1abdb2ad2f36964bfd861371ff96976dd..60a8c252c0b511e145e611814827626a48ed86bd 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -16,86 +16,76 @@ #ifndef _TD_TQ_H_ #define _TD_TQ_H_ +#include "common.h" #include "mallocator.h" #include "os.h" +#include "taoserror.h" +#include "taosmsg.h" +#include "tlist.h" #include "tutil.h" #ifdef __cplusplus extern "C" { #endif -typedef struct TmqMsgHead { +typedef struct STqMsgHead { int32_t protoVer; int32_t msgType; int64_t cgId; int64_t clientId; -} TmqMsgHead; +} STqMsgHead; -typedef struct TmqOneAck { +typedef struct STqOneAck { int64_t topicId; int64_t consumeOffset; -} TmqOneAck; +} STqOneAck; -typedef struct TmqAcks { +typedef struct STqAcks { int32_t ackNum; // should be sorted - TmqOneAck acks[]; -} TmqAcks; + STqOneAck acks[]; +} STqAcks; -// TODO: put msgs into common -typedef struct TmqConnectReq { - TmqMsgHead head; - TmqAcks acks; -} TmqConnectReq; - -typedef struct TmqConnectRsp { - TmqMsgHead head; - int8_t status; -} TmqConnectRsp; - -typedef struct TmqDisconnectReq { - TmqMsgHead head; -} TmqDiscconectReq; - -typedef struct TmqDisconnectRsp { - TmqMsgHead head; - int8_t status; -} TmqDisconnectRsp; +typedef struct STqSetCurReq { + STqMsgHead head; + int64_t topicId; + int64_t offset; +} STqSetCurReq; typedef struct STqConsumeReq { - TmqMsgHead head; - TmqAcks acks; + STqMsgHead head; + STqAcks acks; } STqConsumeReq; -typedef struct TmqMsgContent { +typedef struct STqMsgContent { int64_t topicId; int64_t msgLen; char msg[]; -} TmqMsgContent; +} STqMsgContent; typedef struct STqConsumeRsp { - TmqMsgHead head; + STqMsgHead head; int64_t bodySize; - TmqMsgContent msgs[]; + STqMsgContent msgs[]; } STqConsumeRsp; -typedef struct TmqSubscribeReq { - TmqMsgHead head; +typedef struct STqSubscribeReq { + STqMsgHead head; int32_t topicNum; int64_t topic[]; -} TmqSubscribeReq; +} STqSubscribeReq; -typedef struct tmqSubscribeRsp { - TmqMsgHead head; +typedef struct STqSubscribeRsp { + STqMsgHead head; int64_t vgId; char ep[TSDB_EP_LEN]; // TSDB_EP_LEN -} TmqSubscribeRsp; +} STqSubscribeRsp; -typedef struct TmqHeartbeatReq { -} TmqHeartbeatReq; +typedef struct STqHeartbeatReq { +} STqHeartbeatReq; -typedef struct TmqHeartbeatRsp { -} TmqHeartbeatRsp; +typedef struct STqHeartbeatRsp { +} STqHeartbeatRsp; typedef struct STqTopicVhandle { int64_t topicId; @@ -108,48 +98,54 @@ typedef struct STqTopicVhandle { #define TQ_BUFFER_SIZE 8 +typedef struct STqExec { + void* runtimeEnv; + SSDataBlock* (*exec)(void* runtimeEnv); + void* (*assign)(void* runtimeEnv, SSubmitBlk* inputData); + void (*clear)(void* runtimeEnv); + char* (*serialize)(struct STqExec*); + struct STqExec* (*deserialize)(char*); +} STqExec; + typedef struct STqBufferItem { int64_t offset; // executors are identical but not concurrent // so there must be a copy in each item - void* executor; - int64_t size; - void* content; -} STqBufferItem; + STqExec* executor; + int32_t status; + int64_t size; + void* content; +} STqMsgItem; -typedef struct STqBufferHandle { +typedef struct STqTopic { // char* topic; //c style, end with '\0' // int64_t cgId; // void* ahandle; - int64_t nextConsumeOffset; - int64_t floatingCursor; - int64_t topicId; - int32_t head; - int32_t tail; - STqBufferItem buffer[TQ_BUFFER_SIZE]; -} STqBufferHandle; + int64_t nextConsumeOffset; + int64_t floatingCursor; + int64_t topicId; + int32_t head; + int32_t tail; + STqMsgItem buffer[TQ_BUFFER_SIZE]; +} STqTopic; typedef struct STqListHandle { - STqBufferHandle bufHandle; + STqTopic topic; struct STqListHandle* next; -} STqListHandle; - -typedef struct STqGroupHandle { - int64_t cId; - int64_t cgId; - void* ahandle; - int32_t topicNum; - STqListHandle* head; -} STqGroupHandle; - -typedef struct STqQueryExec { - void* src; - STqBufferItem* dest; - void* executor; -} STqQueryExec; +} STqList; + +typedef struct STqGroup { + int64_t clientId; + int64_t cgId; + void* ahandle; + int32_t topicNum; + STqList* head; + SList* topicList; // SList + void* returnMsg; // SVReadMsg +} STqGroup; typedef struct STqQueryMsg { - STqQueryExec* exec; + STqMsgItem* item; struct STqQueryMsg* next; } STqQueryMsg; @@ -209,15 +205,15 @@ typedef void (*FTqDelete)(void*); #define TQ_DUP_INTXN_REWRITE 0 #define TQ_DUP_INTXN_REJECT 2 -static inline bool TqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } +static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } -static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } +static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; #define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE -typedef struct TqMetaHandle { +typedef struct STqMetaHandle { int64_t key; int64_t offset; int64_t serializedSize; @@ -225,23 +221,25 @@ typedef struct TqMetaHandle { void* valueInTxn; } STqMetaHandle; -typedef struct TqMetaList { - STqMetaHandle handle; - struct TqMetaList* next; - // struct TqMetaList* inTxnPrev; - // struct TqMetaList* inTxnNext; - struct TqMetaList* unpersistPrev; - struct TqMetaList* unpersistNext; +typedef struct STqMetaList { + STqMetaHandle handle; + struct STqMetaList* next; + // struct STqMetaList* inTxnPrev; + // struct STqMetaList* inTxnNext; + struct STqMetaList* unpersistPrev; + struct STqMetaList* unpersistNext; } STqMetaList; -typedef struct TqMetaStore { +typedef struct STqMetaStore { STqMetaList* bucket[TQ_BUCKET_SIZE]; // a table head STqMetaList* unpersistHead; + // TODO:temporaral use, to be replaced by unified tfile int fileFd; // TODO:temporaral use, to be replaced by unified tfile - int idxFd; + int idxFd; + char* dirPath; int32_t tqConfigFlag; FTqSerialize pSerializer; @@ -250,8 +248,8 @@ typedef struct TqMetaStore { } STqMetaStore; typedef struct STQ { - // the collection of group handle - // the handle of kvstore + // the collection of groups + // the handle of meta kvstore char* path; STqCfg* tqConfig; STqLogReader* tqLogReader; @@ -266,23 +264,25 @@ void tqClose(STQ*); // void* will be replace by a msg type int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); -int tqSetCursor(STQ*, void* msg); - int tqConsume(STQ*, STqConsumeReq*); -STqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId); +int tqSetCursor(STQ*, STqSetCurReq* pMsg); +int tqBufferSetOffset(STqTopic*, int64_t offset); + +STqTopic* tqFindTopic(STqGroup*, int64_t topicId); + +STqGroup* tqGetGroup(STQ*, int64_t clientId); + +STqGroup* tqOpenGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); +int tqCloseGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); +int tqRegisterContext(STqGroup*, void* ahandle); +int tqSendLaunchQuery(STqMsgItem*, int64_t offset); -STqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqMoveOffsetToNext(STqGroupHandle*); -int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset); -int tqRegisterContext(STqGroupHandle*, void* ahandle); -int tqLaunchQuery(STqGroupHandle*); -int tqSendLaunchQuery(STqGroupHandle*); +int tqSerializeGroup(const STqGroup*, STqSerializedHead**); -int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead); +const void* tqDeserializeGroup(const STqSerializedHead*, STqGroup**); -const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** gHandle); +static int tqQueryExecuting(int32_t status) { return status; } #ifdef __cplusplus } diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index ee626865fb85e3da5d799cb67a62ac3bc94d1ee0..68eae03f518441aad3aeb17837045cb66b34a92d 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -137,7 +137,7 @@ int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void *pRpc, const S int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); -int32_t catalogGetQnodeList(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet); +int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet); diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 49adaecfdda676ea4173d199977d469d7a7f3e6a..8e26fa98c945973bc6ef90aa262f2b9be20fe2b8 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -209,6 +209,7 @@ typedef struct SSourceParam { SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, const char* funcName, SSourceParam* pSource, SSchema* pResSchema, int16_t interSize); int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); +int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy); int32_t getExprFunctionLevel(SQueryStmtInfo* pQueryInfo); STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 3f4d21a7461d928afb7f2bff18a016da3296dd8e..8b54b88b28fd908bf2ad9040918cdcf9a4c4eea0 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -141,8 +141,8 @@ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* // Set datasource of this subplan, multiple calls may be made to a subplan. // @subplan subplan to be schedule // @templateId templateId of a group of datasource subplans of this @subplan -// @eps Execution location of this group of datasource subplans, is an array of SEpAddr structures -int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps); +// @ep one execution location of this group of datasource subplans +int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SEpAddr* ep); int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str); diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 40c85520d4349fbd9ea39f84c2eabd6d78eab17b..2cbf26f8772927f67c3a882465cd66414be20a93 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -21,9 +21,10 @@ extern "C" { #endif #include "planner.h" +#include "catalog.h" typedef struct SSchedulerCfg { - + int32_t clusterType; } SSchedulerCfg; typedef struct SQueryProfileSummary { @@ -49,15 +50,17 @@ typedef struct SQueryProfileSummary { uint64_t resultSize; // generated result size in Kb. } SQueryProfileSummary; +int32_t schedulerInit(SSchedulerCfg *cfg); + /** * Process the query job, generated according to the query physical plan. * This is a synchronized API, and is also thread-safety. - * @param pJob + * @param qnodeList Qnode address list, element is SEpAddr * @return */ -int32_t scheduleQueryJob(SQueryDag* pDag, void** pJob); +int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob); -int32_t scheduleFetchRows(void *pJob, void *data); +int32_t scheduleFetchRows(void *pJob, void **data); /** @@ -69,6 +72,8 @@ int32_t scheduleCancelJob(void *pJob); void scheduleFreeJob(void *pJob); +void schedulerDestroy(void); + #ifdef __cplusplus } #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ad4b383d03411293ba70f9817f2fadc2ec8cbcd1..95824df5ac9adea0a260012369df2dd0fe4fe73e 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -339,6 +339,20 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) //"Invalid msg length") #define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) //"Invalid msg type") +// tq +#define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00) //"Invalid configuration") +#define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) //"Tq init failed") +#define TSDB_CODE_TQ_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0A02) //"No diskspace for tq") +#define TSDB_CODE_TQ_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0A03) //"No permission for disk files") +#define TSDB_CODE_TQ_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0A04) //"Data file(s) corrupted") +#define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) //"Out of memory") +#define TSDB_CODE_TQ_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0A06) //"File already exists") +#define TSDB_CODE_TQ_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0A07) //"Failed to create dir") +#define TSDB_CODE_TQ_META_NO_SUCH_KEY TAOS_DEF_ERROR_CODE(0, 0x0A08) //"Target key not found") +#define TSDB_CODE_TQ_META_KEY_NOT_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A09) //"Target key not in transaction") +#define TSDB_CODE_TQ_META_KEY_DUP_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A0A) //"Target key duplicated in transaction") +#define TSDB_CODE_TQ_GROUP_NOT_SET TAOS_DEF_ERROR_CODE(0, 0x0A0B) //"Group of corresponding client is not set by mnode") + // wal #define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) //"Unexpected generic error in wal") #define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) //"WAL file is corrupted") @@ -493,6 +507,7 @@ int32_t* taosGetErrno(); //scheduler #define TSDB_CODE_SCH_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2501) //scheduler status error +#define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502) //scheduler internal error diff --git a/include/util/tlog.h b/include/util/tlog.h index 5e6604598dd22ad7755e4af51921fa3ee80a6abe..5c91398cdcf952046406d55e15911109a4c66435 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -42,11 +42,11 @@ extern int32_t qDebugFlag; extern int32_t wDebugFlag; extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; +extern int32_t tqDebugFlag; extern int32_t cqDebugFlag; extern int32_t debugFlag; extern int32_t ctgDebugFlag; - #define DEBUG_FATAL 1U #define DEBUG_ERROR DEBUG_FATAL #define DEBUG_WARN 2U diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index b6200de824822170e2827b041b1c768c3e7a48c1..9532789ca0e14d7871545b0bb1c54747ddec3608 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -153,7 +153,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { void* output = NULL; int32_t outputLen = 0; code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE); - if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_CREATE_DB) { + if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_DROP_ACCT || type == TSDB_SQL_CREATE_DB || type == TSDB_SQL_CREATE_ACCT) { pRequest->type = type; pRequest->body.param = output; pRequest->body.paramLen = outputLen; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 36fa013f27e0b31f7d0124a3f6e8feccd19d345a..f8fd69b89a45e0cfb697ddf4584dd602a9b6be9f 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -407,7 +407,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { } // single table query error need to be handled here. - if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && + if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAG_VAL) && (((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) || rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { @@ -3176,6 +3176,15 @@ int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { case TSDB_SQL_CREATE_USER: pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER; break; + case TSDB_SQL_DROP_USER: + pMsgBody->msgType = TSDB_MSG_TYPE_DROP_USER; + break; + case TSDB_SQL_CREATE_ACCT: + pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_ACCT; + break; + case TSDB_SQL_DROP_ACCT: + pMsgBody->msgType = TSDB_MSG_TYPE_DROP_ACCT; + break; case TSDB_SQL_CREATE_DB: { pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_DB; @@ -3328,7 +3337,7 @@ void initMsgHandleFp() { tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; - tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAG_VAL] = tscBuildUpdateTagMsg; tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; @@ -3383,6 +3392,10 @@ void initMsgHandleFp() { handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp; buildRequestMsgFp[TSDB_SQL_CREATE_USER] = doBuildMsgSupp; + buildRequestMsgFp[TSDB_SQL_DROP_USER] = doBuildMsgSupp; + + buildRequestMsgFp[TSDB_SQL_CREATE_ACCT] = doBuildMsgSupp; + buildRequestMsgFp[TSDB_SQL_DROP_ACCT] = doBuildMsgSupp; buildRequestMsgFp[TSDB_SQL_SHOW] = doBuildMsgSupp; handleRequestRspFp[TSDB_SQL_SHOW] = processShowRsp; diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c index 023bd6ebe977c956f201d45fb63da31066d33ee2..76c37ca2f1734e0e79dc025f9678df48c8fb8e24 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/tscEnv.c @@ -189,7 +189,6 @@ static void doDestroyRequest(void* p) { tfree(pRequest->pInfo); if (pRequest->body.pResInfo != NULL) { - tfree(pRequest->body.pResInfo->pData); tfree(pRequest->body.pResInfo->pMsg); tfree(pRequest->body.pResInfo); } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 944247e88e57901d57bdaf18a1025c53b08ad107..1c8cd6a4b6da560f0deec532c008863b6e7eb786 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -40,13 +40,13 @@ TEST(testCase, driverInit_Test) { TEST(testCase, connect_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); - assert(pConn != NULL); +// assert(pConn != NULL); taos_close(pConn); } TEST(testCase, create_user_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); - assert(pConn != NULL); +// assert(pConn != NULL); TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { @@ -57,23 +57,36 @@ TEST(testCase, create_user_Test) { taos_close(pConn); } -//TEST(testCase, drop_user_Test) { -// TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} +TEST(testCase, create_account_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); -TEST(testCase, show_user_Test) { + TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_account_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_user_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "show users"); TAOS_ROW pRow = NULL; @@ -89,10 +102,23 @@ TEST(testCase, show_user_Test) { taos_close(pConn); } -TEST(testCase, show_db_Test) { +TEST(testCase, drop_user_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "drop user abc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_db_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "show databases"); TAOS_ROW pRow = NULL; @@ -112,7 +138,7 @@ TEST(testCase, create_db_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); - TAOS_RES* pRes = taos_query(pConn, "create database abc"); + TAOS_RES* pRes = taos_query(pConn, "create database abc1"); TAOS_FIELD* pFields = taos_fetch_fields(pRes); ASSERT_TRUE(pFields == NULL); diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 3402d3ff68f398329109d73813d8538b2347f1f0..50b1a1cf20fa2db10c401911aff6b8178f374d64 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -44,7 +44,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET_CUR] = dndProcessVnodeWriteMsg; // msg from client to mnode pMgmt->msgFp[TSDB_MSG_TYPE_CONNECT] = dndProcessMnodeReadMsg; diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/impl/src/vnodeWrite.c index 85e044266ab368fac124c2f38d4008499e51b70e..fc977ce03f6150e58ec324949c2fc7263a8f26e2 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/impl/src/vnodeWrite.c @@ -16,17 +16,13 @@ #include "vnodeDef.h" int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { - SVnodeReq *pVnodeReq; - switch (pMsg->msgType) { - case TSDB_MSG_TYPE_MQ_SET: + case TSDB_MSG_TYPE_MQ_SET_CUR: if (tqSetCursor(pVnode->pTq, pMsg->pCont) < 0) { // TODO: handle error } break; } - - void *pBuf = pMsg->pCont; return 0; } diff --git a/source/dnode/vnode/tq/CMakeLists.txt b/source/dnode/vnode/tq/CMakeLists.txt index 536e97d5f79b58442b2d0a6af5758bb4ec854fc1..8d59c7b07a9c3935187ba6b644d7f95f88179776 100644 --- a/source/dnode/vnode/tq/CMakeLists.txt +++ b/source/dnode/vnode/tq/CMakeLists.txt @@ -11,6 +11,7 @@ target_link_libraries( PUBLIC wal PUBLIC os PUBLIC util + PUBLIC common ) if(${BUILD_TEST}) diff --git a/source/dnode/vnode/tq/inc/tqInt.h b/source/dnode/vnode/tq/inc/tqInt.h index 022b5998162b907382643d1e01a8cf0b05596696..5685a29d03e7dc15a974d9728d0c6ec07d4cceb3 100644 --- a/source/dnode/vnode/tq/inc/tqInt.h +++ b/source/dnode/vnode/tq/inc/tqInt.h @@ -17,11 +17,20 @@ #define _TD_TQ_INT_H_ #include "tq.h" - +#include "tlog.h" #ifdef __cplusplus extern "C" { #endif +extern int32_t tqDebugFlag; + +#define tqFatal(...) { if (tqDebugFlag & DEBUG_FATAL) { taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); }} +#define tqError(...) { if (tqDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); }} +#define tqWarn(...) { if (tqDebugFlag & DEBUG_WARN) { taosPrintLog("TQ WARN ", 255, __VA_ARGS__); }} +#define tqInfo(...) { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", 255, __VA_ARGS__); }} +#define tqDebug(...) { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }} +#define tqTrace(...) { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }} + // create persistent storage for meta info such as consuming offset // return value > 0: cgId // return value <= 0: error code diff --git a/source/dnode/vnode/tq/inc/tqMetaStore.h b/source/dnode/vnode/tq/inc/tqMetaStore.h index 5bcedaed748d7d0668181183d09de7470ff630e2..ef71d8bf145ae535edf7740e864661e576078814 100644 --- a/source/dnode/vnode/tq/inc/tqMetaStore.h +++ b/source/dnode/vnode/tq/inc/tqMetaStore.h @@ -17,7 +17,7 @@ #define _TQ_META_STORE_H_ #include "os.h" -#include "tq.h" +#include "tqInt.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index 1a27870a1b738d52f1c142f010e8e7e44abce50d..d8dfe4ddcf16e0f91988ae6a637376f709f73104 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -24,89 +24,80 @@ // handle management message // -int tqGetgHandleSSize(const STqGroupHandle* gHandle); -int tqBufHandleSSize(); -int tqBufItemSSize(); +int tqGroupSSize(const STqGroup* pGroup); +int tqTopicSSize(); +int tqItemSSize(); -STqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - STqGroupHandle* gHandle; - return NULL; -} +void* tqSerializeListHandle(STqList* listHandle, void* ptr); +void* tqSerializeTopic(STqTopic* pTopic, void* ptr); +void* tqSerializeItem(STqMsgItem* pItem, void* ptr); -void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr); -void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr); -void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr); - -const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle); -const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem); +const void* tqDeserializeTopic(const void* pBytes, STqTopic* pTopic); +const void* tqDeserializeItem(const void* pBytes, STqMsgItem* pItem); STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac) { STQ* pTq = malloc(sizeof(STQ)); if (pTq == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } pTq->path = strdup(path); pTq->tqConfig = tqConfig; pTq->tqLogReader = tqLogReader; pTq->tqMemRef.pAlloctorFactory = allocFac; - // pTq->tqMemRef.pAllocator = allocFac->create(allocFac); + pTq->tqMemRef.pAllocator = allocFac->create(allocFac); if (pTq->tqMemRef.pAllocator == NULL) { - // TODO + // TODO: error code of buffer pool } - pTq->tqMeta = - tqStoreOpen(path, (FTqSerialize)tqSerializeGroupHandle, (FTqDeserialize)tqDeserializeGroupHandle, free, 0); + pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeGroup, (FTqDeserialize)tqDeserializeGroup, free, 0); if (pTq->tqMeta == NULL) { // TODO: free STQ return NULL; } return pTq; } - -void tqClose(STQ*pTq) { +void tqClose(STQ* pTq) { // TODO } -static int tqProtoCheck(TmqMsgHead *pMsg) { - return pMsg->protoVer == 0; -} +static int tqProtoCheck(STqMsgHead* pMsg) { return pMsg->protoVer == 0; } -static int tqAckOneTopic(STqBufferHandle* bHandle, TmqOneAck* pAck, STqQueryMsg** ppQuery) { +static int tqAckOneTopic(STqTopic* pTopic, STqOneAck* pAck, STqQueryMsg** ppQuery) { // clean old item and move forward int32_t consumeOffset = pAck->consumeOffset; int idx = consumeOffset % TQ_BUFFER_SIZE; - ASSERT(bHandle->buffer[idx].content && bHandle->buffer[idx].executor); - tfree(bHandle->buffer[idx].content); + ASSERT(pTopic->buffer[idx].content && pTopic->buffer[idx].executor); + tfree(pTopic->buffer[idx].content); if (1 /* TODO: need to launch new query */) { STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg)); if (pNewQuery == NULL) { - // TODO: memory insufficient + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } // TODO: lock executor - pNewQuery->exec->executor = bHandle->buffer[idx].executor; // TODO: read from wal and assign to src - pNewQuery->exec->src = 0; - pNewQuery->exec->dest = &bHandle->buffer[idx]; - pNewQuery->next = *ppQuery; - *ppQuery = pNewQuery; + /*pNewQuery->exec->executor = pTopic->buffer[idx].executor;*/ + /*pNewQuery->exec->src = 0;*/ + /*pNewQuery->exec->dest = &pTopic->buffer[idx];*/ + /*pNewQuery->next = *ppQuery;*/ + /**ppQuery = pNewQuery;*/ } return 0; } -static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) { +static int tqAck(STqGroup* pGroup, STqAcks* pAcks) { int32_t ackNum = pAcks->ackNum; - TmqOneAck* acks = pAcks->acks; + STqOneAck* acks = pAcks->acks; // double ptr for acks and list - int i = 0; - STqListHandle* node = gHandle->head; - int ackCnt = 0; - STqQueryMsg* pQuery = NULL; + int i = 0; + STqList* node = pGroup->head; + int ackCnt = 0; + STqQueryMsg* pQuery = NULL; while (i < ackNum && node->next) { - if (acks[i].topicId == node->next->bufHandle.topicId) { + if (acks[i].topicId == node->next->topic.topicId) { ackCnt++; - tqAckOneTopic(&node->next->bufHandle, &acks[i], &pQuery); - } else if (acks[i].topicId < node->next->bufHandle.topicId) { + tqAckOneTopic(&node->next->topic, &acks[i], &pQuery); + } else if (acks[i].topicId < node->next->topic.topicId) { i++; } else { node = node->next; @@ -118,52 +109,56 @@ static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) { return ackCnt; } -static int tqCommitTCGroup(STqGroupHandle* handle) { +static int tqCommitGroup(STqGroup* pGroup) { // persist modification into disk return 0; } -int tqCreateTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroupHandle** handle) { +int tqCreateGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroup** ppGroup) { // create in disk - STqGroupHandle* gHandle = (STqGroupHandle*)malloc(sizeof(STqGroupHandle)); - if (gHandle == NULL) { + STqGroup* pGroup = (STqGroup*)malloc(sizeof(STqGroup)); + if (pGroup == NULL) { // TODO return -1; } - memset(gHandle, 0, sizeof(STqGroupHandle)); + *ppGroup = pGroup; + memset(pGroup, 0, sizeof(STqGroup)); return 0; } -STqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - STqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId); - if (gHandle == NULL) { - int code = tqCreateTCGroup(pTq, topicId, cgId, cId, &gHandle); - if (code != 0) { +STqGroup* tqOpenGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { + STqGroup* pGroup = tqHandleGet(pTq->tqMeta, cId); + if (pGroup == NULL) { + int code = tqCreateGroup(pTq, topicId, cgId, cId, &pGroup); + if (code < 0) { // TODO return NULL; } + tqHandleMovePut(pTq->tqMeta, cId, pGroup); } + ASSERT(pGroup); - // create - // open - return gHandle; + return pGroup; } -int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { return 0; } +int tqCloseGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { + // TODO + return 0; +} -int tqDropTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { +int tqDropGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { // delete from disk return 0; } -static int tqFetch(STqGroupHandle* gHandle, void** msg) { - STqListHandle* head = gHandle->head; - STqListHandle* node = head; - int totSize = 0; +static int tqFetch(STqGroup* pGroup, void** msg) { + STqList* head = pGroup->head; + STqList* node = head; + int totSize = 0; // TODO: make it a macro int sizeLimit = 4 * 1024; - TmqMsgContent* buffer = malloc(sizeLimit); + STqMsgContent* buffer = malloc(sizeLimit); if (buffer == NULL) { // TODO:memory insufficient return -1; @@ -172,25 +167,25 @@ static int tqFetch(STqGroupHandle* gHandle, void** msg) { // until all topic iterated or msgs over sizeLimit while (node->next) { node = node->next; - STqBufferHandle* bufHandle = &node->bufHandle; - int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE; - if (bufHandle->buffer[idx].content != NULL && bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset) { - totSize += bufHandle->buffer[idx].size; + STqTopic* topicHandle = &node->topic; + int idx = topicHandle->nextConsumeOffset % TQ_BUFFER_SIZE; + if (topicHandle->buffer[idx].content != NULL && topicHandle->buffer[idx].offset == topicHandle->nextConsumeOffset) { + totSize += topicHandle->buffer[idx].size; if (totSize > sizeLimit) { void* ptr = realloc(buffer, totSize); if (ptr == NULL) { - totSize -= bufHandle->buffer[idx].size; + totSize -= topicHandle->buffer[idx].size; // TODO:memory insufficient // return msgs already copied break; } } - *((int64_t*)buffer) = bufHandle->topicId; + *((int64_t*)buffer) = topicHandle->topicId; buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - *((int64_t*)buffer) = bufHandle->buffer[idx].size; + *((int64_t*)buffer) = topicHandle->buffer[idx].size; buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - memcpy(buffer, bufHandle->buffer[idx].content, bufHandle->buffer[idx].size); - buffer = POINTER_SHIFT(buffer, bufHandle->buffer[idx].size); + memcpy(buffer, topicHandle->buffer[idx].content, topicHandle->buffer[idx].size); + buffer = POINTER_SHIFT(buffer, topicHandle->buffer[idx].size); if (totSize > sizeLimit) { break; } @@ -199,11 +194,19 @@ static int tqFetch(STqGroupHandle* gHandle, void** msg) { return totSize; } -STqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { return NULL; } - -int tqLaunchQuery(STqGroupHandle* gHandle) { return 0; } +STqGroup* tqGetGroup(STQ* pTq, int64_t clientId) { return tqHandleGet(pTq->tqMeta, clientId); } -int tqSendLaunchQuery(STqGroupHandle* gHandle) { return 0; } +int tqSendLaunchQuery(STqMsgItem* bufItem, int64_t offset) { + if (tqQueryExecuting(bufItem->status)) { + return 0; + } + bufItem->status = 1; + // load data from wal or buffer pool + // put into exec + // send exec into non blocking queue + // when query finished, put into buffer pool + return 0; +} /*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/ /*return 0;*/ @@ -220,23 +223,96 @@ int tqCommit(STQ* pTq) { return 0; } -int tqSetCursor(STQ* pTq, void* msg) { +int tqBufferSetOffset(STqTopic* pTopic, int64_t offset) { + int code; + memset(pTopic->buffer, 0, sizeof(pTopic->buffer)); + // launch query + for (int i = offset; i < offset + TQ_BUFFER_SIZE; i++) { + int pos = i % TQ_BUFFER_SIZE; + code = tqSendLaunchQuery(&pTopic->buffer[pos], offset); + if (code < 0) { + // TODO: error handling + } + } + // set offset + pTopic->nextConsumeOffset = offset; + pTopic->floatingCursor = offset; + return 0; +} + +STqTopic* tqFindTopic(STqGroup* pGroup, int64_t topicId) { + // TODO + return NULL; +} + +int tqSetCursor(STQ* pTq, STqSetCurReq* pMsg) { + int code; + int64_t clientId = pMsg->head.clientId; + int64_t topicId = pMsg->topicId; + int64_t offset = pMsg->offset; + STqGroup* gHandle = tqGetGroup(pTq, clientId); + if (gHandle == NULL) { + // client not connect + return -1; + } + STqTopic* topicHandle = tqFindTopic(gHandle, topicId); + if (topicHandle == NULL) { + return -1; + } + if (pMsg->offset == topicHandle->nextConsumeOffset) { + return 0; + } + // TODO: check log last version + + code = tqBufferSetOffset(topicHandle, offset); + if (code < 0) { + // set error code + return -1; + } + + return 0; +} + +int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { + int64_t clientId = pMsg->head.clientId; + STqGroup* pGroup = tqGetGroup(pTq, clientId); + if (pGroup == NULL) { + terrno = TSDB_CODE_TQ_GROUP_NOT_SET; + return -1; + } + + STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg; + int numOfMsgs = tqFetch(pGroup, (void**)&pRsp->msgs); + if (numOfMsgs < 0) { + return -1; + } + if (numOfMsgs == 0) { + // most recent data has been fetched + + // enable timer for blocking wait + // once new data written during wait time + // launch query and response + } + + // fetched a num of msgs, rpc response + return 0; } +#if 0 int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { - if (!tqProtoCheck((TmqMsgHead*)pMsg)) { + if (!tqProtoCheck((STqMsgHead*)pMsg)) { // proto version invalid return -1; } - int64_t clientId = pMsg->head.clientId; - STqGroupHandle* gHandle = tqGetGroupHandle(pTq, clientId); - if (gHandle == NULL) { + int64_t clientId = pMsg->head.clientId; + STqGroup* pGroup = tqGetGroup(pTq, clientId); + if (pGroup == NULL) { // client not connect return -1; } if (pMsg->acks.ackNum != 0) { - if (tqAck(gHandle, &pMsg->acks) != 0) { + if (tqAck(pGroup, &pMsg->acks) != 0) { // ack not success return -1; } @@ -244,22 +320,23 @@ int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg; - if (tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) { + if (tqFetch(pGroup, (void**)&pRsp->msgs) <= 0) { // fetch error return -1; } // judge and launch new query - if (tqLaunchQuery(gHandle)) { - // launch query error - return -1; - } + /*if (tqSendLaunchQuery(gHandle)) {*/ + // launch query error + /*return -1;*/ + /*}*/ return 0; } +#endif -int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead) { +int tqSerializeGroup(const STqGroup* pGroup, STqSerializedHead** ppHead) { // calculate size - int sz = tqGetgHandleSSize(gHandle) + sizeof(STqSerializedHead); + int sz = tqGroupSSize(pGroup) + sizeof(STqSerializedHead); if (sz > (*ppHead)->ssize) { void* tmpPtr = realloc(*ppHead, sz); if (tmpPtr == NULL) { @@ -272,53 +349,53 @@ int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** pp } void* ptr = (*ppHead)->content; // do serialization - *(int64_t*)ptr = gHandle->cId; + *(int64_t*)ptr = pGroup->clientId; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int64_t*)ptr = gHandle->cgId; + *(int64_t*)ptr = pGroup->cgId; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int32_t*)ptr = gHandle->topicNum; + *(int32_t*)ptr = pGroup->topicNum; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - if (gHandle->topicNum > 0) { - tqSerializeListHandle(gHandle->head, ptr); + if (pGroup->topicNum > 0) { + tqSerializeListHandle(pGroup->head, ptr); } return 0; } -void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr) { - STqListHandle* node = listHandle; +void* tqSerializeListHandle(STqList* listHandle, void* ptr) { + STqList* node = listHandle; ASSERT(node != NULL); while (node) { - ptr = tqSerializeBufHandle(&node->bufHandle, ptr); + ptr = tqSerializeTopic(&node->topic, ptr); node = node->next; } return ptr; } -void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr) { - *(int64_t*)ptr = bufHandle->nextConsumeOffset; +void* tqSerializeTopic(STqTopic* pTopic, void* ptr) { + *(int64_t*)ptr = pTopic->nextConsumeOffset; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int64_t*)ptr = bufHandle->topicId; + *(int64_t*)ptr = pTopic->topicId; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int32_t*)ptr = bufHandle->head; + *(int32_t*)ptr = pTopic->head; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - *(int32_t*)ptr = bufHandle->tail; + *(int32_t*)ptr = pTopic->tail; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - ptr = tqSerializeBufItem(&bufHandle->buffer[i], ptr); + ptr = tqSerializeItem(&pTopic->buffer[i], ptr); } return ptr; } -void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr) { +void* tqSerializeItem(STqMsgItem* bufItem, void* ptr) { // TODO: do we need serialize this? // mainly for executor return ptr; } -const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** ppGHandle) { - STqGroupHandle* gHandle = *ppGHandle; - const void* ptr = pHead->content; - gHandle->cId = *(int64_t*)ptr; +const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup) { + STqGroup* gHandle = *ppGroup; + const void* ptr = pHead->content; + gHandle->clientId = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); gHandle->cgId = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); @@ -326,63 +403,63 @@ const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHan gHandle->topicNum = *(int32_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); gHandle->head = NULL; - STqListHandle* node = gHandle->head; + STqList* node = gHandle->head; for (int i = 0; i < gHandle->topicNum; i++) { if (gHandle->head == NULL) { - if ((node = malloc(sizeof(STqListHandle))) == NULL) { + if ((node = malloc(sizeof(STqList))) == NULL) { // TODO: error return NULL; } node->next = NULL; - ptr = tqDeserializeBufHandle(ptr, &node->bufHandle); + ptr = tqDeserializeTopic(ptr, &node->topic); gHandle->head = node; } else { - node->next = malloc(sizeof(STqListHandle)); + node->next = malloc(sizeof(STqList)); if (node->next == NULL) { // TODO: error return NULL; } node->next->next = NULL; - ptr = tqDeserializeBufHandle(ptr, &node->next->bufHandle); + ptr = tqDeserializeTopic(ptr, &node->next->topic); node = node->next; } } return ptr; } -const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle) { +const void* tqDeserializeTopic(const void* pBytes, STqTopic* topic) { const void* ptr = pBytes; - bufHandle->nextConsumeOffset = *(int64_t*)ptr; + topic->nextConsumeOffset = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - bufHandle->topicId = *(int64_t*)ptr; + topic->topicId = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - bufHandle->head = *(int32_t*)ptr; + topic->head = *(int32_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - bufHandle->tail = *(int32_t*)ptr; + topic->tail = *(int32_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - ptr = tqDeserializeBufItem(ptr, &bufHandle->buffer[i]); + ptr = tqDeserializeItem(ptr, &topic->buffer[i]); } return ptr; } -const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem) { return pBytes; } +const void* tqDeserializeItem(const void* pBytes, STqMsgItem* bufItem) { return pBytes; } // TODO: make this a macro -int tqGetgHandleSSize(const STqGroupHandle* gHandle) { +int tqGroupSSize(const STqGroup* gHandle) { return sizeof(int64_t) * 2 // cId + cgId + sizeof(int32_t) // topicNum - + gHandle->topicNum * tqBufHandleSSize(); + + gHandle->topicNum * tqTopicSSize(); } // TODO: make this a macro -int tqBufHandleSSize() { +int tqTopicSSize() { return sizeof(int64_t) * 2 // nextConsumeOffset + topicId + sizeof(int32_t) * 2 // head + tail - + TQ_BUFFER_SIZE * tqBufItemSSize(); + + TQ_BUFFER_SIZE * tqItemSSize(); } -int tqBufItemSSize() { +int tqItemSSize() { // TODO: do this need serialization? // mainly for executor return 0; diff --git a/source/dnode/vnode/tq/src/tqMetaStore.c b/source/dnode/vnode/tq/src/tqMetaStore.c index 082f0ad28e30356794d9cc12b69eac2668a6fb9c..57e20010e3b12b615875e1ecaba1eddd05161562 100644 --- a/source/dnode/vnode/tq/src/tqMetaStore.c +++ b/source/dnode/vnode/tq/src/tqMetaStore.c @@ -56,6 +56,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { int offset = tqSeekLastPage(fd); int nBytes; if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } if (nBytes == 0) { @@ -71,7 +72,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial int32_t tqConfigFlag) { STqMetaStore* pMeta = malloc(sizeof(STqMetaStore)); if (pMeta == NULL) { - // close + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } memset(pMeta, 0, sizeof(STqMetaStore)); @@ -79,8 +80,9 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial // concat data file name and index file name size_t pathLen = strlen(path); pMeta->dirPath = malloc(pathLen + 1); - if (pMeta->dirPath != NULL) { - // TODO: memory insufficient + if (pMeta->dirPath == NULL) { + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + return NULL; } strcpy(pMeta->dirPath, path); @@ -88,13 +90,14 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial strcpy(name, path); if (taosDirExist(name) != 0 && taosMkDir(name) != 0) { - ASSERT(false); + terrno = TSDB_CODE_TQ_FAILED_TO_CREATE_DIR; + tqError("failed to create dir:%s since %s ", name, terrstr()); } strcat(name, "/" TQ_IDX_NAME); int idxFd = open(name, O_RDWR | O_CREAT, 0755); if (idxFd < 0) { - ASSERT(false); - // close file + terrno = TAOS_SYSTEM_ERROR(errno); + tqError("failed to open file:%s since %s ", name, terrstr()); // free memory return NULL; } @@ -102,9 +105,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial pMeta->idxFd = idxFd; pMeta->unpersistHead = malloc(sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { - ASSERT(false); - // close file - // free memory + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } memset(pMeta->unpersistHead, 0, sizeof(STqMetaList)); @@ -114,7 +115,8 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial strcat(name, "/" TQ_META_NAME); int fileFd = open(name, O_RDWR | O_CREAT, 0755); if (fileFd < 0) { - ASSERT(false); + terrno = TAOS_SYSTEM_ERROR(errno); + tqError("failed to open file:%s since %s", name, terrstr()); return NULL; } @@ -129,7 +131,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial STqIdxPageBuf idxBuf; STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE); if (serializedObj == NULL) { - // TODO:memory insufficient + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; } int idxRead; int allocated = TQ_PAGE_SIZE; @@ -137,14 +139,16 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { if (idxRead == -1) { // TODO: handle error - ASSERT(false); + terrno = TAOS_SYSTEM_ERROR(errno); + tqError("failed to read tq index file since %s", terrstr()); } ASSERT(idxBuf.head.writeOffset == idxRead); // loop read every entry for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { STqMetaList* pNode = malloc(sizeof(STqMetaList)); if (pNode == NULL) { - // TODO: free memory and return error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + // TODO: free memory } memset(pNode, 0, sizeof(STqMetaList)); memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); @@ -153,7 +157,8 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial if (allocated < pNode->handle.serializedSize) { void* ptr = realloc(serializedObj, pNode->handle.serializedSize); if (ptr == NULL) { - // TODO: memory insufficient + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + // TODO: free memory } serializedObj = ptr; allocated = pNode->handle.serializedSize; @@ -292,7 +297,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { STqMetaList* pNode = pHead->unpersistNext; STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead)); if (pSHead == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } pSHead->ver = TQ_SVER; @@ -403,7 +408,6 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu STqMetaList* pNode = pMeta->bucket[bucketKey]; while (pNode) { if (pNode->handle.key == key) { - // TODO: think about thread safety if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInUse); } @@ -416,7 +420,7 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu } STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); if (pNewNode == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } memset(pNewNode, 0, sizeof(STqMetaList)); @@ -470,10 +474,10 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va STqMetaList* pNode = pMeta->bucket[bucketKey]; while (pNode) { if (pNode->handle.key == key) { - // TODO: think about thread safety if (pNode->handle.valueInTxn) { - if (TqDupIntxnReject(pMeta->tqConfigFlag)) { - return -2; + if (tqDupIntxnReject(pMeta->tqConfigFlag)) { + terrno = TSDB_CODE_TQ_META_KEY_DUP_IN_TXN; + return -1; } if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInTxn); @@ -488,7 +492,7 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va } STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); if (pNewNode == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } memset(pNewNode, 0, sizeof(STqMetaList)); @@ -505,7 +509,7 @@ int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { void* vmem = malloc(vsize); if (vmem == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } memcpy(vmem, value, vsize); @@ -535,6 +539,7 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) { while (pNode) { if (pNode->handle.key == key) { if (pNode->handle.valueInTxn == NULL) { + terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN; return -1; } if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { @@ -548,7 +553,8 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) { pNode = pNode->next; } } - return -2; + terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; + return -1; } int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { @@ -564,12 +570,14 @@ int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { tqLinkUnpersist(pMeta, pNode); return 0; } + terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN; return -1; } else { pNode = pNode->next; } } - return -2; + terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; + return -1; } int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { @@ -588,7 +596,7 @@ int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { pNode = pNode->next; } } - // no such key + terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; return -1; } diff --git a/source/dnode/vnode/tq/test/tqMetaTest.cpp b/source/dnode/vnode/tq/test/tqMetaTest.cpp index 58263efa71ec77db6a5fbfc973d4f7c9edd55714..d3c9b50e4abd884903f50b2bf5d10bd8ec71eb00 100644 --- a/source/dnode/vnode/tq/test/tqMetaTest.cpp +++ b/source/dnode/vnode/tq/test/tqMetaTest.cpp @@ -10,8 +10,8 @@ struct Foo { }; int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { - Foo* foo = (Foo*) pObj; - if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) { + Foo* foo = (Foo*)pObj; + if ((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) { *ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t)); (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t); } @@ -20,36 +20,28 @@ int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { } const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) { - if(*ppObj == NULL) { + if (*ppObj == NULL) { *ppObj = realloc(*ppObj, sizeof(int32_t)); } Foo* pFoo = *(Foo**)ppObj; - pFoo->a = *(int32_t*)pHead->content; + pFoo->a = *(int32_t*)pHead->content; return NULL; } -void FooDeleter(void* pObj) { - free(pObj); -} +void FooDeleter(void* pObj) { free(pObj); } class TqMetaUpdateAppendTest : public ::testing::Test { - protected: - - void SetUp() override { - taosRemoveDir(pathName); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); - ASSERT(pMeta); - } - - void TearDown() override { - tqStoreClose(pMeta); - } - - TqMetaStore* pMeta; - const char* pathName = "/tmp/tq_test"; + protected: + void SetUp() override { + taosRemoveDir(pathName); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); + ASSERT(pMeta); + } + + void TearDown() override { tqStoreClose(pMeta); } + + STqMetaStore* pMeta; + const char* pathName = "/tmp/tq_test"; }; TEST_F(TqMetaUpdateAppendTest, copyPutTest) { @@ -57,11 +49,11 @@ TEST_F(TqMetaUpdateAppendTest, copyPutTest) { foo.a = 3; tqHandleCopyPut(pMeta, 1, &foo, sizeof(Foo)); - Foo* pFoo = (Foo*) tqHandleGet(pMeta, 1); + Foo* pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); tqHandleCommit(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo->a, 3); } @@ -78,10 +70,7 @@ TEST_F(TqMetaUpdateAppendTest, persistTest) { EXPECT_EQ(pBar == NULL, true); tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); pBar = (Foo*)tqHandleGet(pMeta, 1); @@ -97,7 +86,7 @@ TEST_F(TqMetaUpdateAppendTest, uncommittedTest) { pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); } @@ -106,11 +95,11 @@ TEST_F(TqMetaUpdateAppendTest, abortTest) { pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); tqHandleAbort(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); } @@ -119,32 +108,29 @@ TEST_F(TqMetaUpdateAppendTest, deleteTest) { pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); tqHandleCommit(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); ASSERT_EQ(pFoo != NULL, true); EXPECT_EQ(pFoo->a, 3); tqHandleDel(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); ASSERT_EQ(pFoo != NULL, true); EXPECT_EQ(pFoo->a, 3); tqHandleCommit(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); } @@ -162,10 +148,7 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) { EXPECT_EQ(pFoo1->a, 3); tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); pFoo1 = (Foo*)tqHandleGet(pMeta, 1); @@ -177,10 +160,7 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) { EXPECT_EQ(pFoo1->a, 4); tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); pFoo1 = (Foo*)tqHandleGet(pMeta, 1); @@ -190,13 +170,13 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) { TEST_F(TqMetaUpdateAppendTest, multiplePage) { srand(0); std::vector v; - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { v.push_back(rand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 0; i < 500; i++) { + for (int i = 0; i < 500; i++) { tqHandleCommit(pMeta, i); Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; @@ -204,38 +184,34 @@ TEST_F(TqMetaUpdateAppendTest, multiplePage) { } tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); - - for(int i = 500; i < 1000; i++) { + + for (int i = 500; i < 1000; i++) { tqHandleCommit(pMeta, i); Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; EXPECT_EQ(pFoo->a, v[i]); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; EXPECT_EQ(pFoo->a, v[i]); } - } TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { srand(0); std::vector v; - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { v.push_back(rand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 0; i < 500; i++) { + for (int i = 0; i < 500; i++) { tqHandleCommit(pMeta, i); v[i] = rand(); Foo foo; @@ -243,25 +219,22 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 500; i < 1000; i++) { + for (int i = 500; i < 1000; i++) { v[i] = rand(); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { tqHandleCommit(pMeta, i); } tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); - - for(int i = 500; i < 1000; i++) { + + for (int i = 500; i < 1000; i++) { v[i] = rand(); Foo foo; foo.a = v[i]; @@ -269,40 +242,38 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { tqHandleCommit(pMeta, i); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; EXPECT_EQ(pFoo->a, v[i]); } - } TEST_F(TqMetaUpdateAppendTest, dupCommit) { srand(0); std::vector v; - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { v.push_back(rand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { int ret = tqHandleCommit(pMeta, i); EXPECT_EQ(ret, 0); ret = tqHandleCommit(pMeta, i); EXPECT_EQ(ret, -1); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { int ret = tqHandleCommit(pMeta, i); EXPECT_EQ(ret, -1); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; EXPECT_EQ(pFoo->a, v[i]); } - } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 32cedb82b0ca834ead0822c0340919b508fe2d88..1134c7763a88c22e80ad523cbdb63b84313f1872 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -634,6 +634,16 @@ _return: CTG_RET(code); } +int32_t catalogGetQnodeList(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet) { + if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pQnodeEpSet) { + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + + return TSDB_CODE_SUCCESS; +} + + void catalogDestroy(void) { if (ctgMgmt.pCluster) { taosHashCleanup(ctgMgmt.pCluster); //TBD diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index a258cb834dcbe4a15845c690b706d46b09674c78..22bb9d1d0f2ebbf342b93ef6f40347e7437685f1 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -76,7 +76,7 @@ struct SIndexMultiTermQuery { // field and key; typedef struct SIndexTerm { - int64_t suid; + int64_t suid; SIndexOperOnColumn operType; // oper type, add/del/update uint8_t colType; // term data type, str/interger/json char *colName; diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index ea090389bba1ff1a12b91ebcac2d15827916286a..ac9a59fa042775fa101e9fec1db54fe8fb995200 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -34,8 +34,14 @@ typedef struct WriterCtx { int (*flush)(struct WriterCtx *ctx); WriterType type; union { - int fd; - void *mem; + struct { + int fd; + bool readOnly; + } file; + struct { + int32_t capa; + char *buf; + } mem; }; int32_t offset; int32_t limit; @@ -45,7 +51,7 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len); static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len); static int writeCtxDoFlush(WriterCtx *ctx); -WriterCtx* writerCtxCreate(WriterType type, bool readOnly); +WriterCtx* writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity); void writerCtxDestroy(WriterCtx *w); typedef uint32_t CheckSummer; @@ -66,7 +72,7 @@ int fstCountingWriterFlush(FstCountingWriter *write); uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write); -FstCountingWriter *fstCountingWriterCreate(void *wtr, bool readOnly); +FstCountingWriter *fstCountingWriterCreate(void *wtr); void fstCountingWriterDestroy(FstCountingWriter *w); diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index c3f4bd25e5e6a792120f1ac8880bfbc57915dfef..979c0b06397bb9d928d6c1b48294eadd5bee4246 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -18,23 +18,81 @@ #include "index.h" #include "indexInt.h" #include "tlockfree.h" -#include "tskiplist.h" +#include "index_tfile.h" +#include "index_fst_counting_writer.h" +#include "index_fst.h" #ifdef __cplusplus extern "C" { #endif -typedef struct IndexTFile { + + +typedef struct TFileCacheKey { + uint64_t suid; + uint8_t colType; + int32_t version; + const char *colName; + int32_t nColName; +} TFileCacheKey; + + +// table cache +// refactor to LRU cache later +typedef struct TFileCache { + SHashObj *tableCache; + int16_t capacity; + // add more param +} TFileCache; + + +typedef struct TFileWriter { + FstBuilder *fb; + WriterCtx *wc; +} TFileWriter; + +typedef struct TFileReader { T_REF_DECLARE() + Fst *fst; + +} TFileReader; + +typedef struct IndexTFile { + char *path; + TFileReader *tb; + TFileWriter *tw; } IndexTFile; +typedef struct TFileWriterOpt { + uint64_t suid; + int8_t colType; + char *colName; + int32_t nColName; + int32_t version; +} TFileWriterOpt; +typedef struct TFileReaderOpt { + uint64_t suid; + char *colName; + int32_t nColName; + +} TFileReaderOpt; + +// tfile cache +TFileCache *tfileCacheCreate(); +void tfileCacheDestroy(TFileCache *tcache); +TFileReader* tfileCacheGet(TFileCache *tcache, TFileCacheKey *key); +void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader); + +TFileWriter *tfileWriterCreate(const char *suid, const char *colName); IndexTFile *indexTFileCreate(); +int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid); int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result); + #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h new file mode 100644 index 0000000000000000000000000000000000000000..4ab517ecfafe6927dde70bbbd6fb0b7076e2b5f9 --- /dev/null +++ b/source/libs/index/inc/index_util.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef __INDEX_UTIL_H__ +#define __INDEX_UTIL_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ + do { \ + memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ + buf += sizeof(key->mem); \ + } while (0) + +#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ + do { \ + memcpy((void *)buf, (void *)key->mem, len); \ + buf += len; \ + } while (0) + +#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ + do { \ + type c = var; \ + assert(sizeof(var) == sizeof(type));\ + memcpy((void *)buf, (void *)&c, sizeof(c)); \ + buf += sizeof(c); \ + } while (0) + +#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ + do { \ + memcpy((void *)buf, (void *)var, len); \ + buf += len;\ + } while (0) + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 3c52275a4c96f2f0f12f6fae012c9681f2a37c99..5813c99164ee11f4a196b1d69638ef8ad37847b2 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -15,6 +15,7 @@ #include "index_cache.h" #include "tcompare.h" +#include "index_util.h" #define MAX_INDEX_KEY_LEN 256// test only, change later @@ -110,35 +111,22 @@ int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, if (cache == NULL) { return -1;} IndexCache *pCache = cache; - // encode data int32_t total = CACHE_KEY_LEN(term); - char *buf = calloc(1, total); char *p = buf; - memcpy(p, &total, sizeof(total)); - p += sizeof(total); - - memcpy(p, &colId, sizeof(colId)); - p += sizeof(colId); + SERIALIZE_VAR_TO_BUF(p, total,int32_t); + SERIALIZE_VAR_TO_BUF(p, colId, int16_t); - memcpy(p, &term->colType, sizeof(term->colType)); - p += sizeof(term->colType); + SERIALIZE_MEM_TO_BUF(p, term, colType); + SERIALIZE_MEM_TO_BUF(p, term, nColVal); + SERIALIZE_STR_MEM_TO_BUF(p, term, colVal, term->nColVal); - memcpy(p, &term->nColVal, sizeof(term->nColVal)); - p += sizeof(term->nColVal); - memcpy(p, term->colVal, term->nColVal); - p += term->nColVal; - - memcpy(p, &version, sizeof(version)); - p += sizeof(version); - - memcpy(p, &uid, sizeof(uid)); - p += sizeof(uid); + SERIALIZE_VAR_TO_BUF(p, version, int32_t); + SERIALIZE_VAR_TO_BUF(p, uid, uint64_t); - memcpy(p, &term->operType, sizeof(term->operType)); - p += sizeof(term->operType); + SERIALIZE_MEM_TO_BUF(p, term, operType); tSkipListPut(pCache->skiplist, (void *)buf); return 0; diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 7aaa498864fff74dccccdb2446c190b24272fcb2..0f00aacf3bd0fcd746042977d4f0b62ad3d3ba8f 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -779,7 +779,7 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) { if (NULL == b) { return b; } - b->wrt = fstCountingWriterCreate(w, false); + b->wrt = fstCountingWriterCreate(w); b->unfinished = fstUnFinishedNodesCreate(); b->registry = fstRegistryCreate(10000, 2) ; b->last = fstSliceCreate(NULL, 0); diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 9ec346cebc026cba7d03e1cf39890a6c262d1a9a..3497b9703d2d875d2d093e3fb4fc2ec7c22e32b0 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -23,9 +23,9 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { } if (ctx->type == TFile) { - assert(len == tfWrite(ctx->fd, buf, len)); + assert(len == tfWrite(ctx->file.fd, buf, len)); } else { - memcpy(ctx->mem + ctx->offset, buf, len); + memcpy(ctx->mem.buf+ ctx->offset, buf, len); } ctx->offset += len; return len; @@ -33,9 +33,9 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { int nRead = 0; if (ctx->type == TFile) { - nRead = tfRead(ctx->fd, buf, len); + nRead = tfRead(ctx->file.fd, buf, len); } else { - memcpy(buf, ctx->mem + ctx->offset, len); + memcpy(buf, ctx->mem.buf + ctx->offset, len); } ctx->offset += nRead; @@ -44,63 +44,64 @@ static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { static int writeCtxDoFlush(WriterCtx *ctx) { if (ctx->type == TFile) { //tfFsync(ctx->fd); - //tfFlush(ctx->fd); + //tfFlush(ctx->file.fd); } else { // do nothing } return 1; } -WriterCtx* writerCtxCreate(WriterType type, bool readOnly) { +WriterCtx* writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity) { WriterCtx *ctx = calloc(1, sizeof(WriterCtx)); if (ctx == NULL) { return NULL; } ctx->type = type; if (ctx->type == TFile) { - tfInit(); // ugly code, refactor later + ctx->file.readOnly = readOnly; if (readOnly == false) { - ctx->fd = tfOpenCreateWriteAppend(tmpFile); + ctx->file.fd = tfOpenCreateWriteAppend(tmpFile); } else { - ctx->fd = tfOpenReadWrite(tmpFile); + ctx->file.fd = tfOpenReadWrite(tmpFile); } - if (ctx->fd < 0) { + if (ctx->file.fd < 0) { indexError("open file error %d", errno); } } else if (ctx->type == TMemory) { - ctx->mem = calloc(1, DefaultMem * sizeof(uint8_t)); + ctx->mem.buf = calloc(1, sizeof(char) * capacity); + ctx->mem.capa = capacity; } ctx->write = writeCtxDoWrite; ctx->read = writeCtxDoRead; ctx->flush = writeCtxDoFlush; ctx->offset = 0; - ctx->limit = DefaultMem; + ctx->limit = capacity; return ctx; } void writerCtxDestroy(WriterCtx *ctx) { if (ctx->type == TMemory) { - free(ctx->mem); + free(ctx->mem.buf); } else { - tfClose(ctx->fd); - tfCleanup(); + tfClose(ctx->file.fd); } free(ctx); } -FstCountingWriter *fstCountingWriterCreate(void *wrt, bool readOnly) { +FstCountingWriter *fstCountingWriterCreate(void *wrt) { FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); if (cw == NULL) { return NULL; } - cw->wrt = (void *)(writerCtxCreate(TFile, readOnly)); + cw->wrt = wrt; + //(void *)(writerCtxCreate(TFile, readOnly)); return cw; } void fstCountingWriterDestroy(FstCountingWriter *cw) { // free wrt object: close fd or free mem fstCountingWriterFlush(cw); - writerCtxDestroy((WriterCtx *)(cw->wrt)); + //writerCtxDestroy((WriterCtx *)(cw->wrt)); free(cw); } @@ -124,6 +125,7 @@ int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len) } uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { + return 0; } int fstCountingWriterFlush(FstCountingWriter *write) { diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index a1bba56391ce445899fe1fc23b8e422c8cf3cd8f..81a7f9f44351bff3ea77146f5182c3386bdf6dd9 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -14,6 +14,50 @@ */ #include "index_tfile.h" +#include "index_fst.h" +#include "index_util.h" + + + +static void tfileSerialCacheKey(TFileCacheKey *key, char *buf) { + SERIALIZE_MEM_TO_BUF(buf, key, suid); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_MEM_TO_BUF(buf, key, colType); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_MEM_TO_BUF(buf, key, version); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); +} + +TFileCache *tfileCacheCreate() { + TFileCache *tcache = calloc(1, sizeof(TFileCache)); + if (tcache == NULL) { return NULL; } + + tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + tcache->capacity = 64; + return tcache; +} +void tfileCacheDestroy(TFileCache *tcache) { + + free(tcache); + +} + +TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key) { + char buf[128] = {0}; + tfileSerialCacheKey(key, buf); + TFileReader *reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); + return reader; +} +void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader) { + char buf[128] = {0}; + tfileSerialCacheKey(key, buf); + taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void *)); + return; +} + + + IndexTFile *indexTFileCreate() { IndexTFile *tfile = calloc(1, sizeof(IndexTFile)); @@ -22,10 +66,24 @@ IndexTFile *indexTFileCreate() { void IndexTFileDestroy(IndexTFile *tfile) { free(tfile); } + + int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) { IndexTFile *ptfile = (IndexTFile *)tfile; + return 0; } +int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid) { + TFileWriterOpt wOpt = {.suid = term->suid, + .colType = term->colType, + .colName = term->colName, + .nColName= term->nColName, + .version = 1}; + + + + return 0; +} diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 9dff2e9ea0ac901390cde63feee01f50c3179fe6..9baabb9610c6bea3c04469538891df32c439a840 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -45,7 +45,7 @@ class FstWriter { class FstReadMemory { public: FstReadMemory(size_t size) { - _w = fstCountingWriterCreate(NULL, true); + _w = fstCountingWriterCreate(NULL); _size = size; memset((void *)&_s, 0, sizeof(_s)); } diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index de7cdd58b85227c3a4dbb1742a0789fe01413802..32906f7800de1e724bc2ee1e43511f051ab93eeb 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -4,7 +4,8 @@ #include "parserInt.h" #include "taosmsg.h" -SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen); +SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); +SDropUserMsg* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen); SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen); diff --git a/source/libs/parser/inc/queryInfoUtil.h b/source/libs/parser/inc/queryInfoUtil.h index 5d51293939381bac3274864457e2e413e3184445..798c9bc97fde7f93b8d0a38e2927d124dd7c0cd2 100644 --- a/source/libs/parser/inc/queryInfoUtil.h +++ b/source/libs/parser/inc/queryInfoUtil.h @@ -36,7 +36,6 @@ void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, i void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize); SExprInfo* getExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index); -int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy); void addExprInfoParam(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes); diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 5dc351ee0ca4c6781cf6d8446551456ec3176c4c..6a752cc195462479f6d480d3f5856bb4e9ff7854 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -1,7 +1,7 @@ #include "parserInt.h" #include "parserUtil.h" -SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen) { +SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen) { SCreateUserMsg* pMsg = (SCreateUserMsg*)calloc(1, sizeof(SCreateUserMsg)); if (pMsg == NULL) { // tscError("0x%" PRIx64 " failed to malloc for query msg", id); @@ -20,6 +20,24 @@ SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgB strncpy(pMsg->pass, pUser->passwd.z, pUser->passwd.n); } + *outputLen = sizeof(SUserInfo); + return pMsg; +} + +SDropUserMsg* buildDropUserMsg(SSqlInfo* pInfo, int32_t *msgLen, int64_t id, char* msgBuf, int32_t msgBufLen) { + SToken* pName = taosArrayGet(pInfo->pMiscInfo->a, 0); + if (pName->n >= TSDB_USER_LEN) { + return NULL; + } + + + SDropUserMsg* pMsg = calloc(1, sizeof(SDropUserMsg)); + if (pMsg == NULL) { + return NULL; + } + + strncpy(pMsg->user, pName->z, pName->n); + *msgLen = sizeof(SDropUserMsg); return pMsg; } @@ -89,7 +107,7 @@ static int32_t setTimePrecision(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreate pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default - SToken* pToken = &pCreateDbInfo->precision; + SToken* pToken = (SToken*) &pCreateDbInfo->precision; if (pToken->n > 0) { pToken->n = strdequote(pToken->z); @@ -141,7 +159,6 @@ int32_t setDbOptions(SCreateDbMsg* pCreateDbMsg, const SCreateDbInfo* pCreateDbS // todo configurable pCreateDbMsg->numOfVgroups = htonl(2); - return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 135774cd3b6921130c183efabf1dcacc12c51120..ba96c4a79638aaa5bdf26c68f46dc715daf181e3 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4224,7 +4224,13 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, in } } - *output = buildUserManipulationMsg(pInfo, id, msgBuf, msgBufLen); + *output = buildUserManipulationMsg(pInfo, outputLen, id, msgBuf, msgBufLen); + break; + } + + case TSDB_SQL_DROP_ACCT: + case TSDB_SQL_DROP_USER: { + *output = buildDropUserMsg(pInfo, outputLen, id, msgBuf, msgBufLen); break; } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index c548f1556a64daeedb9c99033245404f594297ce..502946675b40738c8ec7d3e35287669ce04dab71 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -1,10 +1,7 @@ +#include "os.h" #include "queryInfoUtil.h" -#include -#include "astGenerator.h" #include "function.h" -#include "os.h" #include "parser.h" -#include "parserInt.h" #include "parserUtil.h" static struct SSchema _s = { diff --git a/source/libs/parser/test/insertTest.cpp b/source/libs/parser/test/insertTest.cpp index 5877adf41cc303556f9ce5a311493bbffdd0cc6d..85db46d7bf1cfa29576926a0dafc7ef52310a2a8 100644 --- a/source/libs/parser/test/insertTest.cpp +++ b/source/libs/parser/test/insertTest.cpp @@ -57,17 +57,20 @@ void *__wrap_malloc(size_t c) { // [...]; class InsertTest : public Test { protected: - void setDatabase(const string& db) { + void setDatabase(const string& acctId, const string& db) { + acctId_ = acctId; db_ = db; } void bind(const char* sql) { reset(); - cxt_.sqlLen = strlen(sql); + cxt_.pAcctId = acctId_.c_str(); + cxt_.pDbname = db_.c_str(); strcpy(sqlBuf_, sql); + cxt_.sqlLen = strlen(sql); sqlBuf_[cxt_.sqlLen] = '\0'; cxt_.pSql = sqlBuf_; - cxt_.pDbname = db_.c_str(); + } int32_t run() { @@ -95,6 +98,7 @@ private: res_ = nullptr; } + string acctId_; string db_; char errMagBuf_[max_err_len]; char sqlBuf_[max_sql_len]; @@ -105,7 +109,7 @@ private: // INSERT INTO tb_name VALUES (field1_value, ...) TEST_F(InsertTest, simpleTest) { - setDatabase("test"); + setDatabase("root", "test"); bind("insert into t1 values (now, 1, \"wxy\")"); ASSERT_EQ(run(), TSDB_CODE_SUCCESS); @@ -116,7 +120,7 @@ TEST_F(InsertTest, simpleTest) { } TEST_F(InsertTest, toleranceTest) { - setDatabase("test"); + setDatabase("root", "test"); bind("insert into"); ASSERT_NE(run(), TSDB_CODE_SUCCESS); diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index f2b34971efcb1487929fa6274bc6b34001f7053c..4e6b94081b4e993a17d8aeb567fb228b55c8d55b 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -20,34 +20,42 @@ namespace { void generateTestT1(MockCatalogService* mcs) { - ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 3) + ITableBuilder& builder = mcs->createTableBuilder("root.test", "t1", TSDB_NORMAL_TABLE, 3) .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); builder.done(); } void generateTestST1(MockCatalogService* mcs) { - ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 2) + ITableBuilder& builder = mcs->createTableBuilder("root.test", "st1", TSDB_SUPER_TABLE, 3, 2) .setPrecision(TSDB_TIME_PRECISION_MILLI).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) .addTag("tag1", TSDB_DATA_TYPE_INT).addTag("tag2", TSDB_DATA_TYPE_BINARY, 10) .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); builder.done(); - mcs->createSubTable("test", "st1", "st1s1", 1); - mcs->createSubTable("test", "st1", "st1s2", 2); + mcs->createSubTable("root.test", "st1", "st1s1", 1); + mcs->createSubTable("root.test", "st1", "st1s2", 2); } } -void generateMetaData(MockCatalogService* mcs) { - generateTestT1(mcs); - generateTestST1(mcs); - mcs->showTables(); +int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { + return mockCatalogService->catalogGetHandle(clusterId, catalogHandle); } -struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) { - return mockCatalogService->getCatalogHandle(pMgmtEps); +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { + return mockCatalogService->catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, pTableMeta); } -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) { - return mockCatalogService->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); +void initMetaDataEnv() { + mockCatalogService.reset(new MockCatalogService()); +} + +void generateMetaData() { + generateTestT1(mockCatalogService.get()); + generateTestST1(mockCatalogService.get()); + mockCatalogService->showTables(); +} + +void destroyMetaDataEnv() { + mockCatalogService.reset(); } diff --git a/source/libs/parser/test/mockCatalog.h b/source/libs/parser/test/mockCatalog.h index e60f7727e6804cc6fb6a8959ecb2440745671163..c9812d5b946fc4ce99d9be4bd7100ec03b131e68 100644 --- a/source/libs/parser/test/mockCatalog.h +++ b/source/libs/parser/test/mockCatalog.h @@ -18,10 +18,12 @@ #include "mockCatalogService.h" -void generateMetaData(MockCatalogService* mcs); +void initMetaDataEnv(); +void generateMetaData(); +void destroyMetaDataEnv(); // mock -struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData); +int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); +int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); #endif // MOCK_CATALOG_H diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 457f8d88bdb6d54156e1cc8be29e569978350be9..e7271c3cb2a2901f339500369030d3cb95ccf9e4 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -27,27 +27,28 @@ std::unique_ptr mockCatalogService; class TableBuilder : public ITableBuilder { public: virtual TableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) { - assert(colIndex_ < meta_->tableInfo.numOfTags + meta_->tableInfo.numOfColumns); - SSchema* col = meta_->schema + colIndex_; + assert(colId_ < schema()->tableInfo.numOfTags + schema()->tableInfo.numOfColumns); + SSchema* col = schema()->schema + colId_; col->type = type; - col->colId = colIndex_++; + col->colId = colId_++; col->bytes = bytes; strcpy(col->name, name.c_str()); return *this; } virtual TableBuilder& setVgid(int16_t vgid) { - meta_->vgId = vgid; + schema()->vgId = vgid; + meta_->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); return *this; } virtual TableBuilder& setPrecision(uint8_t precision) { - meta_->tableInfo.precision = precision; + schema()->tableInfo.precision = precision; return *this; } virtual void done() { - meta_->tableInfo.rowSize = rowsize_; + schema()->tableInfo.rowSize = rowsize_; } private: @@ -64,61 +65,62 @@ private: return std::unique_ptr(new TableBuilder(meta)); } - TableBuilder(STableMeta* meta) : colIndex_(0), rowsize_(0), meta_(meta) { + TableBuilder(STableMeta* schemaMeta) : colId_(0), rowsize_(0), meta_(new MockTableMeta()) { + meta_->schema.reset(schemaMeta); } - STableMeta* table() { + std::shared_ptr schema() { + return meta_->schema; + } + + std::shared_ptr table() { return meta_; } - int32_t colIndex_; + int32_t colId_; int32_t rowsize_; - STableMeta* meta_; + std::shared_ptr meta_; }; class MockCatalogServiceImpl { public: static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]); - MockCatalogServiceImpl() { + MockCatalogServiceImpl() : id_(1) { } - struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) const { - return (struct SCatalog*)0x01; + int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const { + return 0; } - int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const { - assert(nullptr != pMetaReq && 1 == taosArrayGetSize(pMetaReq->pTableName)); - SName* fullName = (SName*)taosArrayGet(pMetaReq->pTableName, 0); + int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { std::unique_ptr table; - int32_t code = copyTableMeta(fullName->dbname, fullName->tname, &table); + int32_t code = copyTableSchemaMeta(pDBName, pTableName, &table); if (TSDB_CODE_SUCCESS != code) { return code; } - std::unique_ptr tables((SArray*)taosArrayInit(1, sizeof(STableMeta*))); - if (!tables) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - STableMeta* elem = table.release(); - taosArrayPush(tables.get(), &elem); - pMetaData->pTableMeta = tables.release(); + *pTableMeta = table.release(); return TSDB_CODE_SUCCESS; } TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags); - meta_[db][tbname].reset(builder_->table()); - meta_[db][tbname]->uid = id_++; + meta_[db][tbname] = builder_->table(); + meta_[db][tbname]->schema->uid = id_++; return *(builder_.get()); } void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid) { std::unique_ptr table; - if (TSDB_CODE_SUCCESS != copyTableMeta(db, stbname, &table)) { - throw std::runtime_error("copyTableMeta failed"); + if (TSDB_CODE_SUCCESS != copyTableSchemaMeta(db, stbname, &table)) { + throw std::runtime_error("copyTableSchemaMeta failed"); } - meta_[db][tbname].reset(table.release()); - meta_[db][tbname]->uid = id_++; + meta_[db][tbname].reset(new MockTableMeta()); + meta_[db][tbname]->schema.reset(table.release()); + meta_[db][tbname]->schema->uid = id_++; + meta_[db][tbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + // super table + meta_[db][stbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); } void showTables() const { @@ -148,29 +150,43 @@ public: std::cout << SH("Table") << SH("Type") << SH("Precision") << IH("Vgid") << std::endl; std::cout << SL(3, 1) << std::endl; for (const auto& table : db.second) { - std::cout << SF(table.first) << SF(ttToString(table.second->tableType)) << SF(pToString(table.second->tableInfo.precision)) << IF(table.second->vgId) << std::endl; + const auto& schema = table.second->schema; + std::cout << SF(table.first) << SF(ttToString(schema->tableType)) << SF(pToString(schema->tableInfo.precision)) << IF(schema->vgId) << std::endl; } std::cout << std::endl; } for (const auto& db : meta_) { for (const auto& table : db.second) { + const auto& schema = table.second->schema; std::cout << "Table:" << table.first << std::endl; std::cout << SH("Field") << SH("Type") << SH("DataType") << IH("Bytes") << std::endl; std::cout << SL(3, 1) << std::endl; - int16_t numOfTags = table.second->tableInfo.numOfTags; - int16_t numOfFields = numOfTags + table.second->tableInfo.numOfColumns; + int16_t numOfTags = schema->tableInfo.numOfTags; + int16_t numOfFields = numOfTags + schema->tableInfo.numOfColumns; for (int16_t i = 0; i < numOfFields; ++i) { - const SSchema* schema = table.second->schema + i; - std::cout << SF(std::string(schema->name)) << SH(ftToString(i, numOfTags)) << SH(dtToString(schema->type)) << IF(schema->bytes) << std::endl; + const SSchema* col = schema->schema + i; + std::cout << SF(std::string(col->name)) << SH(ftToString(i, numOfTags)) << SH(dtToString(col->type)) << IF(col->bytes) << std::endl; } std::cout << std::endl; } } } + std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const { + DbMetaCache::const_iterator it = meta_.find(db); + if (meta_.end() == it) { + return std::shared_ptr(); + } + TableMetaCache::const_iterator tit = it->second.find(tbname); + if (it->second.end() == tit) { + return std::shared_ptr(); + } + return tit->second; + } + private: - typedef std::map > TableMetaCache; + typedef std::map> TableMetaCache; typedef std::map DbMetaCache; std::string ttToString(int8_t tableType) const { @@ -207,20 +223,13 @@ private: return (0 == colid ? "column" : (colid <= numOfTags ? "tag" : "column")); } - std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const { - DbMetaCache::const_iterator it = meta_.find(db); - if (meta_.end() == it) { - return std::shared_ptr(); - } - TableMetaCache::const_iterator tit = it->second.find(tbname); - if (it->second.end() == tit) { - return std::shared_ptr(); - } - return tit->second; + std::shared_ptr getTableSchemaMeta(const std::string& db, const std::string& tbname) const { + std::shared_ptr table = getTableMeta(db, tbname); + return table ? table->schema : std::shared_ptr(); } - int32_t copyTableMeta(const std::string& db, const std::string& tbname, std::unique_ptr* dst) const { - std::shared_ptr src = getTableMeta(db, tbname); + int32_t copyTableSchemaMeta(const std::string& db, const std::string& tbname, std::unique_ptr* dst) const { + std::shared_ptr src = getTableSchemaMeta(db, tbname); if (!src) { return TSDB_CODE_TSC_INVALID_TABLE_NAME; } @@ -244,14 +253,6 @@ MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) { MockCatalogService::~MockCatalogService() { } -struct SCatalog* MockCatalogService::getCatalogHandle(const SEpSet* pMgmtEps) const { - return impl_->getCatalogHandle(pMgmtEps); -} - -int32_t MockCatalogService::catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const { - return impl_->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); -} - ITableBuilder& MockCatalogService::createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { return impl_->createTableBuilder(db, tbname, tableType, numOfColumns, numOfTags); } @@ -262,4 +263,16 @@ void MockCatalogService::createSubTable(const std::string& db, const std::string void MockCatalogService::showTables() const { impl_->showTables(); -} \ No newline at end of file +} + +std::shared_ptr MockCatalogService::getTableMeta(const std::string& db, const std::string& tbname) const { + return impl_->getTableMeta(db, tbname); +} + +int32_t MockCatalogService::catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const { + return impl_->catalogGetHandle(clusterId, catalogHandle); +} + +int32_t MockCatalogService::catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { + return impl_->catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, pTableMeta); +} diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index 66b439b3e9f46a6cf1d1350d1777e9ee3eee6852..cd01db09cce7ff4c5949671ff44af041a47d80cb 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -18,6 +18,7 @@ #include #include +#include #include "catalog.h" @@ -41,19 +42,24 @@ public: virtual void done() = 0; }; -class MockCatalogServiceImpl; +struct MockTableMeta { + std::shared_ptr schema; + std::vector vgs; +}; +class MockCatalogServiceImpl; class MockCatalogService { public: - static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]); - MockCatalogService(); ~MockCatalogService(); - struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) const; - int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const; ITableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags = 0); void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid); void showTables() const; + std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const; + + // mock interface + int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const; + int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const; private: std::unique_ptr impl_; diff --git a/source/libs/parser/test/parserMain.cpp b/source/libs/parser/test/parserTestMain.cpp similarity index 88% rename from source/libs/parser/test/parserMain.cpp rename to source/libs/parser/test/parserTestMain.cpp index 5ec304a006291f280ed6f57459ba80a17e74f6ba..7de2cb66c202324d1bef2bf3cd47d00eae8feefa 100644 --- a/source/libs/parser/test/parserMain.cpp +++ b/source/libs/parser/test/parserTestMain.cpp @@ -22,12 +22,12 @@ class ParserEnv : public testing::Environment { public: virtual void SetUp() { - mockCatalogService.reset(new MockCatalogService()); - generateMetaData(mockCatalogService.get()); + initMetaDataEnv(); + generateMetaData(); } virtual void TearDown() { - mockCatalogService.reset(); + destroyMetaDataEnv(); } ParserEnv() {} diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 19563a8a0c3c64ff9bd142c98a1c7875464b9f41..b1c4643f154f56be96a096d09161e1f0aa74540a 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -100,7 +100,7 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str); int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag); -int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps); +int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SEpAddr* ep); int32_t subPlanToString(const SSubplan *pPhyNode, char** str); int32_t stringToSubplan(const char* str, SSubplan** subplan); @@ -117,6 +117,7 @@ void destroyQueryPlan(struct SQueryPlanNode* pQueryNode); */ void* destroyQueryPhyPlan(struct SPhyNode* pQueryPhyNode); +const char* opTypeToOpName(int32_t type); int32_t opNameToOpType(const char* name); #ifdef __cplusplus diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index f187ec0ec9a9badcb336c3ff744a63adf6a9920d..1d29e48e308e1b60ee60e35ea6e6c3dcf489e643 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -33,6 +33,10 @@ static const char* gOpName[] = { #undef INCLUDE_AS_NAME }; +const char* opTypeToOpName(int32_t type) { + return gOpName[type]; +} + int32_t opNameToOpType(const char* name) { for (int32_t i = 1; i < sizeof(gOpName) / sizeof(gOpName[0]); ++i) { if (strcmp(name, gOpName[i])) { @@ -42,17 +46,43 @@ int32_t opNameToOpType(const char* name) { return OP_Unknown; } -static void toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { - SWAP(dataBlockSchema->pSchema, pPlanNode->pSchema, SSchema*); +static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { dataBlockSchema->numOfCols = pPlanNode->numOfCols; + dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfCols); + if (NULL == dataBlockSchema->pSchema) { + return false; + } + memcpy(dataBlockSchema->pSchema, pPlanNode->pSchema, sizeof(SSlotSchema) * pPlanNode->numOfCols); + return true; +} + +static bool cloneExprArray(SArray** dst, SArray* src) { + if (NULL == src) { + return true; + } + size_t size = taosArrayGetSize(src); + if (0 == size) { + return true; + } + *dst = taosArrayInit(size, POINTER_BYTES); + if (NULL == *dst) { + return false; + } + return (TSDB_CODE_SUCCESS == copyAllExprInfo(*dst, src, true) ? true : false); } static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) { SPhyNode* node = (SPhyNode*)calloc(1, size); + if (NULL == node) { + return NULL; + } node->info.type = type; - node->info.name = gOpName[type]; - SWAP(node->pTargets, pPlanNode->pExpr, SArray*); - toDataBlockSchema(pPlanNode, &(node->targetSchema)); + node->info.name = opTypeToOpName(type); + if (!cloneExprArray(&node->pTargets, pPlanNode->pExpr) || !toDataBlockSchema(pPlanNode, &(node->targetSchema))) { + free(node); + return NULL; + } + return node; } static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) { @@ -203,9 +233,6 @@ static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE); ++(pCxt->nextId.templateId); subplan->pNode = createPhyNode(pCxt, pRoot); - SArray* l0 = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); - taosArrayPush(l0, &subplan); - taosArrayPush(pCxt->pDag->pSubplans, &l0); // todo deal subquery } @@ -225,6 +252,6 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD return TSDB_CODE_SUCCESS; } -int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps) { +int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SEpAddr* ep) { //todo } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 3a90acb5fdd1a9a9c58b560e97186f76a9a5a200..e9a4591d4afe8b4b0efed0f322c9be6ba5c4ee40 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -46,8 +46,8 @@ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* return TSDB_CODE_SUCCESS; } -int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps) { - return setSubplanExecutionNode(subplan, templateId, eps); +int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SEpAddr* ep) { + return setSubplanExecutionNode(subplan, templateId, ep); } int32_t qSubPlanToString(const SSubplan *subplan, char** str) { diff --git a/source/libs/planner/test/CMakeLists.txt b/source/libs/planner/test/CMakeLists.txt index 58743567846df2f4c80272cf7433cf3af6b9663a..7fbfcfe7efc31ecae16ffd3d4ad72db2cff58090 100644 --- a/source/libs/planner/test/CMakeLists.txt +++ b/source/libs/planner/test/CMakeLists.txt @@ -5,7 +5,12 @@ MESSAGE(STATUS "build planner unit test") SET(CMAKE_CXX_STANDARD 11) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) -ADD_EXECUTABLE(plannerTest ${SOURCE_LIST}) +ADD_EXECUTABLE(plannerTest + ${SOURCE_LIST} + "${SOURCE_LIST}/../../../parser/test/mockCatalog.cpp" + "${SOURCE_LIST}/../../../parser/test/mockCatalogService.cpp" +) + TARGET_LINK_LIBRARIES( plannerTest PUBLIC os util common planner parser catalog transport gtest function qcom @@ -15,4 +20,5 @@ TARGET_INCLUDE_DIRECTORIES( plannerTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/planner/" PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/planner/inc" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/parser/test" ) diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3be3337304534efa3d02a068585ec5f1a916ae3c --- /dev/null +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include "plannerInt.h" +#include "mockCatalogService.h" + +using namespace std; +using namespace testing; + +void* myCalloc(size_t nmemb, size_t size) { + if (void* p = calloc(nmemb, size)) { + return p; + } + throw bad_alloc(); +} + +class PhyPlanTest : public Test { +protected: + void pushScan(const string& db, const string& table, int32_t scanOp) { + shared_ptr meta = mockCatalogService->getTableMeta(db, table); + EXPECT_TRUE(meta); +// typedef struct SQueryPlanNode { +// SArray *pExpr; // the query functions or sql aggregations +// int32_t numOfExpr; // number of result columns, which is also the number of pExprs +// } SQueryPlanNode; + unique_ptr scan((SQueryPlanNode*)calloc(1, sizeof(SQueryPlanNode))); + scan->info.type = scanOp; + scan->numOfCols = meta->schema->tableInfo.numOfColumns; + scan->pSchema = (SSchema*)myCalloc(1, sizeof(SSchema) * scan->numOfCols); + memcpy(scan->pSchema, meta->schema->schema, sizeof(SSchema) * scan->numOfCols); + //todo 'pExpr' 'numOfExpr' + scan->pExtInfo = createScanExtInfo(meta); + pushNode(scan.release()); + } + + int32_t run() { + SQueryDag* dag = nullptr; + int32_t code = createDag(logicPlan_.get(), nullptr, &dag); + dag_.reset(dag); + return code; + } + + void explain() { + size_t level = taosArrayGetSize(dag_->pSubplans); + for (size_t i = 0; i < level; ++i) { + std::cout << "level " << i << ":" << std::endl; + const SArray* subplans = (const SArray*)taosArrayGetP(dag_->pSubplans, i); + size_t num = taosArrayGetSize(subplans); + for (size_t j = 0; j < num; ++j) { + std::cout << "no " << j << ":" << std::endl; + char* str = nullptr; + ASSERT_EQ (TSDB_CODE_SUCCESS, qSubPlanToString((const SSubplan*)taosArrayGetP(subplans, j), &str)); + std::cout << str << std::endl; + free(str); + } + } + } + + SQueryDag* reslut() { + return dag_.get(); + } + +private: + void pushNode(SQueryPlanNode* node) { + if (logicPlan_) { + // todo + } else { + logicPlan_.reset(node); + } + } + + void copySchemaMeta(STableMeta** dst, const STableMeta* src) { + int32_t size = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); + *dst = (STableMeta*)myCalloc(1, size); + memcpy(*dst, src, size); + } + + void copyStorageMeta(SVgroupsInfo** dst, const std::vector& src) { + *dst = (SVgroupsInfo*)myCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * src.size()); + (*dst)->numOfVgroups = src.size(); + for (int32_t i = 0; i < src.size(); ++i) { + (*dst)->vgroups[i].vgId = src[i].vgId; + (*dst)->vgroups[i].numOfEps = src[i].numOfEps; + memcpy((*dst)->vgroups[i].epAddr, src[i].epAddr, src[i].numOfEps); + } + } + + SQueryTableInfo* createScanExtInfo(shared_ptr& meta) { + SQueryTableInfo* info = (SQueryTableInfo*)myCalloc(1, sizeof(SQueryTableInfo)); + info->pMeta = (STableMetaInfo*)myCalloc(1, sizeof(STableMetaInfo)); + copySchemaMeta(&info->pMeta->pTableMeta, meta->schema.get()); + copyStorageMeta(&info->pMeta->vgroupList, meta->vgs); + return info; + } + + shared_ptr meta_; + unique_ptr logicPlan_; + unique_ptr dag_; +}; + +// select * from table +TEST_F(PhyPlanTest, tableScanTest) { + pushScan("root.test", "t1", QNODE_TABLESCAN); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + explain(); + SQueryDag* dag = reslut(); + // todo check +} + +// select * from supertable +TEST_F(PhyPlanTest, superTableScanTest) { + pushScan("root.test", "st1", QNODE_TABLESCAN); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + explain(); + SQueryDag* dag = reslut(); + // todo check +} diff --git a/source/libs/planner/test/plannerTests.cpp b/source/libs/planner/test/plannerTests.cpp index 5ede8dd15578b353953488a3104460bfd63d6cd8..9379b06ac8594bc961c30a57fb18cece1859a407 100644 --- a/source/libs/planner/test/plannerTests.cpp +++ b/source/libs/planner/test/plannerTests.cpp @@ -20,6 +20,7 @@ #include "taos.h" #include "parser.h" +#include "mockCatalog.h" #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -27,9 +28,25 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); +class ParserEnv : public testing::Environment { +public: + virtual void SetUp() { + initMetaDataEnv(); + generateMetaData(); + } + + virtual void TearDown() { + destroyMetaDataEnv(); + } + + ParserEnv() {} + virtual ~ParserEnv() {} +}; + +int main(int argc, char* argv[]) { + testing::AddGlobalTestEnvironment(new ParserEnv()); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); } TEST(testCase, planner_test) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 9ca8b6789b328dc00f8d23e1c35264b70ece8413..728c47af1e71e4c8b15d1f2a921c34149f3c098f 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -294,7 +294,7 @@ void msgInit() { tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; - tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAG_VAL] = tscBuildUpdateTagMsg; tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index 31f1c25beae079107d630ad98c6af6bfc38be70f..6baaab1ef4b5ce436306ebc98057b2c1e28f599d 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -9,5 +9,7 @@ target_include_directories( target_link_libraries( scheduler - PRIVATE os util planner qcom common -) \ No newline at end of file + PRIVATE os util planner qcom common catalog transport +) + +ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 4620a816daf426a4a8a7781a0b35b959310e239f..73e7c4d24e1c2693d174dcb7acad4112939778c3 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -27,6 +27,9 @@ extern "C" { #include "thash.h" #define SCHEDULE_DEFAULT_JOB_NUMBER 1000 +#define SCHEDULE_DEFAULT_TASK_NUMBER 1000 + +#define SCH_MAX_CONDIDATE_EP_NUM TSDB_MAX_REPLICA enum { SCH_STATUS_NOT_START = 1, @@ -39,22 +42,27 @@ enum { typedef struct SSchedulerMgmt { uint64_t taskId; + SSchedulerCfg cfg; SHashObj *Jobs; // key: queryId, value: SQueryJob* } SSchedulerMgmt; typedef struct SQueryTask { - uint64_t taskId; // task id - char *msg; // operator tree - int8_t status; // task status - SQueryProfileSummary summary; // task execution summary + uint64_t taskId; // task id + SSubplan *plan; // subplan + char *msg; // operator tree + int8_t status; // task status + SEpAddr execAddr; // task actual executed node address + SQueryProfileSummary summary; // task execution summary + int32_t childReady; // child task ready number + SArray *children; // the datasource tasks,from which to fetch the result, element is SQueryTask* + SArray *parents; // the data destination tasks, get data from current task, element is SQueryTask* } SQueryTask; typedef struct SQueryLevel { + int32_t level; int8_t status; int32_t taskNum; - SArray *subTasks; // Element is SQueryTask - SArray *subPlans; // Element is SSubplan } SQueryLevel; typedef struct SQueryJob { @@ -63,13 +71,28 @@ typedef struct SQueryJob { int32_t levelIdx; int8_t status; SQueryProfileSummary summary; - - SArray *levels; // Element is SQueryLevel, starting from 0. - SArray *subPlans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. + SEpSet dataSrcEps; + SEpAddr resEp; + void *transport; + SArray *qnodeList; + tsem_t rspSem; + int32_t userFetch; + int32_t remoteFetch; + void *res; + + SHashObj *execTasks; // executing tasks, key:taskid, value:SQueryTask* + SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask* + + SArray *levels; // Element is SQueryLevel, starting from 0. + SArray *subPlans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. } SQueryJob; +#define SCH_HAS_QNODE_IN_CLUSTER(type) (false) //TODO CLUSTER TYPE +#define SCH_TASK_READY_TO_LUNCH(task) ((task)->childReady >= taosArrayGetSize((task)->children)) // MAY NEED TO ENHANCE +#define SCH_IS_DATA_SRC_TASK(task) (task->plan->type == QUERY_TYPE_SCAN) #define SCH_JOB_ERR_LOG(param, ...) qError("QID:%"PRIx64 param, job->queryId, __VA_ARGS__) +#define SCH_TASK_ERR_LOG(param, ...) qError("QID:%"PRIx64",TID:%"PRIx64 param, job->queryId, task->taskId, __VA_ARGS__) #define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) @@ -77,6 +100,8 @@ typedef struct SQueryJob { #define SCH_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) +extern int32_t schLaunchTask(SQueryJob *job, SQueryTask *task); + #ifdef __cplusplus } #endif diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 7d387cbc66f481b6b37c96f34afa6b71a43aa8f7..6014ff9ab6282248f52bfe1aba0535e5b4d4cb71 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -16,6 +16,7 @@ #include "schedulerInt.h" #include "taosmsg.h" #include "query.h" +#include "catalog.h" SSchedulerMgmt schMgmt = {0}; @@ -50,23 +51,109 @@ int32_t schBuildAndSendRequest(void *pRpc, const SEpSet* pMgmtEps, __taos_async_ */ } +int32_t schBuildTaskRalation(SQueryJob *job, SHashObj *planToTask) { + for (int32_t i = 0; i < job->levelNum; ++i) { + SQueryLevel *level = taosArrayGet(job->levels, i); + + for (int32_t m = 0; m < level->taskNum; ++m) { + SQueryTask *task = taosArrayGet(level->subTasks, m); + SSubplan *plan = task->plan; + int32_t childNum = plan->pChildern ? (int32_t)taosArrayGetSize(plan->pChildern) : 0; + int32_t parentNum = plan->pParents ? (int32_t)taosArrayGetSize(plan->pParents) : 0; + + if (childNum > 0) { + task->children = taosArrayInit(childNum, POINTER_BYTES); + if (NULL == task->children) { + qError("taosArrayInit %d failed", childNum); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + for (int32_t n = 0; n < childNum; ++n) { + SSubplan *child = taosArrayGet(plan->pChildern, n); + SQueryTask *childTask = taosHashGet(planToTask, &child, POINTER_BYTES); + if (childTask) { + qError("subplan relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + if (NULL == taosArrayPush(task->children, &childTask)) { + qError("taosArrayPush failed"); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + if (parentNum > 0) { + task->parents = taosArrayInit(parentNum, POINTER_BYTES); + if (NULL == task->parents) { + qError("taosArrayInit %d failed", parentNum); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + + for (int32_t n = 0; n < parentNum; ++n) { + SSubplan *parent = taosArrayGet(plan->pParents, n); + SQueryTask *parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES); + if (parentTask) { + qError("subplan relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + if (NULL == taosArrayPush(task->parents, &parentTask)) { + qError("taosArrayPush failed"); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + } + } + + SQueryLevel *level = taosArrayGet(job->levels, 0); + if (level->taskNum > 1) { + qError("invalid plan info, level 0, taskNum:%d", level->taskNum); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + SQueryTask *task = taosArrayGet(level->subTasks, 0); + if (task->parents && taosArrayGetSize(task->parents) > 0) { + qError("invalid plan info, level 0, parentNum:%d", (int32_t)taosArrayGetSize(task->parents)); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + + return TSDB_CODE_SUCCESS; +} + + int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { int32_t code = 0; + + job->queryId = dag->queryId; + + if (dag->numOfSubplans <= 0) { + qError("invalid subplan num:%d", dag->numOfSubplans); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + int32_t levelNum = (int32_t)taosArrayGetSize(dag->pSubplans); if (levelNum <= 0) { qError("invalid level num:%d", levelNum); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + SHashObj *planToTask = taosHashInit(SCHEDULE_DEFAULT_TASK_NUMBER, taosGetDefaultHashFunction(POINTER_BYTES == sizeof(int64_t) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == planToTask) { + qError("taosHashInit %d failed", SCHEDULE_DEFAULT_TASK_NUMBER); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + job->levels = taosArrayInit(levelNum, sizeof(SQueryLevel)); if (NULL == job->levels) { qError("taosArrayInit %d failed", levelNum); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } job->levelNum = levelNum; job->levelIdx = levelNum - 1; - job->status = SCH_STATUS_NOT_START; job->subPlans = dag->pSubplans; @@ -77,32 +164,45 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { level.status = SCH_STATUS_NOT_START; for (int32_t i = 0; i < levelNum; ++i) { + level.level = i; levelPlans = taosArrayGetP(dag->pSubplans, i); if (NULL == levelPlans) { qError("no level plans for level %d", i); - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } levelPlanNum = (int32_t)taosArrayGetSize(levelPlans); if (levelPlanNum <= 0) { qError("invalid level plans number:%d, level:%d", levelPlanNum, i); - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } level.taskNum = levelPlanNum; - level.subPlans = levelPlans; level.subTasks = taosArrayInit(levelPlanNum, sizeof(SQueryTask)); if (NULL == level.subTasks) { qError("taosArrayInit %d failed", levelPlanNum); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } for (int32_t n = 0; n < levelPlanNum; ++n) { - SQueryTask *task = taosArrayGet(level.subTasks, n); + SSubplan *plan = taosArrayGet(levelPlans, n); + SQueryTask task = {0}; + + task.taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); + task.plan = plan; + task.status = SCH_STATUS_NOT_START; + + void *p = taosArrayPush(level.subTasks, &task); + if (NULL == p) { + qError("taosArrayPush failed"); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } - task->taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); - task->status = SCH_STATUS_NOT_START; + if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &p, POINTER_BYTES)) { + qError("taosHashPut failed"); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } } if (NULL == taosArrayPush(job->levels, &level)) { @@ -111,6 +211,12 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { } } + SCH_ERR_JRET(schBuildTaskRalation(job, planToTask)); + + if (planToTask) { + taosHashCleanup(planToTask); + } + return TSDB_CODE_SUCCESS; _return: @@ -118,20 +224,331 @@ _return: taosArrayDestroy(level.subTasks); } + if (planToTask) { + taosHashCleanup(planToTask); + } + SCH_RET(code); } +int32_t schSetTaskExecEpSet(SQueryJob *job, SEpSet *epSet) { + if (epSet->numOfEps >= SCH_MAX_CONDIDATE_EP_NUM) { + return TSDB_CODE_SUCCESS; + } + + int32_t qnodeNum = taosArrayGetSize(job->qnodeList); + + for (int32_t i = 0; i < qnodeNum && epSet->numOfEps < tListLen(epSet->port); ++i) { + SEpAddr *addr = taosArrayGet(job->qnodeList, i); + + strncpy(epSet->fqdn[epSet->numOfEps], addr->fqdn, sizeof(addr->fqdn)); + epSet->port[epSet->numOfEps] = addr->port; + + ++epSet->numOfEps; + } -int32_t schJobExecute(SQueryJob *job) { - switch (job->status) { - case SCH_STATUS_NOT_START: + for (int32_t i = 0; i < job->dataSrcEps.numOfEps && epSet->numOfEps < tListLen(epSet->port); ++i) { + strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); + epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; + + ++epSet->numOfEps; + } + return TSDB_CODE_SUCCESS; +} + + +int32_t schPushTaskToExecList(SQueryJob *job, SQueryTask *task) { + if (0 != taosHashPut(job->execTasks, &task->taskId, sizeof(task->taskId), &task, POINTER_BYTES)) { + qError("taosHashPut failed"); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t schMoveTaskToSuccList(SQueryJob *job, SQueryTask *task, bool *moved) { + if (0 != taosHashRemove(job->execTasks, &task->taskId, sizeof(task->taskId))) { + qWarn("remove task[%"PRIx64"] from execTasks failed", task->taskId); + return TSDB_CODE_SUCCESS; + } + + if (0 != taosHashPut(job->execTasks, &task->taskId, sizeof(task->taskId), &task, POINTER_BYTES)) { + qError("taosHashPut failed"); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + *moved = true; + + return TSDB_CODE_SUCCESS; +} + + +int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { + int32_t msgSize = 0; + void *msg = NULL; + + switch (msgType) { + case TSDB_MSG_TYPE_QUERY: { + if (NULL == task->msg) { + qError("query msg is NULL"); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + int32_t len = strlen(task->msg); + msgSize = sizeof(SSchedulerQueryMsg) + len; + msg = calloc(1, msgSize); + if (NULL == msg) { + qError("calloc %d failed", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SSchedulerQueryMsg *pMsg = msg; + pMsg->queryId = job->queryId; + pMsg->taskId = task->taskId; + pMsg->contentLen = len; + memcpy(pMsg->msg, task->msg, len); break; - + } + case TSDB_MSG_TYPE_RSP_READY: { + msgSize = sizeof(SSchedulerReadyMsg); + msg = calloc(1, msgSize); + if (NULL == msg) { + qError("calloc %d failed", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SSchedulerReadyMsg *pMsg = msg; + pMsg->queryId = job->queryId; + pMsg->taskId = task->taskId; + break; + } + case TSDB_MSG_TYPE_FETCH: { + msgSize = sizeof(SSchedulerFetchMsg); + msg = calloc(1, msgSize); + if (NULL == msg) { + qError("calloc %d failed", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + SSchedulerFetchMsg *pMsg = msg; + pMsg->queryId = job->queryId; + pMsg->taskId = task->taskId; + break; + } default: - SCH_JOB_ERR_LOG("invalid job status:%d", job->status); - SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + qError("unknown msg type:%d", msgType); + break; } + + //TODO SEND MSG + + return TSDB_CODE_SUCCESS; +} + +int32_t schTaskCheckAndSetRetry(SQueryJob *job, SQueryTask *task, int32_t errCode, bool *needRetry) { + // TODO set retry or not based on task type/errCode/retry times/job status/available eps... + // TODO if needRetry, set task retry info + + *needRetry = false; + + return TSDB_CODE_SUCCESS; +} + + +int32_t schFetchFromRemote(SQueryJob *job) { + int32_t code = 0; + + if (atomic_val_compare_exchange_32(&job->remoteFetch, 0, 1) != 0) { + qInfo("prior fetching not finished"); + return TSDB_CODE_SUCCESS; + } + + SCH_ERR_JRET(schAsyncSendMsg(job, NULL, TSDB_MSG_TYPE_FETCH)); + + return TSDB_CODE_SUCCESS; + +_return: + atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); + + return code; +} + + +int32_t schProcessOnJobSuccess(SQueryJob *job) { + job->status = SCH_STATUS_SUCCEED; + + if (job->userFetch) { + SCH_ERR_RET(schFetchFromRemote(job)); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t schProcessOnJobFailure(SQueryJob *job) { + job->status = SCH_STATUS_FAILED; + + atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); + + if (job->userFetch) { + tsem_post(&job->rspSem); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t schProcessOnDataFetched(SQueryJob *job) { + atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); + + tsem_post(&job->rspSem); +} + + +int32_t schProcessOnTaskSuccess(SQueryJob *job, SQueryTask *task) { + bool moved = false; + + SCH_ERR_RET(schMoveTaskToSuccList(job, task, &moved)); + if (!moved) { + SCH_TASK_ERR_LOG("task may already moved, status:%d", task->status); + return TSDB_CODE_SUCCESS; + } + + task->status = SCH_STATUS_SUCCEED; + + int32_t parentNum = (int32_t)taosArrayGetSize(task->parents); + if (parentNum == 0) { + if (task->plan->level != 0) { + qError("level error"); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + strncpy(job->resEp.fqdn, task->execAddr.fqdn, sizeof(job->resEp.fqdn)); + job->resEp.port = task->execAddr.port; + + SCH_ERR_RET(schProcessOnJobSuccess(job)); + + return TSDB_CODE_SUCCESS; + } + + if (SCH_IS_DATA_SRC_TASK(task) && job->dataSrcEps.numOfEps < SCH_MAX_CONDIDATE_EP_NUM) { + strncpy(job->dataSrcEps.fqdn[job->dataSrcEps.numOfEps], task->execAddr.fqdn, sizeof(task->execAddr.fqdn)); + job->dataSrcEps.port[job->dataSrcEps.numOfEps] = task->execAddr.port; + + ++job->dataSrcEps.numOfEps; + } + + for (int32_t i = 0; i < parentNum; ++i) { + SQueryTask *par = taosArrayGet(task->parents, i); + + ++par->childReady; + + SCH_ERR_RET(qSetSubplanExecutionNode(par->plan, task->plan->id.templateId, &task->execAddr)); + + if (SCH_TASK_READY_TO_LUNCH(par)) { + SCH_ERR_RET(schLaunchTask(job, task)); + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t schProcessOnTaskFailure(SQueryJob *job, SQueryTask *task, int32_t errCode) { + bool needRetry = false; + SCH_ERR_RET(schTaskCheckAndSetRetry(job, task, errCode, &needRetry)); + + if (!needRetry) { + SCH_TASK_ERR_LOG("task failed[%x], no more retry", errCode); + + job->status = SCH_STATUS_FAILED; + SCH_ERR_RET(schProcessOnJobFailure(job)); + + return TSDB_CODE_SUCCESS; + } + + SCH_ERR_RET(schLaunchTask(job, task)); + + return TSDB_CODE_SUCCESS; +} + +int32_t schHandleRspMsg(SQueryJob *job, SQueryTask *task, int32_t msgType, int32_t rspCode) { + int32_t code = 0; + + switch (msgType) { + case TSDB_MSG_TYPE_QUERY: + if (rspCode != TSDB_CODE_SUCCESS) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); + } else { + code = schAsyncSendMsg(job, task, TSDB_MSG_TYPE_RSP_READY); + if (code) { + goto _task_error; + } + } + break; + case TSDB_MSG_TYPE_RSP_READY: + if (rspCode != TSDB_CODE_SUCCESS) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); + } else { + code = schProcessOnTaskSuccess(job, task); + if (code) { + goto _task_error; + } + } + break; + case TSDB_MSG_TYPE_FETCH: + SCH_ERR_JRET(rspCode); + SCH_ERR_JRET(schProcessOnDataFetched(job)); + break; + default: + qError("unknown msg type:%d received", msgType); + return TSDB_CODE_QRY_INVALID_INPUT; + } + + return TSDB_CODE_SUCCESS; + +_task_error: + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, code)); + return TSDB_CODE_SUCCESS; + +_return: + code = schProcessOnJobFailure(job); + return code; +} + + + + +int32_t schLaunchTask(SQueryJob *job, SQueryTask *task) { + SSubplan *plan = task->plan; + + SCH_ERR_RET(qSubPlanToString(plan, &task->msg)); + if (plan->execEpSet.numOfEps <= 0) { + SCH_ERR_RET(schSetTaskExecEpSet(job, &plan->execEpSet)); + } + + if (plan->execEpSet.numOfEps <= 0) { + SCH_TASK_ERR_LOG("invalid execEpSet num:%d", plan->execEpSet.numOfEps); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + SCH_ERR_RET(schAsyncSendMsg(job, task, TSDB_MSG_TYPE_QUERY)); + + SCH_ERR_RET(schPushTaskToExecList(job, task)); + + task->status = SCH_STATUS_EXECUTING; + + return TSDB_CODE_SUCCESS; +} + +int32_t schLaunchJob(SQueryJob *job) { + SQueryLevel *level = taosArrayGet(job->levels, job->levelIdx); + for (int32_t i = 0; i < level->taskNum; ++i) { + SQueryTask *task = taosArrayGet(level->subTasks, i); + SCH_ERR_RET(schLaunchTask(job, task)); + } + + job->status = SCH_STATUS_EXECUTING; + + return TSDB_CODE_SUCCESS; } @@ -141,27 +558,59 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { SCH_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler jobs failed", SCHEDULE_DEFAULT_JOB_NUMBER); } + if (cfg) { + schMgmt.cfg = *cfg; + } + return TSDB_CODE_SUCCESS; } -int32_t scheduleQueryJob(SQueryDag* pDag, void** pJob) { - if (NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { +int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob) { + if (NULL == transport || NULL == transport ||NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + if (taosArrayGetSize(qnodeList) <= 0) { + qInfo("qnodeList is empty"); + } + int32_t code = 0; SQueryJob *job = calloc(1, sizeof(SQueryJob)); if (NULL == job) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + job->transport = transport; + job->qnodeList = qnodeList; + SCH_ERR_JRET(schValidateAndBuildJob(pDag, job)); - SCH_ERR_JRET(schJobExecute(job)); + job->execTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == job->execTasks) { + qError("taosHashInit %d failed", pDag->numOfSubplans); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } - *(SQueryJob **)pJob = job; + job->succTasks = taosHashInit(pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == job->succTasks) { + qError("taosHashInit %d failed", pDag->numOfSubplans); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + tsem_init(&job->rspSem, 0, 0); + + if (0 != taosHashPut(schMgmt.Jobs, &job->queryId, sizeof(job->queryId), &job, POINTER_BYTES)) { + qError("taosHashPut queryId:%"PRIx64" failed", job->queryId); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } + + job->status = SCH_STATUS_NOT_START; + SCH_ERR_JRET(schLaunchJob(job)); + + *(SQueryJob **)pJob = job; + return TSDB_CODE_SUCCESS; _return: @@ -172,17 +621,64 @@ _return: SCH_RET(code); } -int32_t scheduleFetchRows(void *pJob, void *data); +int32_t scheduleFetchRows(void *pJob, void **data) { + if (NULL == pJob || NULL == data) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SQueryJob *job = pJob; + int32_t code = 0; + + if (atomic_val_compare_exchange_32(&job->userFetch, 0, 1) != 0) { + qError("prior fetching not finished"); + return TSDB_CODE_QRY_APP_ERROR; + } + + if (job->status == SCH_STATUS_SUCCEED) { + SCH_ERR_JRET(schFetchFromRemote(job)); + } + + tsem_wait(&job->rspSem); + + *data = job->res; + job->res = NULL; + +_return: + atomic_val_compare_exchange_32(&job->userFetch, 1, 0); + + return code; +} + +int32_t scheduleCancelJob(void *pJob) { + //TODO -int32_t scheduleCancelJob(void *pJob); + return TSDB_CODE_SUCCESS; +} void scheduleFreeJob(void *pJob) { + if (NULL == pJob) { + return; + } + SQueryJob *job = pJob; + + if (job->status > 0) { + if (0 != taosHashRemove(schMgmt.Jobs, &job->queryId, sizeof(job->queryId))) { + qError("remove job:%"PRIx64"from mgmt failed", job->queryId); // maybe already freed + return; + } + + if (job->status == SCH_STATUS_EXECUTING) { + scheduleCancelJob(pJob); + } + } + + //TODO free job } void schedulerDestroy(void) { if (schMgmt.Jobs) { - taosHashCleanup(schMgmt.Jobs); //TBD + taosHashCleanup(schMgmt.Jobs); //TODO schMgmt.Jobs = NULL; } } diff --git a/source/libs/scheduler/test/CMakeLists.txt b/source/libs/scheduler/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..00a6d08e5d7be527005a64f0e4111d9a3e35974d --- /dev/null +++ b/source/libs/scheduler/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build scheduler unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(schedulerTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + schedulerTest + PUBLIC os util common catalog transport gtest qcom taos planner scheduler +) + +TARGET_INCLUDE_DIRECTORIES( + schedulerTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scheduler/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scheduler/inc" +) diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..9e94553058e923086808cdb462d27ffa95e9a96e 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "catalog.h" +#include "scheduler.h" +#include "tep.h" +#include "trpc.h" + +namespace { +void mockBuildDag(SQueryDag *dag) { + uint64_t qId = 0x111111111111; + + dag->queryId = qId; + dag->numOfSubplans = 2; + dag->pSubplans = taosArrayInit(dag->numOfSubplans, POINTER_BYTES); + SArray *scan = taosArrayInit(1, sizeof(SSubplan)); + SArray *merge = taosArrayInit(1, sizeof(SSubplan)); + + SSubplan scanPlan = {0}; + SSubplan mergePlan = {0}; + + scanPlan.id.queryId = qId; + scanPlan.id.templateId = 0x2222222222; + scanPlan.id.subplanId = 0x3333333333; + scanPlan.type = QUERY_TYPE_SCAN; + scanPlan.level = 1; + scanPlan.execEpSet.numOfEps = 1; + scanPlan.pChildern = NULL; + scanPlan.pParents = taosArrayInit(1, POINTER_BYTES); + + mergePlan.id.queryId = qId; + mergePlan.id.templateId = 0x4444444444; + mergePlan.id.subplanId = 0x5555555555; + mergePlan.type = QUERY_TYPE_MERGE; + mergePlan.level = 0; + mergePlan.execEpSet.numOfEps = 1; + mergePlan.pChildern = taosArrayInit(1, POINTER_BYTES); + mergePlan.pParents = NULL; + + SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan); + SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan); + + taosArrayPush(mergePointer->pChildern, &scanPointer); + taosArrayPush(scanPointer->pParents, &mergePointer); + + taosArrayPush(dag->pSubplans, &merge); + taosArrayPush(dag->pSubplans, &scan); +} + +} + +TEST(testCase, normalCase) { + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; + SVgroupInfo vgInfo = {0}; + void *pJob = NULL; + SQueryDag dag = {0}; + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); + + int32_t code = schedulerInit(NULL); + ASSERT_EQ(code, 0); + + mockBuildDag(&dag); + + code = scheduleExecJob(mockPointer, qnodeList, &dag, &pJob); + ASSERT_EQ(code, 0); +} + + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + + + diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3e374b344b4d7a462d221127ba80e961ad5e4efa..ffc59c5d6975833c25d5d53f1b6a391be0a07090 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -499,6 +499,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_CTG_SYS_ERROR, "catalog system error" //scheduler TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR, "scheduler status error") +TAOS_DEFINE_ERROR(TSDB_CODE_SCH_INTERNAL_ERROR, "scheduler internal error") #ifdef TAOS_ERROR_C diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 23a923a315765778690c5ba72400148d99527792..5bac59f913462efd0118852d6f39fb047b3713e2 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -14,21 +14,21 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" #include "tlog.h" +#include "os.h" #include "tnote.h" #include "tutil.h" +#include "ulog.h" #include "zlib.h" - -#define MAX_LOGLINE_SIZE (1000) -#define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) -#define MAX_LOGLINE_CONTENT_SIZE (MAX_LOGLINE_SIZE - 100) -#define MAX_LOGLINE_DUMP_SIZE (65 * 1024) -#define MAX_LOGLINE_DUMP_BUFFER_SIZE (MAX_LOGLINE_DUMP_SIZE + 10) + +#define MAX_LOGLINE_SIZE (1000) +#define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) +#define MAX_LOGLINE_CONTENT_SIZE (MAX_LOGLINE_SIZE - 100) +#define MAX_LOGLINE_DUMP_SIZE (65 * 1024) +#define MAX_LOGLINE_DUMP_BUFFER_SIZE (MAX_LOGLINE_DUMP_SIZE + 10) #define MAX_LOGLINE_DUMP_CONTENT_SIZE (MAX_LOGLINE_DUMP_SIZE - 100) -#define LOG_FILE_NAME_LEN 300 +#define LOG_FILE_NAME_LEN 300 #define TSDB_DEFAULT_LOG_BUF_SIZE (20 * 1024 * 1024) // 20MB #define DEFAULT_LOG_INTERVAL 25 @@ -38,13 +38,13 @@ #define LOG_MAX_WAIT_MSEC 1000 #define LOG_BUF_BUFFER(x) ((x)->buffer) -#define LOG_BUF_START(x) ((x)->buffStart) -#define LOG_BUF_END(x) ((x)->buffEnd) -#define LOG_BUF_SIZE(x) ((x)->buffSize) -#define LOG_BUF_MUTEX(x) ((x)->buffMutex) +#define LOG_BUF_START(x) ((x)->buffStart) +#define LOG_BUF_END(x) ((x)->buffEnd) +#define LOG_BUF_SIZE(x) ((x)->buffSize) +#define LOG_BUF_MUTEX(x) ((x)->buffMutex) typedef struct { - char * buffer; + char *buffer; int32_t buffStart; int32_t buffEnd; int32_t buffSize; @@ -57,18 +57,18 @@ typedef struct { } SLogBuff; typedef struct { - int32_t fileNum; - int32_t maxLines; - int32_t lines; - int32_t flag; - int32_t openInProgress; - pid_t pid; - char logName[LOG_FILE_NAME_LEN]; - SLogBuff * logHandle; + int32_t fileNum; + int32_t maxLines; + int32_t lines; + int32_t flag; + int32_t openInProgress; + pid_t pid; + char logName[LOG_FILE_NAME_LEN]; + SLogBuff *logHandle; pthread_mutex_t logMutex; } SLogObj; -int8_t tscEmbeddedInUtil = 0; +int8_t tscEmbeddedInUtil = 0; int32_t tsLogKeepDays = 0; int8_t tsAsyncLog = 1; @@ -93,19 +93,19 @@ int32_t debugFlag = 0; int32_t sDebugFlag = 135; int32_t wDebugFlag = 135; int32_t tsdbDebugFlag = 131; +int32_t tqDebugFlag = 131; int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; int32_t ctgDebugFlag = 131; - int64_t dbgEmptyW = 0; int64_t dbgWN = 0; int64_t dbgSmallWN = 0; int64_t dbgBigWN = 0; int64_t dbgWSize = 0; -static SLogObj tsLogObj = { .fileNum = 1 }; -static void * taosAsyncOutputLog(void *param); +static SLogObj tsLogObj = {.fileNum = 1}; +static void *taosAsyncOutputLog(void *param); static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen); static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(int32_t oldFd); @@ -139,8 +139,8 @@ static void taosStopLog() { void taosCloseLog() { taosStopLog(); - //tsem_post(&(tsLogObj.logHandle->buffNotEmpty)); - taosMsleep(MAX_LOG_INTERVAL/1000); + // tsem_post(&(tsLogObj.logHandle->buffNotEmpty)); + taosMsleep(MAX_LOG_INTERVAL / 1000); if (taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { pthread_join(tsLogObj.logHandle->asyncThread, NULL); } @@ -217,7 +217,7 @@ static void *taosThreadToOpenNewFile(void *param) { tsLogObj.lines = 0; tsLogObj.openInProgress = 0; taosCloseLogByFd(oldFd); - + uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); taosPrintCfg(); @@ -308,9 +308,9 @@ static void taosGetLogFileName(char *fn) { static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { #ifdef WINDOWS /* - * always set maxFileNum to 1 - * means client log filename is unique in windows - */ + * always set maxFileNum to 1 + * means client log filename is unique in windows + */ maxFileNum = 1; #endif @@ -381,13 +381,14 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); + printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, + tsTotalLogDirGB); fflush(stdout); return; } va_list argpointer; - char buffer[MAX_LOGLINE_BUFFER_SIZE] = { 0 }; + char buffer[MAX_LOGLINE_BUFFER_SIZE] = {0}; int32_t len; struct tm Tm, *ptm; struct timeval timeSecs; @@ -434,20 +435,20 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } } - if (dflag & DEBUG_SCREEN) - taosWriteFile(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); if (dflag == 255) nInfo(buffer, len); } void taosDumpData(unsigned char *msg, int32_t len) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); + printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, + tsTotalLogDirGB); fflush(stdout); return; } - char temp[256]; - int32_t i, pos = 0, c = 0; + char temp[256]; + int32_t i, pos = 0, c = 0; for (i = 0; i < len; ++i) { sprintf(temp + pos, "%02x ", msg[i]); @@ -468,7 +469,8 @@ void taosDumpData(unsigned char *msg, int32_t len) { void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); + printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, + tsTotalLogDirGB); fflush(stdout); return; } @@ -503,7 +505,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . } else { taosWriteFile(tsLogObj.logHandle->fd, buffer, len); } - + if (tsLogObj.maxLines > 0) { atomic_add_fetch_32(&tsLogObj.lines, 1); @@ -542,7 +544,7 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { tLogBuff->stop = 0; if (pthread_mutex_init(&LOG_BUF_MUTEX(tLogBuff), NULL) < 0) goto _err; - //tsem_init(&(tLogBuff->buffNotEmpty), 0, 0); + // tsem_init(&(tLogBuff->buffNotEmpty), 0, 0); return tLogBuff; @@ -576,12 +578,12 @@ static void taosCopyLogBuffer(SLogBuff *tLogBuff, int32_t start, int32_t end, ch } static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) { - int32_t start = 0; - int32_t end = 0; - int32_t remainSize = 0; + int32_t start = 0; + int32_t end = 0; + int32_t remainSize = 0; static int64_t lostLine = 0; - char tmpBuf[40] = {0}; - int32_t tmpBufLen = 0; + char tmpBuf[40] = {0}; + int32_t tmpBufLen = 0; if (tLogBuff == NULL || tLogBuff->stop) return -1; @@ -592,7 +594,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) remainSize = (start > end) ? (start - end - 1) : (start + LOG_BUF_SIZE(tLogBuff) - end - 1); if (lostLine > 0) { - sprintf(tmpBuf, "...Lost %"PRId64" lines here...\n", lostLine); + sprintf(tmpBuf, "...Lost %" PRId64 " lines here...\n", lostLine); tmpBufLen = (int32_t)strlen(tmpBuf); } @@ -610,7 +612,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) taosCopyLogBuffer(tLogBuff, LOG_BUF_START(tLogBuff), LOG_BUF_END(tLogBuff), msg, msgLen); - //int32_t w = atomic_sub_fetch_32(&waitLock, 1); + // int32_t w = atomic_sub_fetch_32(&waitLock, 1); /* if (w <= 0 || ((remainSize - msgLen - tmpBufLen) < (LOG_BUF_SIZE(tLogBuff) * 4 /5))) { tsem_post(&(tLogBuff->buffNotEmpty)); @@ -622,7 +624,6 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); - return 0; } @@ -634,9 +635,9 @@ static int32_t taosGetLogRemainSize(SLogBuff *tLogBuff, int32_t start, int32_t e static void taosWriteLog(SLogBuff *tLogBuff) { static int32_t lastDuration = 0; - int32_t remainChecked = 0; - int32_t start, end, pollSize; - + int32_t remainChecked = 0; + int32_t start, end, pollSize; + do { if (remainChecked == 0) { start = LOG_BUF_START(tLogBuff); @@ -662,24 +663,24 @@ static void taosWriteLog(SLogBuff *tLogBuff) { if (start < end) { taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); } else { - int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); + int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; + taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); + taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); } dbgWN++; - dbgWSize+=pollSize; - + dbgWSize += pollSize; + if (pollSize < tLogBuff->minBuffSize) { dbgSmallWN++; if (writeInterval < MAX_LOG_INTERVAL) { writeInterval += LOG_INTERVAL_STEP; } - } else if (pollSize > LOG_BUF_SIZE(tLogBuff)/3) { + } else if (pollSize > LOG_BUF_SIZE(tLogBuff) / 3) { dbgBigWN++; writeInterval = MIN_LOG_INTERVAL; - } else if (pollSize > LOG_BUF_SIZE(tLogBuff)/4) { + } else if (pollSize > LOG_BUF_SIZE(tLogBuff) / 4) { if (writeInterval > MIN_LOG_INTERVAL) { writeInterval -= LOG_INTERVAL_STEP; } @@ -698,13 +699,13 @@ static void taosWriteLog(SLogBuff *tLogBuff) { writeInterval = MIN_LOG_INTERVAL; remainChecked = 1; - }while (1); + } while (1); } static void *taosAsyncOutputLog(void *param) { SLogBuff *tLogBuff = (SLogBuff *)param; setThreadName("log"); - + while (1) { taosMsleep(writeInterval); @@ -721,8 +722,8 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { int32_t compressSize = 163840; int32_t ret = 0; int32_t len = 0; - char * data = malloc(compressSize); - FILE * srcFp = NULL; + char *data = malloc(compressSize); + FILE *srcFp = NULL; gzFile dstFp = NULL; srcFp = fopen(srcFileName, "r");