diff --git a/Jenkinsfile b/Jenkinsfile index 2558df777b5698de59093316da0e18407a160d77..c47d37a9a072d707eabc45cfa7099b7342b5c184 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,7 +42,6 @@ def pre_test(){ killall -9 gdb || echo "no gdb running" killall -9 python3.8 || echo "no python program running" cd ${WKC} - git reset --hard HEAD~10 >/dev/null ''' script { if (env.CHANGE_TARGET == 'master') { @@ -75,11 +74,9 @@ def pre_test(){ git pull >/dev/null git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD - git clean -dfx - - git clean -dfx export TZ=Asia/Harbin date + rm -rf debug mkdir debug cd debug cmake .. > /dev/null @@ -109,6 +106,10 @@ pipeline { abortPreviousBuilds() } pre_test() + sh''' + cd ${WKC}/tests + ./test-all.sh b1fq + ''' } } // stage('Parallel test stage') { diff --git a/cmake/cmake.options b/cmake/cmake.options index 8eec896af1ec910936a800cb294ae26ae0fed199..44fa8c7e4b13bed3cf75a07c66c79054a0c253d0 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -44,6 +44,12 @@ option( OFF ) +option( + BUILD_WITH_UV + "If build with libuv" + OFF +) + option( BUILD_WITH_CRAFT "If build with canonical-raft" diff --git a/cmake/craft_CMakeLists.txt.in b/cmake/craft_CMakeLists.txt.in index a77fa679e015db7c7b78b3156ddb71358e5ca762..3a951c1c99abfb04e9bc7bb13aef55e7fb79a395 100644 --- a/cmake/craft_CMakeLists.txt.in +++ b/cmake/craft_CMakeLists.txt.in @@ -4,9 +4,10 @@ ExternalProject_Add(craft GIT_REPOSITORY https://github.com/canonical/raft.git GIT_TAG v0.11.2 SOURCE_DIR "${CMAKE_CONTRIB_DIR}/craft" - BINARY_DIR "${CMAKE_CONTRIB_DIR}/craft/.libs" + BINARY_DIR "${CMAKE_CONTRIB_DIR}/craft" #BUILD_IN_SOURCE TRUE - CONFIGURE_COMMAND "autoreconf -i && ./configure" + # https://answers.ros.org/question/333125/how-to-include-external-automakeautoconf-projects-into-ament_cmake/ + CONFIGURE_COMMAND COMMAND autoreconf -i COMMAND ./configure --enable-example BUILD_COMMAND "$(MAKE)" INSTALL_COMMAND "" TEST_COMMAND "" diff --git a/cmake/libuv_CMakeLists.txt.in b/cmake/libuv_CMakeLists.txt.in new file mode 100644 index 0000000000000000000000000000000000000000..ed406e089e0f641f0a87c1a80fd5a501acbaf490 --- /dev/null +++ b/cmake/libuv_CMakeLists.txt.in @@ -0,0 +1,12 @@ + +# libuv +ExternalProject_Add(libuv + GIT_REPOSITORY https://github.com/libuv/libuv.git + GIT_TAG v1.42.0 + SOURCE_DIR "${CMAKE_CONTRIB_DIR}/libuv" + BINARY_DIR "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + ) \ No newline at end of file diff --git a/cmake/stub_CMakeLists.txt.in b/cmake/stub_CMakeLists.txt.in new file mode 100644 index 0000000000000000000000000000000000000000..e227e820d639865c9b440e3137b3b98f2cf7aade --- /dev/null +++ b/cmake/stub_CMakeLists.txt.in @@ -0,0 +1,12 @@ + +# stub +ExternalProject_Add(stub + GIT_REPOSITORY https://github.com/coolxv/cpp-stub.git + GIT_SUBMODULES "src" + SOURCE_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub" + BINARY_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub/src" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 0aadaccfa15de6cf42e69d49486530523a321f55..c08f894fe7e8f7dc83a0269ba606607974e4f1bd 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -12,6 +12,7 @@ configure_file("${CMAKE_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE} # googletest if(${BUILD_TEST}) cat("${CMAKE_SUPPORT_DIR}/gtest_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + cat("${CMAKE_SUPPORT_DIR}/stub_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_TEST}) # lz4 @@ -37,12 +38,18 @@ endif(${BUILD_WITH_ROCKSDB}) # canonical-raft if(${BUILD_WITH_CRAFT}) cat("${CMAKE_SUPPORT_DIR}/craft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + SET(BUILD_WITH_UV ON CACHE BOOL "craft need libuv" FORCE) endif(${BUILD_WITH_CRAFT}) +#libuv +if(${BUILD_WITH_UV}) + cat("${CMAKE_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif(${BUILD_WITH_UV}) + # bdb if(${BUILD_WITH_BDB}) cat("${CMAKE_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif(${BUILD_WITH_DBD}) +endif(${BUILD_WITH_BDB}) # sqlite if(${BUILD_WITH_SQLITE}) @@ -73,6 +80,11 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build . # googletest if(${BUILD_TEST}) add_subdirectory(googletest) + target_include_directories( + gtest + PUBLIC $ + PUBLIC $ + ) endif(${BUILD_TEST}) # cJson @@ -154,13 +166,18 @@ if(${BUILD_WITH_CRAFT}) add_library(craft STATIC IMPORTED GLOBAL) set_target_properties(craft PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/craft/.libs/libraft.a" - INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/craft" - ) - target_link_libraries(craft - INTERFACE pthread + INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/craft/include" ) + # target_link_libraries(craft + # INTERFACE pthread + # ) endif(${BUILD_WITH_CRAFT}) +# LIBUV +if(${BUILD_WITH_UV}) + add_subdirectory(libuv) +endif(${BUILD_WITH_UV}) + # BDB if(${BUILD_WITH_BDB}) add_library(bdb STATIC IMPORTED GLOBAL) diff --git a/contrib/test/CMakeLists.txt b/contrib/test/CMakeLists.txt index 0a333f604c1ac20edb166e386fcbf42b55572661..330fe8f70f8d9bfcd9e09e77611ae67794cc86a0 100644 --- a/contrib/test/CMakeLists.txt +++ b/contrib/test/CMakeLists.txt @@ -15,4 +15,8 @@ if(${BUILD_WITH_SQLITE}) add_subdirectory(sqlite) endif(${BUILD_WITH_SQLITE}) +if(${BUILD_WITH_CRAFT}) + add_subdirectory(craft) +endif(${BUILD_WITH_CRAFT}) + add_subdirectory(tdev) diff --git a/contrib/test/craft/CMakeLists.txt b/contrib/test/craft/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..e0f6ae64bd3e9c998db15c1006cebf1b15702f5b --- /dev/null +++ b/contrib/test/craft/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(simulate_vnode "simulate_vnode.c") +target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) \ No newline at end of file diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 0efaf038de6c217a72e74dc7c025bd50c06215d7..a8281d95a5e5b07cd8958457f53efc8f8ec96e1b 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -50,9 +50,8 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONSUME, "mq-consume" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET, "mq-set" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET_CUR, "mq-set-cur" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RES_READY, "res-ready" ) - // message from client to mnode TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" ) @@ -95,22 +94,22 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" ) // message from vnode to dnode // message from mnode to vnode -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB_IN, "create-stb-in" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB_IN, "alter-stb-in" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB_IN, "drop-stb-in" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB_IN, "create-stb-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB_IN, "alter-stb-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB_IN, "drop-stb-internal" ) // message from mnode to mnode // message from mnode to qnode // message from mnode to dnode -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_VNODE_IN, "create-vnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_VNODE_IN, "alter-vnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_VNODE_IN, "drop-vnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_AUTH_VNODE_IN, "auth-vnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SYNC_VNODE_IN, "sync-vnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_VNODE_IN, "compact-vnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_MNODE_IN, "create-mnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_MNODE_IN, "alter-mnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_MNODE_IN, "drop-mnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONFIG_DNODE_IN, "config-dnode" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_VNODE_IN, "create-vnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_VNODE_IN, "alter-vnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_VNODE_IN, "drop-vnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_AUTH_VNODE_IN, "auth-vnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SYNC_VNODE_IN, "sync-vnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_VNODE_IN, "compact-vnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_MNODE_IN, "create-mnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_MNODE_IN, "alter-mnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_MNODE_IN, "drop-mnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONFIG_DNODE_IN, "config-dnode-internal" ) // message from qnode to vnode // message from qnode to mnode @@ -225,6 +224,7 @@ typedef struct SBuildUseDBInput { int32_t vgVersion; } SBuildUseDBInput; + #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta @@ -290,37 +290,6 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; -typedef struct { - int32_t contLen; - int32_t vgId; - int8_t tableType; - int16_t numOfColumns; - int16_t numOfTags; - int32_t tid; - int32_t sversion; - int32_t tversion; - int32_t tagDataLen; - int32_t sqlDataLen; - uint64_t uid; - uint64_t superTableUid; - uint64_t createdTime; - char tableFname[TSDB_TABLE_FNAME_LEN]; - char stbFname[TSDB_TABLE_FNAME_LEN]; - char data[]; -} SMDCreateTableMsg; - -typedef struct { - int32_t len; // one create table message - char tableName[TSDB_TABLE_FNAME_LEN]; - int8_t igExists; - int8_t getMeta; - int16_t numOfTags; - int16_t numOfColumns; - int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string - int8_t reserved[16]; - char schema[]; -} SCreateTableMsg; - typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; @@ -341,16 +310,50 @@ typedef struct { } SAlterStbMsg; typedef struct { - char tableFname[TSDB_TABLE_FNAME_LEN]; - char db[TSDB_FULL_DB_NAME_LEN]; - int16_t type; /* operation type */ - int16_t numOfCols; /* number of schema */ - int32_t tagValLen; - SSchema schema[]; - // tagVal is padded after schema - // char tagVal[]; + SMsgHead head; + char name[TSDB_TABLE_FNAME_LEN]; + uint64_t suid; + int32_t sverson; + uint32_t ttl; + uint32_t keep; + int32_t numOfTags; + int32_t numOfColumns; + SSchema pSchema[]; +} SCreateStbInternalMsg; + +typedef struct { + SMsgHead head; + char name[TSDB_TABLE_FNAME_LEN]; + uint64_t suid; +} SDropStbInternalMsg; + +typedef struct { + SMsgHead head; + char name[TSDB_TABLE_FNAME_LEN]; + char stbFname[TSDB_TABLE_FNAME_LEN]; + int8_t tableType; + uint64_t suid; + int32_t sversion; + int32_t numOfTags; + int32_t numOfColumns; + int32_t tagDataLen; + char data[]; +} SCreateTableMsg; + +typedef struct { + SMsgHead head; + char name[TSDB_TABLE_FNAME_LEN]; + int8_t type; /* operation type */ + int32_t numOfCols; /* number of schema */ + int32_t numOfTags; + char data[]; } SAlterTableMsg; +typedef struct { + SMsgHead head; + char name[TSDB_TABLE_FNAME_LEN]; +} SDropTableMsg; + typedef struct { SMsgHead head; int64_t uid; @@ -545,8 +548,8 @@ typedef struct { int32_t sqlstrLen; // sql query string int32_t prevResultLen; // previous result length int32_t numOfOperator; - int32_t tableScanOperator; // table scan operator. -1 means no scan operator - int32_t udfNum; // number of udf function + int32_t tableScanOperator;// table scan operator. -1 means no scan operator + int32_t udfNum; // number of udf function int32_t udfContentOffset; int32_t udfContentLen; SColumnInfo tableCols[]; @@ -591,8 +594,8 @@ typedef struct { int32_t daysToKeep0; int32_t daysToKeep1; int32_t daysToKeep2; - int32_t minRowsPerFileBlock; - int32_t maxRowsPerFileBlock; + int32_t minRows; + int32_t maxRows; int32_t commitTime; int32_t fsyncPeriod; int8_t walLevel; @@ -706,7 +709,7 @@ typedef struct { SVnodeLoad data[]; } SVnodeLoads; -typedef struct SStatusMsg { +typedef struct { int32_t sver; int32_t dnodeId; int32_t clusterId; @@ -756,6 +759,7 @@ typedef struct { int32_t dnodeId; char db[TSDB_FULL_DB_NAME_LEN]; uint64_t dbUid; + int32_t vgVersion; int32_t cacheBlockSize; int32_t totalBlocks; int32_t daysPerFile; @@ -1005,27 +1009,35 @@ typedef struct { // mq related typedef struct { + } SMqConnectReq; typedef struct { + } SMqConnectRsp; typedef struct { + } SMqDisconnectReq; typedef struct { + } SMqDisconnectRsp; typedef struct { + } SMqAckReq; typedef struct { + } SMqAckRsp; typedef struct { + } SMqResetReq; typedef struct { + } SMqResetRsp; // mq related end diff --git a/include/common/tmsgtype.h b/include/common/tmsgtype.h index 1fb10ae15b692d21f8a07937393399fe49664ce3..09765031155679accc6f6d4043e23b3b89409135 100644 --- a/include/common/tmsgtype.h +++ b/include/common/tmsgtype.h @@ -35,7 +35,7 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SELECT, "select" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_FETCH, "fetch" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_INSERT, "insert" ) - TSDB_DEFINE_SQL_TYPE( TSDB_SQL_UPDATE_TAGS_VAL, "update-tag-val" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_UPDATE_TAG_VAL, "update-tag-val" ) // the SQL below is for mgmt node TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" ) @@ -54,7 +54,7 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_TABLE, "alter-table" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_DB, "alter-db" ) - TSDB_DEFINE_SQL_TYPE(TSDB_SQL_SYNC_DB_REPLICA, "sync db-replica") + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SYNC_DB_REPLICA, "sync db-replica") TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_MNODE, "create-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_MNODE, "drop-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DNODE, "create-dnode" ) diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index 7993a8f1abdb2ad2f36964bfd861371ff96976dd..60a8c252c0b511e145e611814827626a48ed86bd 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -16,86 +16,76 @@ #ifndef _TD_TQ_H_ #define _TD_TQ_H_ +#include "common.h" #include "mallocator.h" #include "os.h" +#include "taoserror.h" +#include "taosmsg.h" +#include "tlist.h" #include "tutil.h" #ifdef __cplusplus extern "C" { #endif -typedef struct TmqMsgHead { +typedef struct STqMsgHead { int32_t protoVer; int32_t msgType; int64_t cgId; int64_t clientId; -} TmqMsgHead; +} STqMsgHead; -typedef struct TmqOneAck { +typedef struct STqOneAck { int64_t topicId; int64_t consumeOffset; -} TmqOneAck; +} STqOneAck; -typedef struct TmqAcks { +typedef struct STqAcks { int32_t ackNum; // should be sorted - TmqOneAck acks[]; -} TmqAcks; + STqOneAck acks[]; +} STqAcks; -// TODO: put msgs into common -typedef struct TmqConnectReq { - TmqMsgHead head; - TmqAcks acks; -} TmqConnectReq; - -typedef struct TmqConnectRsp { - TmqMsgHead head; - int8_t status; -} TmqConnectRsp; - -typedef struct TmqDisconnectReq { - TmqMsgHead head; -} TmqDiscconectReq; - -typedef struct TmqDisconnectRsp { - TmqMsgHead head; - int8_t status; -} TmqDisconnectRsp; +typedef struct STqSetCurReq { + STqMsgHead head; + int64_t topicId; + int64_t offset; +} STqSetCurReq; typedef struct STqConsumeReq { - TmqMsgHead head; - TmqAcks acks; + STqMsgHead head; + STqAcks acks; } STqConsumeReq; -typedef struct TmqMsgContent { +typedef struct STqMsgContent { int64_t topicId; int64_t msgLen; char msg[]; -} TmqMsgContent; +} STqMsgContent; typedef struct STqConsumeRsp { - TmqMsgHead head; + STqMsgHead head; int64_t bodySize; - TmqMsgContent msgs[]; + STqMsgContent msgs[]; } STqConsumeRsp; -typedef struct TmqSubscribeReq { - TmqMsgHead head; +typedef struct STqSubscribeReq { + STqMsgHead head; int32_t topicNum; int64_t topic[]; -} TmqSubscribeReq; +} STqSubscribeReq; -typedef struct tmqSubscribeRsp { - TmqMsgHead head; +typedef struct STqSubscribeRsp { + STqMsgHead head; int64_t vgId; char ep[TSDB_EP_LEN]; // TSDB_EP_LEN -} TmqSubscribeRsp; +} STqSubscribeRsp; -typedef struct TmqHeartbeatReq { -} TmqHeartbeatReq; +typedef struct STqHeartbeatReq { +} STqHeartbeatReq; -typedef struct TmqHeartbeatRsp { -} TmqHeartbeatRsp; +typedef struct STqHeartbeatRsp { +} STqHeartbeatRsp; typedef struct STqTopicVhandle { int64_t topicId; @@ -108,48 +98,54 @@ typedef struct STqTopicVhandle { #define TQ_BUFFER_SIZE 8 +typedef struct STqExec { + void* runtimeEnv; + SSDataBlock* (*exec)(void* runtimeEnv); + void* (*assign)(void* runtimeEnv, SSubmitBlk* inputData); + void (*clear)(void* runtimeEnv); + char* (*serialize)(struct STqExec*); + struct STqExec* (*deserialize)(char*); +} STqExec; + typedef struct STqBufferItem { int64_t offset; // executors are identical but not concurrent // so there must be a copy in each item - void* executor; - int64_t size; - void* content; -} STqBufferItem; + STqExec* executor; + int32_t status; + int64_t size; + void* content; +} STqMsgItem; -typedef struct STqBufferHandle { +typedef struct STqTopic { // char* topic; //c style, end with '\0' // int64_t cgId; // void* ahandle; - int64_t nextConsumeOffset; - int64_t floatingCursor; - int64_t topicId; - int32_t head; - int32_t tail; - STqBufferItem buffer[TQ_BUFFER_SIZE]; -} STqBufferHandle; + int64_t nextConsumeOffset; + int64_t floatingCursor; + int64_t topicId; + int32_t head; + int32_t tail; + STqMsgItem buffer[TQ_BUFFER_SIZE]; +} STqTopic; typedef struct STqListHandle { - STqBufferHandle bufHandle; + STqTopic topic; struct STqListHandle* next; -} STqListHandle; - -typedef struct STqGroupHandle { - int64_t cId; - int64_t cgId; - void* ahandle; - int32_t topicNum; - STqListHandle* head; -} STqGroupHandle; - -typedef struct STqQueryExec { - void* src; - STqBufferItem* dest; - void* executor; -} STqQueryExec; +} STqList; + +typedef struct STqGroup { + int64_t clientId; + int64_t cgId; + void* ahandle; + int32_t topicNum; + STqList* head; + SList* topicList; // SList + void* returnMsg; // SVReadMsg +} STqGroup; typedef struct STqQueryMsg { - STqQueryExec* exec; + STqMsgItem* item; struct STqQueryMsg* next; } STqQueryMsg; @@ -209,15 +205,15 @@ typedef void (*FTqDelete)(void*); #define TQ_DUP_INTXN_REWRITE 0 #define TQ_DUP_INTXN_REJECT 2 -static inline bool TqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } +static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } -static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } +static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; #define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE -typedef struct TqMetaHandle { +typedef struct STqMetaHandle { int64_t key; int64_t offset; int64_t serializedSize; @@ -225,23 +221,25 @@ typedef struct TqMetaHandle { void* valueInTxn; } STqMetaHandle; -typedef struct TqMetaList { - STqMetaHandle handle; - struct TqMetaList* next; - // struct TqMetaList* inTxnPrev; - // struct TqMetaList* inTxnNext; - struct TqMetaList* unpersistPrev; - struct TqMetaList* unpersistNext; +typedef struct STqMetaList { + STqMetaHandle handle; + struct STqMetaList* next; + // struct STqMetaList* inTxnPrev; + // struct STqMetaList* inTxnNext; + struct STqMetaList* unpersistPrev; + struct STqMetaList* unpersistNext; } STqMetaList; -typedef struct TqMetaStore { +typedef struct STqMetaStore { STqMetaList* bucket[TQ_BUCKET_SIZE]; // a table head STqMetaList* unpersistHead; + // TODO:temporaral use, to be replaced by unified tfile int fileFd; // TODO:temporaral use, to be replaced by unified tfile - int idxFd; + int idxFd; + char* dirPath; int32_t tqConfigFlag; FTqSerialize pSerializer; @@ -250,8 +248,8 @@ typedef struct TqMetaStore { } STqMetaStore; typedef struct STQ { - // the collection of group handle - // the handle of kvstore + // the collection of groups + // the handle of meta kvstore char* path; STqCfg* tqConfig; STqLogReader* tqLogReader; @@ -266,23 +264,25 @@ void tqClose(STQ*); // void* will be replace by a msg type int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); -int tqSetCursor(STQ*, void* msg); - int tqConsume(STQ*, STqConsumeReq*); -STqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId); +int tqSetCursor(STQ*, STqSetCurReq* pMsg); +int tqBufferSetOffset(STqTopic*, int64_t offset); + +STqTopic* tqFindTopic(STqGroup*, int64_t topicId); + +STqGroup* tqGetGroup(STQ*, int64_t clientId); + +STqGroup* tqOpenGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); +int tqCloseGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); +int tqRegisterContext(STqGroup*, void* ahandle); +int tqSendLaunchQuery(STqMsgItem*, int64_t offset); -STqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqMoveOffsetToNext(STqGroupHandle*); -int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset); -int tqRegisterContext(STqGroupHandle*, void* ahandle); -int tqLaunchQuery(STqGroupHandle*); -int tqSendLaunchQuery(STqGroupHandle*); +int tqSerializeGroup(const STqGroup*, STqSerializedHead**); -int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead); +const void* tqDeserializeGroup(const STqSerializedHead*, STqGroup**); -const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** gHandle); +static int tqQueryExecuting(int32_t status) { return status; } #ifdef __cplusplus } diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 8458ad9da38d009524f8eee25d902ca305c0cdb2..d51ac48f017426cae719d8b8816753da5d7ce753 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -132,6 +132,36 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); */ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +/** + * @brief Process a query message. + * + * @param pVnode The vnode object. + * @param pMsg The request message + * @param pRsp The response message + * @return int 0 for success, -1 for failure + */ +int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); + +/** + * @brief Process a fetch message. + * + * @param pVnode The vnode object. + * @param pMsg The request message + * @param pRsp The response message + * @return int 0 for success, -1 for failure + */ +int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); + +/** + * @brief Process a consume message. + * + * @param pVnode The vnode object. + * @param pMsg The request message + * @param pRsp The response message + * @return int 0 for success, -1 for failure + */ +int vnodeProcessConsumeReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); + /* ------------------------ SVnodeCfg ------------------------ */ /** * @brief Initialize VNODE options. @@ -170,73 +200,52 @@ typedef struct { char info[]; } SVnodeRsp; -#define VNODE_INIT_CREATE_STB_REQ(NAME, TTL, KEEP, SUID, PSCHEMA, PTAGSCHEMA) \ - { .ver = 0, .ctReq = META_INIT_STB_CFG(NAME, TTL, KEEP, SUID, PSCHEMA, PTAGSCHEMA) } +static FORCE_INLINE void vnodeSetCreateStbReq(SVnodeReq *pReq, char *name, uint32_t ttl, uint32_t keep, tb_uid_t suid, + STSchema *pSchema, STSchema *pTagSchema) { + pReq->ver = 0; + + pReq->ctReq.name = name; + pReq->ctReq.ttl = ttl; + pReq->ctReq.keep = keep; + pReq->ctReq.type = META_SUPER_TABLE; + pReq->ctReq.stbCfg.suid = suid; + pReq->ctReq.stbCfg.pSchema = pSchema; + pReq->ctReq.stbCfg.pTagSchema = pTagSchema; +} + +static FORCE_INLINE void vnodeSetCreateCtbReq(SVnodeReq *pReq, char *name, uint32_t ttl, uint32_t keep, tb_uid_t suid, + SKVRow pTag) { + pReq->ver = 0; -#define VNODE_INIT_CREATE_CTB_REQ(NAME, TTL, KEEP, SUID, PTAG) \ - { .ver = 0, .ctReq = META_INIT_CTB_CFG(NAME, TTL, KEEP, SUID, PTAG) } + pReq->ctReq.name = name; + pReq->ctReq.ttl = ttl; + pReq->ctReq.keep = keep; + pReq->ctReq.type = META_CHILD_TABLE; + pReq->ctReq.ctbCfg.suid = suid; + pReq->ctReq.ctbCfg.pTag = pTag; +} -#define VNODE_INIT_CREATE_NTB_REQ(NAME, TTL, KEEP, SUID, PSCHEMA) \ - { .ver = 0, .ctReq = META_INIT_NTB_CFG(NAME, TTL, KEEP, SUID, PSCHEMA) } +static FORCE_INLINE void vnodeSetCreateNtbReq(SVnodeReq *pReq, char *name, uint32_t ttl, uint32_t keep, + STSchema *pSchema) { + pReq->ver = 0; + + pReq->ctReq.name = name; + pReq->ctReq.ttl = ttl; + pReq->ctReq.keep = keep; + pReq->ctReq.type = META_NORMAL_TABLE; + pReq->ctReq.ntbCfg.pSchema = pSchema; +} int vnodeBuildReq(void **buf, const SVnodeReq *pReq, uint8_t type); void *vnodeParseReq(void *buf, SVnodeReq *pReq, uint8_t type); /* ------------------------ FOR COMPILE ------------------------ */ -#if 1 - -#include "taosmsg.h" -#include "trpc.h" - -// typedef struct { -// char db[TSDB_FULL_DB_NAME_LEN]; -// int32_t cacheBlockSize; // MB -// int32_t totalBlocks; -// int32_t daysPerFile; -// int32_t daysToKeep0; -// int32_t daysToKeep1; -// int32_t daysToKeep2; -// int32_t minRowsPerFileBlock; -// int32_t maxRowsPerFileBlock; -// int8_t precision; // time resolution -// int8_t compression; -// int8_t cacheLastRow; -// int8_t update; -// int8_t quorum; -// int8_t replica; -// int8_t selfIndex; -// int8_t walLevel; -// int32_t fsyncPeriod; // millisecond -// SReplica replicas[TSDB_MAX_REPLICA]; -// } SVnodeCfg; - -typedef enum { - VN_MSG_TYPE_WRITE = 1, - VN_MSG_TYPE_APPLY, - VN_MSG_TYPE_SYNC, - VN_MSG_TYPE_QUERY, - VN_MSG_TYPE_FETCH -} EVnMsgType; - -typedef struct { - int32_t curNum; - int32_t allocNum; - SRpcMsg rpcMsg[]; -} SVnodeMsg; - int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg); int32_t vnodeCompact(SVnode *pVnode); int32_t vnodeSync(SVnode *pVnode); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); -SVnodeMsg *vnodeInitMsg(int32_t msgNum); -int32_t vnodeAppendMsg(SVnodeMsg *pMsg, SRpcMsg *pRpcMsg); -void vnodeCleanupMsg(SVnodeMsg *pMsg); -void vnodeProcessMsg(SVnode *pVnode, SVnodeMsg *pMsg, EVnMsgType msgType); - -#endif - #ifdef __cplusplus } #endif diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 2535ec8a5b31f460dc19befb8274652a75942112..f4d45477c11af7161221e493f8c15118d358d252 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -23,64 +23,63 @@ extern "C" { #endif -typedef struct SIndex SIndex; -typedef struct SIndexTerm SIndexTerm; -typedef struct SIndexOpts SIndexOpts; +typedef struct SIndex SIndex; +typedef struct SIndexTerm SIndexTerm; +typedef struct SIndexOpts SIndexOpts; typedef struct SIndexMultiTermQuery SIndexMultiTermQuery; typedef struct SArray SIndexMultiTerm; -typedef enum { - ADD_VALUE, // add index colume value - DEL_VALUE, // delete index column value - UPDATE_VALUE, // update index column value - ADD_INDEX, // add index on specify column - DROP_INDEX, // drop existed index - DROP_SATBLE // drop stable +typedef enum { + ADD_VALUE, // add index colume value + DEL_VALUE, // delete index column value + UPDATE_VALUE, // update index column value + ADD_INDEX, // add index on specify column + DROP_INDEX, // drop existed index + DROP_SATBLE // drop stable } SIndexOperOnColumn; -typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; -typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = 3} EIndexQueryType; +typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; +typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3 } EIndexQueryType; /* - * @param: oper + * @param: oper * -*/ + */ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType oper); -void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery); -int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type); -/* - * @param: +void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery); +int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type); +/* + * @param: * @param: */ -int indexOpen(SIndexOpts *opt, const char *path, SIndex **index); -void indexClose(SIndex *index); -int indexPut(SIndex *index, SIndexMultiTerm *terms, uint64_t uid); -int indexDelete(SIndex *index, SIndexMultiTermQuery *query); -int indexSearch(SIndex *index, SIndexMultiTermQuery *query, SArray *result); -int indexRebuild(SIndex *index, SIndexOpts *opt); +int indexOpen(SIndexOpts *opt, const char *path, SIndex **index); +void indexClose(SIndex *index); +int indexPut(SIndex *index, SIndexMultiTerm *terms, uint64_t uid); +int indexDelete(SIndex *index, SIndexMultiTermQuery *query); +int indexSearch(SIndex *index, SIndexMultiTermQuery *query, SArray *result); +int indexRebuild(SIndex *index, SIndexOpts *opt); /* * @param * @param */ -SIndexMultiTerm *indexMultiTermCreate(); -int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term); -void indexMultiTermDestroy(SIndexMultiTerm *terms); +SIndexMultiTerm *indexMultiTermCreate(); +int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term); +void indexMultiTermDestroy(SIndexMultiTerm *terms); /* - * @param: + * @param: * @param: */ SIndexOpts *indexOptsCreate(); -void indexOptsDestroy(SIndexOpts *opts); +void indexOptsDestroy(SIndexOpts *opts); /* * @param: * @param: */ -SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, - const char *colName, int32_t nColName, const char *colVal, int32_t nColVal); +SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char *colName, + int32_t nColName, const char *colVal, int32_t nColVal); void indexTermDestroy(SIndexTerm *p); - #ifdef __cplusplus } #endif diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 49adaecfdda676ea4173d199977d469d7a7f3e6a..8e26fa98c945973bc6ef90aa262f2b9be20fe2b8 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -209,6 +209,7 @@ typedef struct SSourceParam { SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, const char* funcName, SSourceParam* pSource, SSchema* pResSchema, int16_t interSize); int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); +int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy); int32_t getExprFunctionLevel(SQueryStmtInfo* pQueryInfo); STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex); diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 180f78a0852240f01c3326f8743ddf7e209846b5..975b10353871f269abecb70d4974836d3e4a44b2 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -51,13 +51,15 @@ typedef struct SQueryProfileSummary { uint64_t resultSize; // generated result size in Kb. } SQueryProfileSummary; +int32_t schedulerInit(SSchedulerCfg *cfg); + /** * Process the query job, generated according to the query physical plan. * This is a synchronized API, and is also thread-safety. - * @param pJob + * @param qnodeList Qnode address list, element is SEpAddr * @return */ -int32_t scheduleQueryJob(struct SCatalog *pCatalog, void *pRpc, const SEpSet* pMgmtEps, SQueryDag* pDag, void** pJob); +int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob); int32_t scheduleFetchRows(void *pJob, void **data); @@ -71,6 +73,8 @@ int32_t scheduleCancelJob(void *pJob); void scheduleFreeJob(void *pJob); +void schedulerDestroy(void); + #ifdef __cplusplus } #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 75aab24f83822ea4dfc2ec497fe71d9cf97df185..7e8df3add27cd16ca7db47f3e996185e9da49fc1 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -230,6 +230,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x03C5) #define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x03C6) +// mnode-trans +#define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) +#define TSDB_CODE_MND_TRANS_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D1) + // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) #define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0401) @@ -339,6 +343,20 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) //"Invalid msg length") #define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) //"Invalid msg type") +// tq +#define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00) //"Invalid configuration") +#define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) //"Tq init failed") +#define TSDB_CODE_TQ_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0A02) //"No diskspace for tq") +#define TSDB_CODE_TQ_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0A03) //"No permission for disk files") +#define TSDB_CODE_TQ_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0A04) //"Data file(s) corrupted") +#define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) //"Out of memory") +#define TSDB_CODE_TQ_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0A06) //"File already exists") +#define TSDB_CODE_TQ_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0A07) //"Failed to create dir") +#define TSDB_CODE_TQ_META_NO_SUCH_KEY TAOS_DEF_ERROR_CODE(0, 0x0A08) //"Target key not found") +#define TSDB_CODE_TQ_META_KEY_NOT_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A09) //"Target key not in transaction") +#define TSDB_CODE_TQ_META_KEY_DUP_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A0A) //"Target key duplicated in transaction") +#define TSDB_CODE_TQ_GROUP_NOT_SET TAOS_DEF_ERROR_CODE(0, 0x0A0B) //"Group of corresponding client is not set by mnode") + // wal #define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) //"Unexpected generic error in wal") #define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) //"WAL file is corrupted") diff --git a/include/util/tlog.h b/include/util/tlog.h index 5e6604598dd22ad7755e4af51921fa3ee80a6abe..5c91398cdcf952046406d55e15911109a4c66435 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -42,11 +42,11 @@ extern int32_t qDebugFlag; extern int32_t wDebugFlag; extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; +extern int32_t tqDebugFlag; extern int32_t cqDebugFlag; extern int32_t debugFlag; extern int32_t ctgDebugFlag; - #define DEBUG_FATAL 1U #define DEBUG_ERROR DEBUG_FATAL #define DEBUG_WARN 2U diff --git a/include/util/tthread.h b/include/util/tthread.h index 0ff267dd1fef8e2e52d5032d15441604110655c0..7a5fd1f4c877024f2e3b8ade79a4094c1ef91ebc 100644 --- a/include/util/tthread.h +++ b/include/util/tthread.h @@ -24,8 +24,8 @@ extern "C" { #include "tdef.h" // create new thread -pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param); -// destory thread +pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param); +// destory thread bool taosDestoryThread(pthread_t* pthread); // thread running return true bool taosThreadRunning(pthread_t* pthread); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 4e7fff06b6bd60f015764745d8d093046e812fd6..a1a9155d16d1a46b608e8ae0fe6fe999babc0e21 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -86,26 +86,32 @@ typedef struct STscObj { SAppInstInfo *pAppInfo; } STscObj; -typedef struct SClientResultInfo { - const char *pMsg; +typedef struct SReqResultInfo { + const char *pRspMsg; const char *pData; TAOS_FIELD *fields; - int32_t numOfCols; - int32_t numOfRows; - int32_t current; + uint32_t numOfCols; + int32_t *length; TAOS_ROW row; char **pCol; -} SClientResultInfo; -typedef struct SReqBody { - tsem_t rspSem; // not used now - void* fp; - void* param; - int32_t paramLen; - int64_t execId; // showId/queryId - SClientResultInfo* pResInfo; -} SRequestBody; + uint32_t numOfRows; + uint32_t current; +} SReqResultInfo; + +typedef struct SReqMsg { + void *pMsg; + uint32_t len; +} SReqMsgInfo; + +typedef struct SRequestSendRecvBody { + tsem_t rspSem; // not used now + void* fp; + int64_t execId; // showId/queryId + SReqMsgInfo requestMsg; + SReqResultInfo resInfo; +} SRequestSendRecvBody; #define ERROR_MSG_BUF_DEFAULT_SIZE 512 @@ -115,7 +121,7 @@ typedef struct SRequestObj { STscObj *pTscObj; SQueryExecMetric metric; char *sqlstr; // sql string - SRequestBody body; + SRequestSendRecvBody body; int64_t self; char *msgBuf; int32_t code; @@ -123,11 +129,10 @@ typedef struct SRequestObj { } SRequestObj; typedef struct SRequestMsgBody { - int32_t msgType; - void *pData; - int32_t msgLen; - uint64_t requestId; - uint64_t requestObjRefId; + int32_t msgType; + SReqMsgInfo msgInfo; + uint64_t requestId; + uint64_t requestObjRefId; } SRequestMsgBody; extern SAppInfo appInfo; @@ -158,7 +163,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); void* doFetchRow(SRequestObj* pRequest); -void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); +void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); #ifdef __cplusplus } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index b6200de824822170e2827b041b1c768c3e7a48c1..bab1f9cc9b0aa8ece40d6c3e8a5dccbe677aae3e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -153,10 +153,9 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { void* output = NULL; int32_t outputLen = 0; code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE); - if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_CREATE_DB) { + if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_DROP_ACCT || type == TSDB_SQL_CREATE_DB || type == TSDB_SQL_CREATE_ACCT) { pRequest->type = type; - pRequest->body.param = output; - pRequest->body.paramLen = outputLen; + pRequest->body.requestMsg = (SReqMsgInfo){.pMsg = output, .len = outputLen}; SRequestMsgBody body = {0}; buildRequestMsgFp[type](pRequest, &body); @@ -165,6 +164,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); tsem_wait(&pRequest->body.rspSem); + + destroyRequestMsgBody(&body); } else { assert(0); @@ -255,7 +256,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { pMsgBody->msgType = TSDB_MSG_TYPE_CONNECT; - pMsgBody->msgLen = sizeof(SConnectMsg); + pMsgBody->msgInfo.len = sizeof(SConnectMsg); pMsgBody->requestObjRefId = pRequest->self; SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg)); @@ -279,28 +280,28 @@ static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) pConnect->startTime = htobe64(appInfo.startTime); tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app)); - pMsgBody->pData = pConnect; + pMsgBody->msgInfo.pMsg = pConnect; return 0; } static void destroyRequestMsgBody(SRequestMsgBody* pMsgBody) { assert(pMsgBody != NULL); - tfree(pMsgBody->pData); + tfree(pMsgBody->msgInfo.pMsg); } int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId) { - char *pMsg = rpcMallocCont(pBody->msgLen); + char *pMsg = rpcMallocCont(pBody->msgInfo.len); if (NULL == pMsg) { tscError("0x%"PRIx64" msg:%s malloc failed", pBody->requestId, taosMsg[pBody->msgType]); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return -1; } - memcpy(pMsg, pBody->pData, pBody->msgLen); + memcpy(pMsg, pBody->msgInfo.pMsg, pBody->msgInfo.len); SRpcMsg rpcMsg = { .msgType = pBody->msgType, .pCont = pMsg, - .contLen = pBody->msgLen, + .contLen = pBody->msgInfo.len, .ahandle = (void*) pBody->requestObjRefId, .handle = NULL, .code = 0 @@ -388,7 +389,7 @@ TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, c void* doFetchRow(SRequestObj* pRequest) { assert(pRequest != NULL); - SClientResultInfo* pResultInfo = pRequest->body.pResInfo; + SReqResultInfo* pResultInfo = &pRequest->body.resInfo; if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { pRequest->type = TSDB_SQL_RETRIEVE_MNODE; @@ -421,7 +422,7 @@ void* doFetchRow(SRequestObj* pRequest) { return pResultInfo->row; } -void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) { +void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) { assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL); if (numOfRows == 0) { return; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 21e632db8d5d3e5a5cb1885e85a0c50e12e9f3e7..8a75799ed78323c2cd63e97456e01858be60f30b 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -115,12 +115,7 @@ int taos_field_count(TAOS_RES *res) { } SRequestObj* pRequest = (SRequestObj*) res; - - SClientResultInfo* pResInfo = pRequest->body.pResInfo; - if (pResInfo == NULL) { - return 0; - } - + SReqResultInfo* pResInfo = &pRequest->body.resInfo; return pResInfo->numOfCols; } @@ -133,7 +128,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { return NULL; } - SClientResultInfo* pResInfo = ((SRequestObj*) res)->body.pResInfo; + SReqResultInfo* pResInfo = &(((SRequestObj*) res)->body.resInfo); return pResInfo->fields; } @@ -248,7 +243,7 @@ int* taos_fetch_lengths(TAOS_RES *res) { return NULL; } - return ((SRequestObj*) res)->body.pResInfo->length; + return ((SRequestObj*) res)->body.resInfo.length; } const char *taos_data_type(int type) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 36fa013f27e0b31f7d0124a3f6e8feccd19d345a..e70dcc63b38f988c055253bed0436aba0834bf4e 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#include -#include +#include "os.h" +#include "catalog.h" +#include "tname.h" #include "clientInt.h" #include "clientLog.h" -#include "os.h" #include "tmsgtype.h" #include "trpc.h" @@ -29,16 +29,6 @@ void tscProcessActivityTimer(void *handle, void *tmrId); static int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo); -static int32_t minMsgSize() { return tsRpcHeadSize + 100; } -static int32_t getWaitingTimeInterval(int32_t count) { - int32_t initial = 100; // 100 ms by default - if (count <= 1) { - return 0; - } - - return initial * ((2u)<<(count - 2)); -} - static int32_t vgIdCompare(const void *lhs, const void *rhs) { int32_t left = *(int32_t *)lhs; int32_t right = *(int32_t *)rhs; @@ -298,36 +288,6 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { taosReleaseRef(tscRefId, rid); } -int tscSendMsgToServer(SSqlObj *pSql) { - STscObj* pObj = pSql->pTscObj; - SSqlCmd* pCmd = &pSql->cmd; - - char *pMsg = rpcMallocCont(pCmd->payloadLen); - if (NULL == pMsg) { - tscError("0x%"PRIx64" msg:%s malloc failed", pSql->self, taosMsg[pSql->cmd.msgType]); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - // set the mgmt ip list - if (pSql->cmd.command >= TSDB_SQL_MGMT) { - tscDumpMgmtEpSet(pSql); - } - - memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); - - SRpcMsg rpcMsg = { - .msgType = pSql->cmd.msgType, - .pCont = pMsg, - .contLen = pSql->cmd.payloadLen, - .ahandle = (void*)pSql->self, - .handle = NULL, - .code = 0 - }; - - rpcSendRequest(pObj->pRpcObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid); - return TSDB_CODE_SUCCESS; -} - // handle three situation // 1. epset retry, only return last failure ep // 2. no epset retry, like 'taos -h invalidFqdn', return invalidFqdn @@ -354,176 +314,6 @@ void tscSetFqdnErrorMsg(SSqlObj* pSql, SRpcEpSet* pEpSet) { } } -void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { - TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle; - SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle); - if (pSql == NULL) { - rpcFreeCont(rpcMsg->pCont); - return; - } - - assert(pSql->self == handle); - - STscObj *pObj = pSql->pTscObj; - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - pSql->rpcRid = -1; - if (pObj->signature != pObj) { - tscDebug("0x%"PRIx64" DB connection is closed, cmd:%d pObj:%p signature:%p", pSql->self, pCmd->command, pObj, pObj->signature); - - taosRemoveRef(tscObjRef, handle); - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); - return; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) { - tscDebug("0x%"PRIx64" sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", - pSql->self, pCmd->command, pQueryInfo->type, pObj, pObj->signature); - - taosRemoveRef(tscObjRef, handle); - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); - return; - } - - if (pEpSet) { - if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { - if (pCmd->command < TSDB_SQL_MGMT) { - tscUpdateVgroupInfo(pSql, pEpSet); - } else { - tscUpdateMgmtEpSet(pSql, pEpSet); - } - } - } - - int32_t cmd = pCmd->command; - - // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema - if (cmd == TSDB_SQL_INSERT && rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - pSql->cmd.insertParam.schemaAttached = 1; - } - - // single table query error need to be handled here. - if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && - (((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) || - rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { - - // 1. super table subquery - // 2. nest queries are all not updated the tablemeta and retry parse the sql after cleanup local tablemeta/vgroup id buffer - if ((TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY | - TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) || - (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY)) || (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->distinct)) { - // do nothing in case of super table subquery - } else { - pSql->retry += 1; - tscWarn("0x%" PRIx64 " it shall renew table meta, code:%s, retry:%d", pSql->self, tstrerror(rpcMsg->code), pSql->retry); - - pSql->res.code = rpcMsg->code; // keep the previous error code - if (pSql->retry > pSql->maxRetry) { - tscError("0x%" PRIx64 " max retry %d reached, give up", pSql->self, pSql->maxRetry); - } else { - // wait for a little bit moment and then retry - // todo do not sleep in rpc callback thread, add this process into queue to process - if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - int32_t duration = getWaitingTimeInterval(pSql->retry); - taosMsleep(duration); - } - - pSql->retryReason = rpcMsg->code; - rpcMsg->code = tscRenewTableMeta(pSql, 0); - // if there is an error occurring, proceed to the following error handling procedure. - if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); - return; - } - } - } - } - - pRes->rspLen = 0; - - if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { - tscDebug("0x%"PRIx64" query is cancelled, code:%s", pSql->self, tstrerror(pRes->code)); - } else { - pRes->code = rpcMsg->code; - } - - if (pRes->code == TSDB_CODE_SUCCESS) { - tscDebug("0x%"PRIx64" reset retry counter to be 0 due to success rsp, old:%d", pSql->self, pSql->retry); - pSql->retry = 0; - } - - if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { - assert(rpcMsg->msgType == pCmd->msgType + 1); - pRes->code = rpcMsg->code; - pRes->rspType = rpcMsg->msgType; - pRes->rspLen = rpcMsg->contLen; - - if (pRes->rspLen > 0 && rpcMsg->pCont) { - char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); - if (tmp == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - } else { - pRes->pRsp = tmp; - memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen); - } - } else { - tfree(pRes->pRsp); - } - - /* - * There is not response callback function for submit response. - * The actual inserted number of points is the first number. - */ - if (rpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT_RSP && pRes->pRsp != NULL) { - SShellSubmitRspMsg *pMsg = (SShellSubmitRspMsg*)pRes->pRsp; - pMsg->code = htonl(pMsg->code); - pMsg->numOfRows = htonl(pMsg->numOfRows); - pMsg->affectedRows = htonl(pMsg->affectedRows); - pMsg->failedRows = htonl(pMsg->failedRows); - pMsg->numOfFailedBlocks = htonl(pMsg->numOfFailedBlocks); - - pRes->numOfRows += pMsg->affectedRows; - tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s inserted rows:%d rspLen:%d", pSql->self, sqlCmd[pCmd->command], - tstrerror(pRes->code), pMsg->affectedRows, pRes->rspLen); - } else { - tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s rspLen:%d", pSql->self, sqlCmd[pCmd->command], tstrerror(pRes->code), pRes->rspLen); - } - } - - if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) { - rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); - } - - bool shouldFree = tscShouldBeFreed(pSql); - if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - if (rpcMsg->code != TSDB_CODE_SUCCESS) { - pRes->code = rpcMsg->code; - } - - rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; - if (rpcMsg->code == TSDB_CODE_RPC_FQDN_ERROR) { - tscAllocPayload(pCmd, TSDB_FQDN_LEN + 64); - tscSetFqdnErrorMsg(pSql, pEpSet); - } - - (*pSql->fp)(pSql->param, pSql, rpcMsg->code); - } - - if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it - tscDebug("0x%"PRIx64" sqlObj is automatically freed", pSql->self); - taosRemoveRef(tscObjRef, handle); - } - - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); -} - int doBuildAndSendMsg(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; @@ -2987,51 +2777,6 @@ int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) { return code; } -static void freeElem(void* p) { - tfree(*(char**)p); -} - -/** - * retrieve table meta from mnode, and then update the local table meta hashmap. - * @param pSql sql object - * @param tableIndex table index - * @return status code - */ -int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { - SSqlCmd* pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, name); - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" failed to generate the table full name", pSql->self); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pTableMeta) { - tscDebug("0x%"PRIx64" update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRIu64, pSql->self, name, - tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); - } - - - // remove stored tableMeta info in hash table - tscResetSqlCmd(pCmd, true, pSql->self); - - SArray* pNameList = taosArrayInit(1, POINTER_BYTES); - SArray* vgroupList = taosArrayInit(1, POINTER_BYTES); - - char* n = strdup(name); - taosArrayPush(pNameList, &n); - code = getMultiTableMetaFromMnode(pSql, pNameList, vgroupList, NULL, tscTableMetaCallBack, true); - taosArrayDestroyEx(pNameList, freeElem); - taosArrayDestroyEx(vgroupList, freeElem); - - return code; -} - static bool allVgroupInfoRetrieved(SQueryInfo* pQueryInfo) { for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); @@ -3044,58 +2789,11 @@ static bool allVgroupInfoRetrieved(SQueryInfo* pQueryInfo) { return true; } -int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) { - int32_t code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - if (allVgroupInfoRetrieved(pQueryInfo)) { - return TSDB_CODE_SUCCESS; - } - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - - pNew->cmd.command = TSDB_SQL_STABLEVGROUP; - - // TODO TEST IT - SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); - if (pNewQueryInfo == NULL) { - tscFreeSqlObj(pNew); - return code; - } - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); - STableMeta* pTableMeta = tscTableMetaDup(pMInfo->pTableMeta); - tscAddTableMetaInfo(pNewQueryInfo, &pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); - } - - if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pNew); - return code; - } - - pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; - registerSqlObj(pNew); - - tscDebug("0x%"PRIx64" svgroupRid from %" PRId64 " to %" PRId64 , pSql->self, pSql->svgroupRid, pNew->self); - - pSql->svgroupRid = pNew->self; - tscDebug("0x%"PRIx64" new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql->self, pNew, pNewQueryInfo->numOfTables); - - pNew->fp = tscTableMetaCallBack; - pNew->param = (void *)pSql->self; - code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; - } - - return code; -} - #endif int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { pMsgBody->msgType = TSDB_MSG_TYPE_CONNECT; - pMsgBody->msgLen = sizeof(SConnectMsg); + pMsgBody->msgInfo.len = sizeof(SConnectMsg); pMsgBody->requestObjRefId = pRequest->self; SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg)); @@ -3119,7 +2817,7 @@ int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { pConnect->startTime = htobe64(appInfo.startTime); tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app)); - pMsgBody->pData = pConnect; + pMsgBody->msgInfo.pMsg = pConnect; return 0; } @@ -3160,26 +2858,32 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { pTscObj->pAppInfo->clusterId = pConnect->clusterId; atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); - pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo)); - pRequest->body.pResInfo->pMsg = pMsg; - + pRequest->body.resInfo.pRspMsg = pMsg; tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns); return 0; } int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { pMsgBody->requestObjRefId = pRequest->self; - pMsgBody->msgLen = pRequest->body.paramLen; - pMsgBody->pData = pRequest->body.param; + pMsgBody->msgInfo = pRequest->body.requestMsg; switch(pRequest->type) { case TSDB_SQL_CREATE_USER: pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER; break; + case TSDB_SQL_DROP_USER: + pMsgBody->msgType = TSDB_MSG_TYPE_DROP_USER; + break; + case TSDB_SQL_CREATE_ACCT: + pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_ACCT; + break; + case TSDB_SQL_DROP_ACCT: + pMsgBody->msgType = TSDB_MSG_TYPE_DROP_ACCT; + break; case TSDB_SQL_CREATE_DB: { pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_DB; - SCreateDbMsg* pCreateMsg = pRequest->body.param; + SCreateDbMsg* pCreateMsg = pRequest->body.requestMsg.pMsg; SName name = {0}; int32_t ret = tNameSetDbName(&name, pRequest->pTscObj->acctId, pCreateMsg->db, strnlen(pCreateMsg->db, tListLen(pCreateMsg->db))); if (ret != TSDB_CODE_SUCCESS) { @@ -3196,36 +2900,6 @@ int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { } } -STableMeta* createTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { - assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2); - - size_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema); - STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize); - - pTableMeta->tableType = pTableMetaMsg->tableType; - pTableMeta->vgId = pTableMetaMsg->vgId; - pTableMeta->suid = pTableMetaMsg->suid; - pTableMeta->uid = pTableMetaMsg->tuid; - - pTableMeta->tableInfo = (STableComInfo) { - .numOfTags = pTableMetaMsg->numOfTags, - .precision = pTableMetaMsg->precision, - .numOfColumns = pTableMetaMsg->numOfColumns, - }; - - pTableMeta->sversion = pTableMetaMsg->sversion; - pTableMeta->tversion = pTableMetaMsg->tversion; - - memcpy(pTableMeta->schema, pTableMetaMsg->pSchema, schemaSize); - - int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns; - for(int32_t i = 0; i < numOfTotalCols; ++i) { - pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; - } - - return pTableMeta; -} - int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { SShowRsp* pShow = (SShowRsp *)pMsg; pShow->showId = htonl(pShow->showId); @@ -3248,12 +2922,8 @@ int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) pFields[i].bytes = pSchema[i].bytes; } - if (pRequest->body.pResInfo == NULL) { - pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo)); - } - - pRequest->body.pResInfo->pMsg = pMsg; - SClientResultInfo* pResInfo = pRequest->body.pResInfo; + pRequest->body.resInfo.pRspMsg = pMsg; + SReqResultInfo* pResInfo = &pRequest->body.resInfo; pResInfo->fields = pFields; pResInfo->numOfCols = pMetaMsg->numOfColumns; @@ -3267,27 +2937,27 @@ int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) int buildRetrieveMnodeMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { pMsgBody->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - pMsgBody->msgLen = sizeof(SRetrieveTableMsg); + pMsgBody->msgInfo.len = sizeof(SRetrieveTableMsg); pMsgBody->requestObjRefId = pRequest->self; SRetrieveTableMsg *pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg)); pRetrieveMsg->showId = htonl(pRequest->body.execId); - pMsgBody->pData = pRetrieveMsg; + pMsgBody->msgInfo.pMsg = pRetrieveMsg; return TSDB_CODE_SUCCESS; } int32_t processRetrieveMnodeRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { assert(msgLen >= sizeof(SRetrieveTableRsp)); - tfree(pRequest->body.pResInfo->pMsg); - pRequest->body.pResInfo->pMsg = pMsg; + tfree(pRequest->body.resInfo.pRspMsg); + pRequest->body.resInfo.pRspMsg = pMsg; SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *) pMsg; pRetrieve->numOfRows = htonl(pRetrieve->numOfRows); pRetrieve->precision = htons(pRetrieve->precision); - SClientResultInfo* pResInfo = pRequest->body.pResInfo; + SReqResultInfo* pResInfo = &pRequest->body.resInfo; pResInfo->numOfRows = pRetrieve->numOfRows; pResInfo->pData = pRetrieve->data; // todo fix this in async model @@ -3328,7 +2998,7 @@ void initMsgHandleFp() { tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; - tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAG_VAL] = tscBuildUpdateTagMsg; tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; @@ -3383,6 +3053,10 @@ void initMsgHandleFp() { handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp; buildRequestMsgFp[TSDB_SQL_CREATE_USER] = doBuildMsgSupp; + buildRequestMsgFp[TSDB_SQL_DROP_USER] = doBuildMsgSupp; + + buildRequestMsgFp[TSDB_SQL_CREATE_ACCT] = doBuildMsgSupp; + buildRequestMsgFp[TSDB_SQL_DROP_ACCT] = doBuildMsgSupp; buildRequestMsgFp[TSDB_SQL_SHOW] = doBuildMsgSupp; handleRequestRspFp[TSDB_SQL_SHOW] = processShowRsp; diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c index 023bd6ebe977c956f201d45fb63da31066d33ee2..182e330df7f9419c09ec1f9643e77df830a9a786 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/tscEnv.c @@ -170,7 +170,7 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty pRequest->type = type; pRequest->pTscObj = pObj; pRequest->body.fp = fp; - pRequest->body.param = param; +// pRequest->body.requestMsg. = param; pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); tsem_init(&pRequest->body.rspSem, 0, 0); @@ -188,11 +188,7 @@ static void doDestroyRequest(void* p) { tfree(pRequest->sqlstr); tfree(pRequest->pInfo); - if (pRequest->body.pResInfo != NULL) { - tfree(pRequest->body.pResInfo->pData); - tfree(pRequest->body.pResInfo->pMsg); - tfree(pRequest->body.pResInfo); - } + tfree(pRequest->body.resInfo.pRspMsg); deregisterRequest(pRequest); tfree(pRequest); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 944247e88e57901d57bdaf18a1025c53b08ad107..1c8cd6a4b6da560f0deec532c008863b6e7eb786 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -40,13 +40,13 @@ TEST(testCase, driverInit_Test) { TEST(testCase, connect_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); - assert(pConn != NULL); +// assert(pConn != NULL); taos_close(pConn); } TEST(testCase, create_user_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); - assert(pConn != NULL); +// assert(pConn != NULL); TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { @@ -57,23 +57,36 @@ TEST(testCase, create_user_Test) { taos_close(pConn); } -//TEST(testCase, drop_user_Test) { -// TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} +TEST(testCase, create_account_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); -TEST(testCase, show_user_Test) { + TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_account_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_user_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "show users"); TAOS_ROW pRow = NULL; @@ -89,10 +102,23 @@ TEST(testCase, show_user_Test) { taos_close(pConn); } -TEST(testCase, show_db_Test) { +TEST(testCase, drop_user_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "drop user abc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_db_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "show databases"); TAOS_ROW pRow = NULL; @@ -112,7 +138,7 @@ TEST(testCase, create_db_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); - TAOS_RES* pRes = taos_query(pConn, "create database abc"); + TAOS_RES* pRes = taos_query(pConn, "create database abc1"); TAOS_FIELD* pFields = taos_fetch_fields(pRes); ASSERT_TRUE(pFields == NULL); diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index b9a1e4cecbfe333591844c3ebbaebe5a96a0c741..083935c706e06abee8d07a74e9b4515350671e39 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -31,7 +31,7 @@ static struct { } global = {0}; void dmnSigintHandle(int signum, void *info, void *ctx) { - uError("singal:%d is received", signum); + uInfo("singal:%d is received", signum); global.stop = true; } diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 3402d3ff68f398329109d73813d8538b2347f1f0..50b1a1cf20fa2db10c401911aff6b8178f374d64 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -44,7 +44,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET_CUR] = dndProcessVnodeWriteMsg; // msg from client to mnode pMgmt->msgFp[TSDB_MSG_TYPE_CONNECT] = dndProcessMnodeReadMsg; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index d3f1b06a4a88fd5d96f760ed1ae7e2f9a968a3bf..598d6e47bed1987587dbc7fcc9dff25df68457dc 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -17,11 +17,23 @@ #include "dndVnodes.h" #include "dndTransport.h" +typedef struct { + int32_t vgId; + int32_t vgVersion; + int8_t dropped; + uint64_t dbUid; + char db[TSDB_FULL_DB_NAME_LEN]; + char path[PATH_MAX + 20]; +} SWrapperCfg; + typedef struct { int32_t vgId; int32_t refCount; + int32_t vgVersion; int8_t dropped; int8_t accessState; + uint64_t dbUid; + char *db; char *path; SVnode *pImpl; taos_queue pWriteQ; @@ -32,13 +44,13 @@ typedef struct { } SVnodeObj; typedef struct { - int32_t vnodeNum; - int32_t opened; - int32_t failed; - int32_t threadIndex; - pthread_t *pThreadId; - SVnodeObj *pVnodes; - SDnode * pDnode; + int32_t vnodeNum; + int32_t opened; + int32_t failed; + int32_t threadIndex; + pthread_t thread; + SDnode *pDnode; + SWrapperCfg *pCfgs; } SVnodeThread; static int32_t dndInitVnodeReadWorker(SDnode *pDnode); @@ -60,8 +72,8 @@ static void dndFreeVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode); static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode); static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg); -static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg); +static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); +static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); @@ -71,18 +83,16 @@ void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pE void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg); +static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg); -static SVnodeObj * dndAcquireVnode(SDnode *pDnode, int32_t vgId); +static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId); static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode); -static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, SVnode *pImpl); -static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode); +static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl); +static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode); static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes); -static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes); +static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); static int32_t dndWriteVnodesToFile(SDnode *pDnode); -static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg); -static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode); static int32_t dndOpenVnodes(SDnode *pDnode); static void dndCloseVnodes(SDnode *pDnode); @@ -126,22 +136,25 @@ static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) { dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); } -static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, SVnode *pImpl) { +static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodeObj * pVnode = calloc(1, sizeof(SVnodeObj)); + SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj)); if (pVnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pVnode->vgId = vgId; - pVnode->refCount = 0; + pVnode->vgId = pCfg->vgId; + pVnode->refCount = 1; pVnode->dropped = 0; pVnode->accessState = TSDB_VN_ALL_ACCCESS; pVnode->pImpl = pImpl; + pVnode->vgVersion = pCfg->vgVersion; + pVnode->dbUid = pCfg->dbUid; + pVnode->db = tstrdup(pCfg->db); + pVnode->path = tstrdup(pCfg->path); - pVnode->path = tstrdup(path); - if (pVnode->path == NULL) { + if (pVnode->path == NULL || pVnode->db == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -167,7 +180,7 @@ static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, S } taosWLockLatch(&pMgmt->latch); - int32_t code = taosHashPut(pMgmt->hash, &vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); + int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); taosWUnLockLatch(&pMgmt->latch); if (code != 0) { @@ -176,7 +189,7 @@ static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, S return code; } -static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode) { +static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; taosWLockLatch(&pMgmt->latch); taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); @@ -195,6 +208,9 @@ static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode) { dndFreeVnodeWriteQueue(pDnode, pVnode); dndFreeVnodeApplyQueue(pDnode, pVnode); dndFreeVnodeSyncQueue(pDnode, pVnode); + free(pVnode->path); + free(pVnode->db); + free(pVnode); } static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { @@ -208,16 +224,16 @@ static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { void *pIter = taosHashIterate(pMgmt->hash, NULL); while (pIter) { SVnodeObj **ppVnode = pIter; - SVnodeObj * pVnode = *ppVnode; - if (pVnode) { + SVnodeObj *pVnode = *ppVnode; + if (pVnode && num < size) { + int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); + pVnodes[num] = (*ppVnode); num++; - if (num < size) { - int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); - pVnodes[num] = (*ppVnode); - } + pIter = taosHashIterate(pMgmt->hash, pIter); + } else { + taosHashCancelIterate(pMgmt->hash, pIter); } - pIter = taosHashIterate(pMgmt->hash, pIter); } taosRUnLockLatch(&pMgmt->latch); @@ -226,15 +242,15 @@ static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { return pVnodes; } -static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes) { - int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 30000; - char * content = calloc(1, maxLen + 1); - cJSON * root = NULL; - FILE * fp = NULL; - char file[PATH_MAX + 20] = {0}; - SVnodeObj *pVnodes = NULL; +static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { + int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 30000; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + FILE *fp = NULL; + char file[PATH_MAX + 20] = {0}; + SWrapperCfg *pCfgs = NULL; snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); @@ -270,31 +286,55 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_ goto PRASE_VNODE_OVER; } - pVnodes = calloc(vnodesNum, sizeof(SVnodeObj)); - if (pVnodes == NULL) { + pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg)); + if (pCfgs == NULL) { dError("failed to read %s since out of memory", file); goto PRASE_VNODE_OVER; } for (int32_t i = 0; i < vnodesNum; ++i) { - cJSON * vnode = cJSON_GetArrayItem(vnodes, i); - SVnodeObj *pVnode = &pVnodes[i]; + cJSON *vnode = cJSON_GetArrayItem(vnodes, i); + SWrapperCfg *pCfg = &pCfgs[i]; cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); - if (!vgId || vgId->type != cJSON_String) { + if (!vgId || vgId->type != cJSON_Number) { dError("failed to read %s since vgId not found", file); goto PRASE_VNODE_OVER; } - pVnode->vgId = atoi(vgId->valuestring); + pCfg->vgId = vgId->valueint; + snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCfg->vgId); cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); - if (!dropped || dropped->type != cJSON_String) { + if (!dropped || dropped->type != cJSON_Number) { dError("failed to read %s since dropped not found", file); goto PRASE_VNODE_OVER; } - pVnode->dropped = atoi(vnode->valuestring); + pCfg->dropped = dropped->valueint; + + cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion"); + if (!vgVersion || vgVersion->type != cJSON_Number) { + dError("failed to read %s since vgVersion not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->vgVersion = vgVersion->valueint; + + cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid"); + if (!dbUid || dbUid->type != cJSON_String) { + dError("failed to read %s since dbUid not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->dbUid = atoll(dbUid->valuestring); + + cJSON *db = cJSON_GetObjectItem(vnode, "db"); + if (!db || db->type != cJSON_String) { + dError("failed to read %s since db not found", file); + goto PRASE_VNODE_OVER; + } + tstrncpy(pCfg->db, db->valuestring, TSDB_FULL_DB_NAME_LEN); } + *ppCfgs = pCfgs; + *numOfVnodes = vnodesNum; code = 0; dInfo("succcessed to read file %s", file); @@ -313,30 +353,35 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); FILE *fp = fopen(file, "w"); - if (fp != NULL) { + if (fp == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write %s since %s", file, terrstr()); return -1; } - - int32_t len = 0; - int32_t maxLen = 30000; - char * content = calloc(1, maxLen + 1); int32_t numOfVnodes = 0; SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); + int32_t len = 0; + int32_t maxLen = 65536; + char *content = calloc(1, maxLen + 1); + len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"vnodes\": [{\n"); + len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); for (int32_t i = 0; i < numOfVnodes; ++i) { SVnodeObj *pVnode = pVnodes[i]; - len += snprintf(content + len, maxLen - len, " \"vgId\": \"%d\",\n", pVnode->vgId); - len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\"\n", pVnode->dropped); + len += snprintf(content + len, maxLen - len, " {\n"); + len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped); + len += snprintf(content + len, maxLen - len, " \"vgVersion\": %d,\n", pVnode->vgVersion); + len += snprintf(content + len, maxLen - len, " \"dbUid\": \"%" PRIu64 "\",\n", pVnode->dbUid); + len += snprintf(content + len, maxLen - len, " \"db\": \"%s\"\n", pVnode->db); if (i < numOfVnodes - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); + len += snprintf(content + len, maxLen - len, " },\n"); } else { - len += snprintf(content + len, maxLen - len, " }]\n"); + len += snprintf(content + len, maxLen - len, " }\n"); } } + len += snprintf(content + len, maxLen - len, " ]\n"); len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); @@ -358,74 +403,29 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { return taosRenameFile(file, realfile); } -static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg) { - char path[PATH_MAX + 20] = {0}; - snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, vgId); - // SVnode *pImpl = vnodeCreate(vgId, path, pCfg); - - SVnode *pImpl = vnodeOpen(path, NULL); - if (pImpl == NULL) { - return -1; - } - - int32_t code = dndCreateVnodeWrapper(pDnode, vgId, path, pImpl); - if (code != 0) { - vnodeClose(pImpl); - vnodeDestroy(path); - terrno = code; - return code; - } - - code = dndWriteVnodesToFile(pDnode); - if (code != 0) { - vnodeClose(pImpl); - vnodeDestroy(path); - terrno = code; - return code; - } - - return 0; -} - -static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode) { - pVnode->dropped = 1; - if (dndWriteVnodesToFile(pDnode) != 0) { - pVnode->dropped = 0; - return -1; - } - - dndDropVnodeWrapper(pDnode, pVnode); - vnodeClose(pVnode->pImpl); - vnodeDestroy(pVnode->path); - dndWriteVnodesToFile(pDnode); - return 0; -} - static void *dnodeOpenVnodeFunc(void *param) { SVnodeThread *pThread = param; - SDnode * pDnode = pThread->pDnode; - SVnodesMgmt * pMgmt = &pDnode->vmgmt; + SDnode *pDnode = pThread->pDnode; + SVnodesMgmt *pMgmt = &pDnode->vmgmt; dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); setThreadName("open-vnodes"); for (int32_t v = 0; v < pThread->vnodeNum; ++v) { - SVnodeObj *pVnode = &pThread->pVnodes[v]; + SWrapperCfg *pCfg = &pThread->pCfgs[v]; char stepDesc[TSDB_STEP_DESC_LEN] = {0}; - snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pVnode->vgId, + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId, pMgmt->openVnodes, pMgmt->totalVnodes); dndReportStartup(pDnode, "open-vnodes", stepDesc); - char path[PATH_MAX + 20] = {0}; - snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, pVnode->vgId); - SVnode *pImpl = vnodeOpen(path, NULL); + SVnode *pImpl = vnodeOpen(pCfg->path, NULL); if (pImpl == NULL) { - dError("vgId:%d, failed to open vnode by thread:%d", pVnode->vgId, pThread->threadIndex); + dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->failed++; } else { - dndCreateVnodeWrapper(pDnode, pVnode->vgId, path, pImpl); - dDebug("vgId:%d, is opened by thread:%d", pVnode->vgId, pThread->threadIndex); + dndOpenVnode(pDnode, pCfg, pImpl); + dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->opened++; } @@ -448,9 +448,9 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { return -1; } - SVnodeObj *pVnodes = NULL; - int32_t numOfVnodes = 0; - if (dndGetVnodesFromFile(pDnode, &pVnodes, &numOfVnodes) != 0) { + SWrapperCfg *pCfgs = NULL; + int32_t numOfVnodes = 0; + if (dndGetVnodesFromFile(pDnode, &pCfgs, &numOfVnodes) != 0) { dInfo("failed to get vnode list from disk since %s", terrstr()); return -1; } @@ -463,13 +463,14 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); for (int32_t t = 0; t < threadNum; ++t) { threads[t].threadIndex = t; - threads[t].pVnodes = calloc(vnodesPerThread, sizeof(SVnodeObj)); + threads[t].pDnode = pDnode; + threads[t].pCfgs = calloc(vnodesPerThread, sizeof(SWrapperCfg)); } for (int32_t v = 0; v < numOfVnodes; ++v) { int32_t t = v % threadNum; SVnodeThread *pThread = &threads[t]; - pThread->pVnodes[pThread->vnodeNum++] = pVnodes[v]; + pThread->pCfgs[pThread->vnodeNum++] = pCfgs[v]; } dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); @@ -478,19 +479,25 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { SVnodeThread *pThread = &threads[t]; if (pThread->vnodeNum == 0) continue; - pThread->pThreadId = taosCreateThread(dnodeOpenVnodeFunc, pThread); - if (pThread->pThreadId == NULL) { + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnodeFunc, pThread) != 0) { dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); } + + pthread_attr_destroy(&thAttr); } for (int32_t t = 0; t < threadNum; ++t) { SVnodeThread *pThread = &threads[t]; - taosDestoryThread(pThread->pThreadId); - pThread->pThreadId = NULL; - free(pThread->pVnodes); + if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { + pthread_join(pThread->thread, NULL); + } + free(pThread->pCfgs); } free(threads); + free(pCfgs); if (pMgmt->openVnodes != pMgmt->totalVnodes) { dError("there are total vnodes:%d, opened:%d", pMgmt->totalVnodes, pMgmt->openVnodes); @@ -508,7 +515,8 @@ static void dndCloseVnodes(SDnode *pDnode) { SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); for (int32_t i = 0; i < numOfVnodes; ++i) { - dndDropVnodeWrapper(pDnode, pVnodes[i]); + dndReleaseVnode(pDnode, pVnodes[i]); + dndCloseVnode(pDnode, pVnodes[i]); } if (pVnodes != NULL) { @@ -523,11 +531,12 @@ static void dndCloseVnodes(SDnode *pDnode) { dInfo("total vnodes:%d are all closed", numOfVnodes); } -static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg *pCfg) { +static SCreateVnodeMsg *dndParseCreateVnodeReq(SRpcMsg *rpcMsg) { SCreateVnodeMsg *pCreate = rpcMsg->pCont; pCreate->vgId = htonl(pCreate->vgId); pCreate->dnodeId = htonl(pCreate->dnodeId); pCreate->dbUid = htobe64(pCreate->dbUid); + pCreate->vgVersion = htonl(pCreate->vgVersion); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); pCreate->totalBlocks = htonl(pCreate->totalBlocks); pCreate->daysPerFile = htonl(pCreate->daysPerFile); @@ -544,9 +553,10 @@ static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg pReplica->port = htons(pReplica->port); } - *vgId = pCreate->vgId; + return pCreate; +} -#if 0 +static void dndGenerateVnodeCfg(SCreateVnodeMsg *pCreate, SVnodeCfg *pCfg) { pCfg->wsize = pCreate->cacheBlockSize; pCfg->ssize = pCreate->cacheBlockSize; pCfg->wsize = pCreate->cacheBlockSize; @@ -567,8 +577,15 @@ static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg pCfg->walCfg.rollPeriod = 128; pCfg->walCfg.segSize = 128; pCfg->walCfg.vgId = pCreate->vgId; -#endif - return 0; +} + +static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeMsg *pCreate, SWrapperCfg *pCfg) { + memcpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN); + pCfg->dbUid = pCreate->dbUid; + pCfg->dropped = 0; + snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCreate->vgId); + pCfg->vgId = pCreate->vgId; + pCfg->vgVersion = pCreate->vgVersion; } static SDropVnodeMsg *vnodeParseDropVnodeReq(SRpcMsg *rpcMsg) { @@ -584,48 +601,83 @@ static SAuthVnodeMsg *vnodeParseAuthVnodeReq(SRpcMsg *rpcMsg) { } static int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { + SCreateVnodeMsg *pCreate = dndParseCreateVnodeReq(rpcMsg); + dDebug("vgId:%d, create vnode req is received", pCreate->vgId); + SVnodeCfg vnodeCfg = {0}; - int32_t vgId = 0; + dndGenerateVnodeCfg(pCreate, &vnodeCfg); - dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg); - dDebug("vgId:%d, create vnode req is received", vgId); + SWrapperCfg wrapperCfg = {0}; + dndGenerateWrapperCfg(pDnode, pCreate, &wrapperCfg); - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); + SVnodeObj *pVnode = dndAcquireVnode(pDnode, pCreate->vgId); if (pVnode != NULL) { - dDebug("vgId:%d, already exist, return success", vgId); + dDebug("vgId:%d, already exist, return success", pCreate->vgId); dndReleaseVnode(pDnode, pVnode); return 0; } - if (dndCreateVnode(pDnode, vgId, &vnodeCfg) != 0) { - dError("vgId:%d, failed to create vnode since %s", vgId, terrstr()); - return terrno; + SVnode *pImpl = vnodeOpen(wrapperCfg.path, NULL /*pCfg*/); + if (pImpl == NULL) { + return -1; + } + + int32_t code = dndOpenVnode(pDnode, &wrapperCfg, pImpl); + if (code != 0) { + vnodeClose(pImpl); + vnodeDestroy(wrapperCfg.path); + terrno = code; + return code; + } + + code = dndWriteVnodesToFile(pDnode); + if (code != 0) { + vnodeClose(pImpl); + vnodeDestroy(wrapperCfg.path); + terrno = code; + return code; } return 0; } static int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { + SAlterVnodeMsg *pAlter = (SAlterVnodeMsg *)dndParseCreateVnodeReq(rpcMsg); + dDebug("vgId:%d, alter vnode req is received", pAlter->vgId); + SVnodeCfg vnodeCfg = {0}; - int32_t vgId = 0; + dndGenerateVnodeCfg(pAlter, &vnodeCfg); - dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg); - dDebug("vgId:%d, alter vnode req is received", vgId); + SWrapperCfg wrapperCfg = {0}; + dndGenerateWrapperCfg(pDnode, pAlter, &wrapperCfg); - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); + SVnodeObj *pVnode = dndAcquireVnode(pDnode, pAlter->vgId); if (pVnode == NULL) { - dDebug("vgId:%d, failed to alter vnode since %s", vgId, terrstr()); + dDebug("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr()); return terrno; } + if (wrapperCfg.vgVersion == pVnode->vgVersion) { + dndReleaseVnode(pDnode, pVnode); + dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", pAlter->vgId); + return 0; + } + if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { - dError("vgId:%d, failed to alter vnode since %s", vgId, terrstr()); + dError("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr()); dndReleaseVnode(pDnode, pVnode); return terrno; } + int32_t oldVersion = pVnode->vgVersion; + pVnode->vgVersion = wrapperCfg.vgVersion; + int32_t code = dndWriteVnodesToFile(pDnode); + if (code != 0) { + pVnode->vgVersion = oldVersion; + } + dndReleaseVnode(pDnode, pVnode); - return 0; + return code; } static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { @@ -637,15 +689,21 @@ static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); if (pVnode == NULL) { dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); - return terrno; + return 0; } - if (dndDropVnode(pDnode, pVnode) != 0) { - dError("vgId:%d, failed to drop vnode since %s", vgId, terrstr()); - dndReleaseVnode(pDnode, pVnode); + pVnode->dropped = 1; + if (dndWriteVnodesToFile(pDnode) != 0) { + pVnode->dropped = 0; return terrno; } + dndReleaseVnode(pDnode, pVnode); + dndCloseVnode(pDnode, pVnode); + vnodeClose(pVnode->pImpl); + vnodeDestroy(pVnode->path); + dndWriteVnodesToFile(pDnode); + return 0; } @@ -738,48 +796,76 @@ static void dndProcessVnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { break; } - if (code != 0) { - SRpcMsg rsp = {.code = code, .handle = pMsg->handle}; - rpcSendResponse(&rsp); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } + SRpcMsg rsp = {.code = code, .handle = pMsg->handle, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rsp); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); } -static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) { - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_QUERY); +static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { + SRpcMsg *pRsp = NULL; + vnodeProcessQueryReq(pVnode->pImpl, pMsg, &pRsp); } -static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) { - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_FETCH); +static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { + SRpcMsg *pRsp = NULL; + vnodeProcessFetchReq(pVnode->pImpl, pMsg, &pRsp); } static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { - SVnodeMsg *pMsg = vnodeInitMsg(numOfMsgs); - SRpcMsg * pRpcMsg = NULL; + SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pRpcMsg); - vnodeAppendMsg(pMsg, pRpcMsg); - taosFreeQitem(pRpcMsg); + SRpcMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + void *ptr = taosArrayPush(pArray, &pMsg); + assert(ptr != NULL); } - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_WRITE); + vnodeProcessWMsgs(pVnode->pImpl, pArray); + + for (size_t i = 0; i < numOfMsgs; i++) { + SRpcMsg *pRsp = NULL; + SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); + int32_t code = vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp); + if (pRsp != NULL) { + rpcSendResponse(pRsp); + free(pRsp); + } else { + if (code != 0) code = terrno; + SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; + rpcSendResponse(&rpcRsp); + } + } + + for (size_t i = 0; i < numOfMsgs; i++) { + SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + } + + taosArrayDestroy(pArray); } static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { - SVnodeMsg *pMsg = NULL; + SRpcMsg *pMsg = NULL; + for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_APPLY); + + SRpcMsg *pRsp = NULL; + (void)vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp); } } static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { - SVnodeMsg *pMsg = NULL; + SRpcMsg *pMsg = NULL; + for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_SYNC); + + SRpcMsg *pRsp = NULL; + (void)vnodeProcessSyncReq(pVnode->pImpl, pMsg, &pRsp); } } @@ -807,40 +893,14 @@ static int32_t dndWriteRpcMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) { } } -static int32_t dndWriteVnodeMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) { - int32_t code = 0; - - if (pQueue == NULL) { - code = TSDB_CODE_MSG_NOT_PROCESSED; - } else { - SVnodeMsg *pMsg = vnodeInitMsg(1); - if (pMsg == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - if (vnodeAppendMsg(pMsg, pRpcMsg) != 0) { - code = terrno; - } else { - if (taosWriteQitem(pQueue, pMsg) != 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - } - } - } - - if (code != TSDB_CODE_SUCCESS) { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; - rpcSendResponse(&rsp); - rpcFreeCont(pRpcMsg->pCont); - } -} - static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) { SMsgHead *pHead = (SMsgHead *)pMsg->pCont; + pHead->contLen = htonl(pHead->contLen); pHead->vgId = htonl(pHead->vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); if (pVnode == NULL) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; + SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; rpcSendResponse(&rsp); rpcFreeCont(pMsg->pCont); } @@ -864,7 +924,7 @@ void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); if (pVnode != NULL) { - dndWriteVnodeMsgToVnodeQueue(pVnode->pSyncQ, pMsg); + dndWriteRpcMsgToVnodeQueue(pVnode->pSyncQ, pMsg); dndReleaseVnode(pDnode, pVnode); } } @@ -872,7 +932,7 @@ void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); if (pVnode != NULL) { - dndWriteVnodeMsgToVnodeQueue(pVnode->pQueryQ, pMsg); + dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg); dndReleaseVnode(pDnode, pVnode); } } @@ -880,12 +940,12 @@ void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); if (pVnode != NULL) { - dndWriteVnodeMsgToVnodeQueue(pVnode->pFetchQ, pMsg); + dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg); dndReleaseVnode(pDnode, pVnode); } } -static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg) { +static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); if (pVnode == NULL) { return -1; @@ -1029,7 +1089,7 @@ static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) { } static int32_t dndInitVnodeWriteWorker(SDnode *pDnode) { - SVnodesMgmt * pMgmt = &pDnode->vmgmt; + SVnodesMgmt *pMgmt = &pDnode->vmgmt; SMWorkerPool *pPool = &pMgmt->writePool; pPool->name = "vnode-write"; pPool->max = pDnode->opt.numOfCores; @@ -1050,7 +1110,7 @@ static void dndCleanupVnodeWriteWorker(SDnode *pDnode) { static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue); + pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue); if (pVnode->pSyncQ == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -1061,7 +1121,7 @@ static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) { static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pSyncQ); + tMWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); pVnode->pSyncQ = NULL; } @@ -1137,12 +1197,12 @@ void dndGetVnodeLoads(SDnode *pDnode, SVnodeLoads *pLoads) { pLoads->num = taosHashGetSize(pMgmt->hash); int32_t v = 0; - void * pIter = taosHashIterate(pMgmt->hash, NULL); + void *pIter = taosHashIterate(pMgmt->hash, NULL); while (pIter) { SVnodeObj **ppVnode = pIter; if (ppVnode == NULL || *ppVnode == NULL) continue; - SVnodeObj * pVnode = *ppVnode; + SVnodeObj *pVnode = *ppVnode; SVnodeLoad *pLoad = &pLoads->data[v++]; vnodeGetLoad(pVnode->pImpl, pLoad); diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index 0dbee0d3379daa55e69ba6ed5e0b2beaa400dec7..a4996ecb3bfc0792add96700c51c4b83dcf6d80e 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -176,6 +176,12 @@ SDnode *dndInit(SDnodeOpt *pOption) { return NULL; } + if (vnodeInit(1) != 0) { + dError("failed to init vnode env"); + dndCleanup(pDnode); + return NULL; + } + if (dndInitDnode(pDnode) != 0) { dError("failed to init dnode"); dndCleanup(pDnode); @@ -222,8 +228,10 @@ void dndCleanup(SDnode *pDnode) { dndCleanupMnode(pDnode); dndCleanupVnodes(pDnode); dndCleanupDnode(pDnode); + vnodeClear(); walCleanUp(); rpcCleanup(); + dndCleanupEnv(pDnode); free(pDnode); dInfo("TDengine is cleaned up successfully"); diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp index d465a62f2d5269632c4f5022e0bafca3a1e45670..204afa111fdd509aac44937774410a8165905314 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -210,8 +210,8 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep2 = htonl(3650); - pReq->minRowsPerFileBlock = htonl(100); - pReq->maxRowsPerFileBlock = htonl(4096); + pReq->minRows = htonl(100); + pReq->maxRows = htonl(4096); pReq->commitTime = htonl(3600); pReq->fsyncPeriod = htonl(3000); pReq->walLevel = 1; @@ -375,8 +375,8 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep2 = htonl(3650); - pReq->minRowsPerFileBlock = htonl(100); - pReq->maxRowsPerFileBlock = htonl(4096); + pReq->minRows = htonl(100); + pReq->maxRows = htonl(4096); pReq->commitTime = htonl(3600); pReq->fsyncPeriod = htonl(3000); pReq->walLevel = 1; diff --git a/source/dnode/mgmt/impl/test/stb/stb.cpp b/source/dnode/mgmt/impl/test/stb/stb.cpp index ee8c86bcea3b5764ca3158deabbced027b70d4ee..c12e8eadf45111af660126837fa1157212f8bd57 100644 --- a/source/dnode/mgmt/impl/test/stb/stb.cpp +++ b/source/dnode/mgmt/impl/test/stb/stb.cpp @@ -187,8 +187,8 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep2 = htonl(3650); - pReq->minRowsPerFileBlock = htonl(100); - pReq->maxRowsPerFileBlock = htonl(4096); + pReq->minRows = htonl(100); + pReq->maxRows = htonl(4096); pReq->commitTime = htonl(3600); pReq->fsyncPeriod = htonl(3000); pReq->walLevel = 1; diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index f2010b581388f24359ff2c15f4517a3fded22047..7cf469f8e253f0354b74cbdda7d2e8cf6cc8a2c5 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -16,7 +16,7 @@ #include "deploy.h" void initLog(const char* path) { - dDebugFlag = 143; + dDebugFlag = 207; vDebugFlag = 0; mDebugFlag = 207; cDebugFlag = 0; diff --git a/source/dnode/mgmt/impl/test/sut/deploy.h b/source/dnode/mgmt/impl/test/sut/deploy.h index 4c082df5f3b1592b474e1d62fa0068f28bde1233..88d9b06fbb52ca421bec4a100fb5849bc2626b1f 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.h +++ b/source/dnode/mgmt/impl/test/sut/deploy.h @@ -46,3 +46,4 @@ void stopServer(SServer* pServer); SClient* createClient(const char* user, const char* pass, const char* fqdn, uint16_t port); void dropClient(SClient* pClient); void sendMsg(SClient* pClient, SRpcMsg* pMsg); + diff --git a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp b/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp index 3f16cd87d889902b3e7d34fbbf6a2cb2d83e6771..9149be9cd838be6ddd708a1fa56e85f6dc5f2dae 100644 --- a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp +++ b/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp @@ -176,49 +176,112 @@ SServer* DndTestVgroup::pServer; SClient* DndTestVgroup::pClient; int32_t DndTestVgroup::connId; - TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { { - SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(sizeof(SCreateVnodeMsg)); - pReq->vgId = htonl(2); - pReq->dnodeId = htonl(1); - strcpy(pReq->db, "1.d1"); - pReq->dbUid = htobe64(9527); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->minRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replica = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->selfIndex = 0; - for (int r = 0; r < pReq->replica; ++r) { - SReplica* pReplica = &pReq->replicas[r]; - pReplica->id = htonl(1); - pReplica->port = htons(9150); + for (int i = 0; i < 3; ++i) { + SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(sizeof(SCreateVnodeMsg)); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(1); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + pReq->vgVersion = htonl(1); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRows = htonl(100); + pReq->minRows = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replica = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->selfIndex = 0; + for (int r = 0; r < pReq->replica; ++r) { + SReplica* pReplica = &pReq->replicas[r]; + pReplica->id = htonl(1); + pReplica->port = htons(9150); + } + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateVnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); } + } - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateVnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - taosMsleep(1000000); + { + for (int i = 0; i < 3; ++i) { + SAlterVnodeMsg* pReq = (SAlterVnodeMsg*)rpcMallocCont(sizeof(SAlterVnodeMsg)); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(1); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + pReq->vgVersion = htonl(2); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRows = htonl(100); + pReq->minRows = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replica = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->selfIndex = 0; + for (int r = 0; r < pReq->replica; ++r) { + SReplica* pReplica = &pReq->replicas[r]; + pReplica->id = htonl(1); + pReplica->port = htons(9150); + } + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SAlterVnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_VNODE_IN; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } } + { + for (int i = 0; i < 3; ++i) { + SDropVnodeMsg* pReq = (SDropVnodeMsg*)rpcMallocCont(sizeof(SDropVnodeMsg)); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(1); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropVnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_VNODE_IN; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + } } - diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 36b6725737bb3b4cd6a070da92fe01060f80d17e..da5492057458b09aed332168919ce1b38a46e959 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -61,15 +61,14 @@ typedef enum { } EAuthOp; typedef enum { - TRN_STAGE_PREPARE = 1, - TRN_STAGE_EXECUTE = 2, + TRN_STAGE_PREPARE = 0, + TRN_STAGE_EXECUTE = 1, + TRN_STAGE_ROLLBACK = 2, TRN_STAGE_COMMIT = 3, - TRN_STAGE_ROLLBACK = 4, - TRN_STAGE_RETRY = 5, - TRN_STAGE_OVER = 6, + TRN_STAGE_OVER = 4, } ETrnStage; -typedef enum { TRN_POLICY_ROLLBACK = 1, TRN_POLICY_RETRY = 2 } ETrnPolicy; +typedef enum { TRN_POLICY_ROLLBACK = 0, TRN_POLICY_RETRY = 1 } ETrnPolicy; typedef enum { DND_STATUS_OFFLINE = 0, diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index e9913803bd7f2fe88db87c2ea2b44aafb062f554..ba72d9553721b9220aa520b8bb11b46c1b7baeb5 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -70,6 +70,7 @@ typedef struct SMnode { tmr_h timer; char *path; SMnodeCfg cfg; + int64_t checkTime; SSdb *pSdb; SDnode *pDnode; SArray *pSteps; diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 5c15e2f987d89ad00c26201bfcdc85352d2a71b0..a8d37ba655ef52c6f4fdf4b77938cab4bb0a6046 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -22,6 +22,16 @@ extern "C" { #endif +typedef struct { + SEpSet epSet; + int8_t msgType; + int8_t msgSent; + int8_t msgReceived; + int32_t errCode; + int32_t contLen; + void *pCont; +} STransAction; + int32_t mndInitTrans(SMnode *pMnode); void mndCleanupTrans(SMnode *pMnode); @@ -30,12 +40,15 @@ void mndTransDrop(STrans *pTrans); int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); -int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); -int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); +int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction); +int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); + int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code); -char *mndTransStageStr(ETrnStage stage); -char *mndTransPolicyStr(ETrnPolicy policy); +void mndTransHandleActionRsp(SMnodeMsg *pMsg); + +char *mndTransStageStr(ETrnStage stage); +char *mndTransPolicyStr(ETrnPolicy policy); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index e9cdedd332bd2f9d29a3e3d606dc856df14b9eec..8a3a2c798af479ca3a419cf2be2560735b100d81 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -26,8 +26,9 @@ int32_t mndInitVgroup(SMnode *pMnode); void mndCleanupVgroup(SMnode *pMnode); SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); -int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); +int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); +SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup); SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 75b0acbd0a6df8e98b1a3304edb15701284805dc..4c2706f4819a8477a99353a3a05409894aad20c0 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -17,7 +17,8 @@ #include "mndAcct.h" #include "mndShow.h" -#define SDB_ACCT_VER 1 +#define TSDB_ACCT_VER_NUMBER 1 +#define TSDB_ACCT_RESERVE_SIZE 64 static int32_t mndCreateDefaultAcct(SMnode *pMnode); static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct); @@ -70,7 +71,7 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) { } static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, SDB_ACCT_VER, sizeof(SAcctObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, TSDB_ACCT_VER_NUMBER, sizeof(SAcctObj) + TSDB_ACCT_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -85,6 +86,7 @@ static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) { SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.maxStreams) SDB_SET_INT64(pRaw, dataPos, pAcct->cfg.maxStorage) SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.accessState) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_ACCT_RESERVE_SIZE) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; @@ -94,7 +96,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != SDB_ACCT_VER) { + if (sver != TSDB_ACCT_VER_NUMBER) { mError("failed to decode acct since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -116,13 +118,13 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, pRow, dataPos, &pAcct->cfg.maxStreams) SDB_GET_INT64(pRaw, pRow, dataPos, &pAcct->cfg.maxStorage) SDB_GET_INT32(pRaw, pRow, dataPos, &pAcct->cfg.accessState) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_ACCT_RESERVE_SIZE) return pRow; } static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct) { mTrace("acct:%s, perform insert action", pAcct->acct); - memset(&pAcct->info, 0, sizeof(SAcctInfo)); return 0; } @@ -134,12 +136,9 @@ static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) { static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct) { mTrace("acct:%s, perform update action", pOldAcct->acct); - memcpy(pOldAcct->acct, pNewAcct->acct, TSDB_USER_LEN); - pOldAcct->createdTime = pNewAcct->createdTime; pOldAcct->updateTime = pNewAcct->updateTime; - pOldAcct->acctId = pNewAcct->acctId; pOldAcct->status = pNewAcct->status; - pOldAcct->cfg = pNewAcct->cfg; + memcpy(&pOldAcct->cfg, &pNewAcct->cfg, sizeof(SAcctInfo)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 7c53a4ebd0516c13856e9d36d506c8babfc880c8..cf8511c0541cc4642e2e18c104496a4acbb259ee 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -18,7 +18,8 @@ #include "mndShow.h" #include "mndTrans.h" -#define SDB_CLUSTER_VER 1 +#define TSDB_CLUSTER_VER_NUMBE 1 +#define TSDB_CLUSTER_RESERVE_SIZE 64 static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw); @@ -62,7 +63,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { } static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, SDB_CLUSTER_VER, sizeof(SClusterObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, TSDB_CLUSTER_VER_NUMBE, sizeof(SClusterObj) + TSDB_CLUSTER_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -70,6 +71,7 @@ static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { SDB_SET_INT64(pRaw, dataPos, pCluster->createdTime) SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime) SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_CLUSTER_RESERVE_SIZE) return pRaw; } @@ -78,7 +80,7 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != SDB_CLUSTER_VER) { + if (sver != TSDB_CLUSTER_VER_NUMBE) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to decode cluster since %s", terrstr()); return NULL; @@ -93,6 +95,7 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pCluster->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pCluster->updateTime) SDB_GET_BINARY(pRaw, pRow, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_CLUSTER_RESERVE_SIZE) return pRow; } @@ -169,6 +172,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->numOfRows = 1; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e85edf66da85011fe9b1c32e18a3568fb3ecc477..108896c121eeb1d3d93ffd3a1a9c93cc6f413235 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -16,13 +16,12 @@ #define _DEFAULT_SOURCE #include "mndDb.h" #include "mndDnode.h" -#include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" -#define TSDB_DB_VER_NUM 1 +#define TSDB_DB_VER_NUMBER 1 #define TSDB_DB_RESERVE_SIZE 64 static SSdbRaw *mndDbActionEncode(SDbObj *pDb); @@ -66,7 +65,7 @@ int32_t mndInitDb(SMnode *pMnode) { void mndCleanupDb(SMnode *pMnode) {} static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUM, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -106,7 +105,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_DB_VER_NUM) { + if (sver != TSDB_DB_VER_NUMBER) { mError("failed to decode db since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -160,7 +159,7 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { mTrace("db:%s, perform update action", pOldDb->name); - pOldDb->updateTime = pNewDb->createdTime; + pOldDb->updateTime = pNewDb->updateTime; pOldDb->cfgVersion = pNewDb->cfgVersion; pOldDb->vgVersion = pNewDb->vgVersion; memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg)); @@ -168,8 +167,12 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { } SDbObj *mndAcquireDb(SMnode *pMnode, char *db) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_DB, db); + SSdb *pSdb = pMnode->pSdb; + SDbObj *pDb = sdbAcquire(pSdb, SDB_DB, db); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + } + return pDb; } void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { @@ -242,68 +245,74 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; } -static int32_t mndSetRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { +static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { SSdbRaw *pDbRaw = mndDbActionEncode(pDb); - if (pDbRaw == NULL || mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1; - sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING); + if (pDbRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1; + if (sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING) != 0) return -1; - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { + for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) { SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); - if (pVgRaw == NULL || mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; - sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING); + if (pVgRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1; } return 0; } -static int32_t mndSetUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { +static int32_t mndSetCreateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { SSdbRaw *pDbRaw = mndDbActionEncode(pDb); - if (pDbRaw == NULL || mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1; - sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED); + if (pDbRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1; + if (sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED) != 0) return -1; - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { + for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) { SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); - if (pVgRaw == NULL || mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1; - sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED); + if (pVgRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED) != 0) return -1; } return 0; } -static int32_t mndSetCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { +static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { SSdbRaw *pDbRaw = mndDbActionEncode(pDb); - if (pDbRaw == NULL || mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1; - sdbSetRawStatus(pDbRaw, SDB_STATUS_READY); + if (pDbRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1; + if (sdbSetRawStatus(pDbRaw, SDB_STATUS_READY) != 0) return -1; - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { + for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) { SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); - if (pVgRaw == NULL || mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; - sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; } return 0; } -static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { - SVgObj *pVgroup = pVgroups + v; +static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + for (int32_t vg = 0; vg < pDb->cfg.numOfVgroups; ++vg) { + SVgObj *pVgroup = pVgroups + vg; for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) { - return -1; - } + STransAction action = {0}; + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SEpSet epset = mndGetDnodeEpset(pDnode); + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); SCreateVnodeMsg *pMsg = mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup); - if (pMsg == NULL) { - return -1; - } + if (pMsg == NULL) return -1; - if (mndTransAppendRedoAction(pTrans, &epset, TSDB_MSG_TYPE_ALTER_VNODE_IN, sizeof(SCreateVnodeMsg), pMsg) != 0) { + action.pCont = pMsg; + action.contLen = sizeof(SCreateVnodeMsg); + action.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); return -1; } @@ -313,26 +322,26 @@ static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SV return 0; } -static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { - SVgObj *pVgroup = pVgroups + v; +static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + for (int32_t vg = 0; vg < pDb->cfg.numOfVgroups; ++vg) { + SVgObj *pVgroup = pVgroups + vg; for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) { - return -1; - } + STransAction action = {0}; + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SEpSet epset = mndGetDnodeEpset(pDnode); + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); SDropVnodeMsg *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup); - if (pMsg == NULL) { - return -1; - } + if (pMsg == NULL) return -1; - if (mndTransAppendUndoAction(pTrans, &epset, TSDB_MSG_TYPE_DROP_VNODE_IN, sizeof(SDropVnodeMsg), pMsg) != 0) { + action.pCont = pMsg; + action.contLen = sizeof(SDropVnodeMsg); + action.msgType = TSDB_MSG_TYPE_DROP_VNODE_IN; + if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pMsg); return -1; } @@ -344,14 +353,14 @@ static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SV static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, SUserObj *pUser) { SDbObj dbObj = {0}; - tstrncpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN); - tstrncpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); + memcpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN); + memcpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); dbObj.createdTime = taosGetTimestampMs(); dbObj.updateTime = dbObj.createdTime; dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN); - dbObj.hashMethod = 1; dbObj.cfgVersion = 1; dbObj.vgVersion = 1; + dbObj.hashMethod = 1; dbObj.cfg = (SDbCfg){.numOfVgroups = pCreate->numOfVgroups, .cacheBlockSize = pCreate->cacheBlockSize, .totalBlocks = pCreate->totalBlocks, @@ -359,8 +368,8 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat .daysToKeep0 = pCreate->daysToKeep0, .daysToKeep1 = pCreate->daysToKeep1, .daysToKeep2 = pCreate->daysToKeep2, - .minRows = pCreate->minRowsPerFileBlock, - .maxRows = pCreate->maxRowsPerFileBlock, + .minRows = pCreate->minRows, + .maxRows = pCreate->maxRows, .fsyncPeriod = pCreate->fsyncPeriod, .commitTime = pCreate->commitTime, .precision = pCreate->precision, @@ -396,29 +405,30 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat mError("db:%s, failed to create since %s", pCreate->db, terrstr()); goto CREATE_DB_OVER; } + mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); - if (mndSetRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } - if (mndSetUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } - if (mndSetCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } - if (mndSetRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } - if (mndSetUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } @@ -447,8 +457,8 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0); pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); - pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); - pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); + pCreate->minRows = htonl(pCreate->minRows); + pCreate->maxRows = htonl(pCreate->maxRows); pCreate->commitTime = htonl(pCreate->commitTime); pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod); @@ -456,7 +466,7 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { SDbObj *pDb = mndAcquireDb(pMnode, pCreate->db); if (pDb != NULL) { - sdbRelease(pMnode->pSdb, pDb); + mndReleaseDb(pMnode, pDb); if (pCreate->ignoreExist) { mDebug("db:%s, already exist, ignore exist is set", pCreate->db); return 0; @@ -485,57 +495,77 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { } static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) { - bool changed = false; + terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) { pDb->cfg.totalBlocks = pAlter->totalBlocks; - changed = true; + terrno = 0; } if (pAlter->daysToKeep0 >= 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) { pDb->cfg.daysToKeep0 = pAlter->daysToKeep0; - changed = true; + terrno = 0; } if (pAlter->daysToKeep1 >= 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) { pDb->cfg.daysToKeep1 = pAlter->daysToKeep1; - changed = true; + terrno = 0; } if (pAlter->daysToKeep2 >= 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) { pDb->cfg.daysToKeep2 = pAlter->daysToKeep2; - changed = true; + terrno = 0; } if (pAlter->fsyncPeriod >= 0 && pAlter->fsyncPeriod != pDb->cfg.fsyncPeriod) { pDb->cfg.fsyncPeriod = pAlter->fsyncPeriod; - changed = true; + terrno = 0; } if (pAlter->walLevel >= 0 && pAlter->walLevel != pDb->cfg.walLevel) { pDb->cfg.walLevel = pAlter->walLevel; - changed = true; + terrno = 0; } if (pAlter->quorum >= 0 && pAlter->quorum != pDb->cfg.quorum) { pDb->cfg.quorum = pAlter->quorum; - changed = true; + terrno = 0; } if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) { pDb->cfg.cacheLastRow = pAlter->cacheLastRow; - changed = true; + terrno = 0; } - if (!changed) { - terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; - return -1; - } + return terrno; +} + +static int32_t mndSetUpdateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { + SSdbRaw *pRedoRaw = mndDbActionEncode(pNewDb); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; return 0; } +static int32_t mndSetUpdateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { + SSdbRaw *pUndoRaw = mndDbActionEncode(pOldDb); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; } + +static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; } + +static int32_t mndSetUpdateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; } + static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbObj *pNewDb) { + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { mError("db:%s, failed to update since %s", pOldDb->name, terrstr()); @@ -544,30 +574,41 @@ static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbO mDebug("trans:%d, used to update db:%s", pTrans->id, pOldDb->name); - SSdbRaw *pRedoRaw = mndDbActionEncode(pNewDb); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetUpdateDbRedoLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - SSdbRaw *pUndoRaw = mndDbActionEncode(pOldDb); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetUpdateDbUndoLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; + } + + if (mndSetUpdateDbCommitLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; + } + + if (mndSetUpdateDbRedoActions(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; + } + + if (mndSetUpdateDbUndoActions(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + goto UPDATE_DB_OVER; } + code = 0; + +UPDATE_DB_OVER: mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { @@ -610,46 +651,82 @@ static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } +static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdbRaw *pRedoRaw = mndDbActionEncode(pDb); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdbRaw *pUndoRaw = mndDbActionEncode(pDb); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdbRaw *pCommitRaw = mndDbActionEncode(pDb); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return 0; } + +static int32_t mndSetDropDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return 0; } + static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pDb) { + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { mError("db:%s, failed to drop since %s", pDb->name, terrstr()); return -1; } + mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); - SSdbRaw *pRedoRaw = mndDbActionEncode(pDb); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); - SSdbRaw *pUndoRaw = mndDbActionEncode(pDb); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropDbUndoLogs(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); - SSdbRaw *pCommitRaw = mndDbActionEncode(pDb); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; + } + + if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; + } + + if (mndSetDropDbUndoActions(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + goto DROP_DB_OVER; } + code = 0; + +DROP_DB_OVER: mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { @@ -751,29 +828,27 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) { static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SSyncDbMsg *pSync = pMsg->rpcMsg.pCont; - - SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); if (pDb == NULL) { mError("db:%s, failed to process sync db msg since %s", pMsg->db, terrstr()); return -1; - } else { - mndReleaseDb(pMnode, pDb); - return 0; } + + mndReleaseDb(pMnode, pDb); + return 0; } static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SCompactDbMsg *pCompact = pMsg->rpcMsg.pCont; - - SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); if (pDb == NULL) { mError("db:%s, failed to process compact db msg since %s", pMsg->db, terrstr()); return -1; - } else { - mndReleaseDb(pMnode, pDb); - return 0; } + + mndReleaseDb(pMnode, pDb); + return 0; } static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 493f20bc9af3319521fd602e2ed8e6648305ca22..9751771773c53fa55ec965a1fed8be02013f7588 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -18,10 +18,10 @@ #include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" -#include "ttime.h" #include "tep.h" +#include "ttime.h" -#define TSDB_DNODE_VER 1 +#define TSDB_DNODE_VER_NUMBER 1 #define TSDB_DNODE_RESERVE_SIZE 64 #define TSDB_CONFIG_OPTION_LEN 16 #define TSDB_CONIIG_VALUE_LEN 48 @@ -101,14 +101,14 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { SSdbRaw *pRaw = mndDnodeActionEncode(&dnodeObj); if (pRaw == NULL) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); + if (sdbSetRawStatus(pRaw, SDB_STATUS_READY) != 0) return -1; mDebug("dnode:%d, will be created while deploy sdb", dnodeObj.id); return sdbWrite(pMnode->pSdb, pRaw); } static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER, sizeof(SDnodeObj) + TSDB_DNODE_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER_NUMBER, sizeof(SDnodeObj) + TSDB_DNODE_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -127,7 +127,7 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_DNODE_VER) { + if (sver != TSDB_DNODE_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to decode dnode since %s", terrstr()); return NULL; @@ -150,21 +150,8 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) { mTrace("dnode:%d, perform insert action", pDnode->id); - - pDnode->rebootTime = 0; - pDnode->lastAccessTime = 0; - pDnode->accessTimes = 0; - pDnode->numOfMnodes = 0; - pDnode->numOfVnodes = 0; - pDnode->numOfQnodes = 0; - pDnode->numOfSupportMnodes = 0; - pDnode->numOfSupportVnodes = 0; - pDnode->numOfSupportQnodes = 0; - pDnode->numOfCores = 0; - pDnode->status = DND_STATUS_OFFLINE; pDnode->offlineReason = DND_REASON_STATUS_NOT_RECEIVED; snprintf(pDnode->ep, TSDB_EP_LEN, "%s:%u", pDnode->fqdn, pDnode->port); - return 0; } @@ -225,7 +212,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode) { int64_t ms = taosGetTimestampMs(); int64_t interval = ABS(pDnode->lastAccessTime - ms); - if (interval > 3000 * pMnode->cfg.statusInterval) { + if (interval > 3500 * pMnode->cfg.statusInterval) { return false; } return true; @@ -267,12 +254,9 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { return DND_REASON_STATUS_INTERVAL_NOT_MATCH; } - int64_t checkTime = 0; - char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (checkTime != pCfg->checkTime)) { + if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (pMnode->checkTime != pCfg->checkTime)) { mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone, - pCfg->checkTime, checkTime); + pCfg->checkTime, pMnode->checkTime); return DND_REASON_TIME_ZONE_NOT_MATCH; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 16733cbf94bd93fc6193d6ca7bf0cd6c39e88f94..869b6e538b711b8782144041c450a410210fb67a 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -19,7 +19,8 @@ #include "mndShow.h" #include "mndTrans.h" -#define SDB_MNODE_VER 1 +#define TSDB_MNODE_VER_NUMBER 1 +#define TSDB_MNODE_RESERVE_SIZE 64 static int32_t mndCreateDefaultMnode(SMnode *pMnode); static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj); @@ -60,8 +61,12 @@ int32_t mndInitMnode(SMnode *pMnode) { void mndCleanupMnode(SMnode *pMnode) {} static SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_MNODE, &mnodeId); + SSdb *pSdb = pMnode->pSdb; + SMnodeObj *pObj = sdbAcquire(pSdb, SDB_MNODE, &mnodeId); + if (pObj == NULL) { + terrno = TSDB_CODE_MND_MNODE_NOT_EXIST; + } + return pObj; } static void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pMnodeObj) { @@ -97,13 +102,14 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) { } static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, SDB_MNODE_VER, sizeof(SMnodeObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, TSDB_MNODE_VER_NUMBER, sizeof(SMnodeObj) + TSDB_MNODE_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; SDB_SET_INT32(pRaw, dataPos, pMnodeObj->id); SDB_SET_INT64(pRaw, dataPos, pMnodeObj->createdTime) SDB_SET_INT64(pRaw, dataPos, pMnodeObj->updateTime) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_MNODE_RESERVE_SIZE) return pRaw; } @@ -112,7 +118,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != SDB_MNODE_VER) { + if (sver != TSDB_MNODE_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to decode mnode since %s", terrstr()); return NULL; @@ -126,6 +132,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, pRow, dataPos, &pMnodeObj->id) SDB_GET_INT64(pRaw, pRow, dataPos, &pMnodeObj->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pMnodeObj->updateTime) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_MNODE_RESERVE_SIZE) return pRow; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 822036b599b5c2cfcb086403b9122416b729b76e..a51312a2a924e5d51cd5c8279426faf6e6016b59 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -21,9 +21,10 @@ #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" +#include "mndVgroup.h" #include "tname.h" -#define TSDB_STB_VER_NUM 1 +#define TSDB_STB_VER_NUMBER 1 #define TSDB_STB_RESERVE_SIZE 64 static SSdbRaw *mndStbActionEncode(SStbObj *pStb); @@ -70,7 +71,7 @@ void mndCleanupStb(SMnode *pMnode) {} static SSdbRaw *mndStbActionEncode(SStbObj *pStb) { int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + TSDB_STB_RESERVE_SIZE; - SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUM, size); + SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -103,7 +104,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_STB_VER_NUM) { + if (sver != TSDB_STB_VER_NUMBER) { mError("failed to decode stable since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -176,8 +177,12 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb } SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_STB, stbName); + SSdb *pSdb = pMnode->pSdb; + SStbObj *pStb = sdbAcquire(pSdb, SDB_STB, stbName); + if (pStb == NULL) { + terrno = TSDB_CODE_MND_STB_NOT_EXIST; + } + return pStb; } void mndReleaseStb(SMnode *pMnode, SStbObj *pStb) { @@ -195,7 +200,54 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, char *stbName) { return mndAcquireDb(pMnode, db); } -static int32_t mndCheckStbMsg(SCreateStbMsg *pCreate) { +static SCreateStbInternalMsg *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { + int32_t totalCols = pStb->numOfTags + pStb->numOfColumns; + int32_t contLen = totalCols * sizeof(SSchema) + sizeof(SCreateStbInternalMsg); + + SCreateStbInternalMsg *pCreate = calloc(1, contLen); + if (pCreate == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pCreate->head.contLen = htonl(contLen); + pCreate->head.vgId = htonl(pVgroup->vgId); + memcpy(pCreate->name, pStb->name, TSDB_TABLE_FNAME_LEN); + pCreate->suid = htobe64(pStb->uid); + pCreate->sverson = htonl(pStb->version); + pCreate->ttl = 0; + pCreate->keep = 0; + pCreate->numOfTags = htonl(pStb->numOfTags); + pCreate->numOfColumns = htonl(pStb->numOfColumns); + + memcpy(pCreate->pSchema, pStb->pSchema, totalCols * sizeof(SSchema)); + for (int32_t t = 0; t < totalCols; ++t) { + SSchema *pSchema = &pCreate->pSchema[t]; + pSchema->bytes = htonl(pSchema->bytes); + pSchema->colId = htonl(pSchema->colId); + } + + return pCreate; +} + +static SDropStbInternalMsg *mndBuildDropStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { + int32_t contLen = sizeof(SDropStbInternalMsg); + + SDropStbInternalMsg *pDrop = calloc(1, contLen); + if (pDrop == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pDrop->head.contLen = htonl(contLen); + pDrop->head.vgId = htonl(pVgroup->vgId); + memcpy(pDrop->name, pStb->name, TSDB_TABLE_FNAME_LEN); + pDrop->suid = htobe64(pStb->uid); + + return pDrop; +} + +static int32_t mndCheckCreateStbMsg(SCreateStbMsg *pCreate) { pCreate->numOfColumns = htonl(pCreate->numOfColumns); pCreate->numOfTags = htonl(pCreate->numOfTags); int32_t totalCols = pCreate->numOfColumns + pCreate->numOfTags; @@ -244,6 +296,103 @@ static int32_t mndCheckStbMsg(SCreateStbMsg *pCreate) { return 0; } +static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateStbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdbRaw *pUndoRaw = mndStbActionEncode(pStb); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) continue; + + SCreateStbInternalMsg *pMsg = mndBuildCreateStbMsg(pMnode, pVgroup, pStb); + if (pMsg == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pMsg; + action.contLen = htonl(pMsg->head.contLen); + action.msgType = TSDB_MSG_TYPE_CREATE_STB_IN; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) continue; + + SDropStbInternalMsg *pMsg = mndBuildDropStbMsg(pMnode, pVgroup, pStb); + if (pMsg == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pMsg; + action.contLen = sizeof(SDropStbInternalMsg); + action.msgType = TSDB_MSG_TYPE_DROP_STB_IN; + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + free(pMsg); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCreate, SDbObj *pDb) { SStbObj stbObj = {0}; tstrncpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); @@ -265,6 +414,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre } memcpy(stbObj.pSchema, pCreate->pSchema, totalSize); + int32_t code = 0; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); @@ -272,29 +422,30 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre } mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); - SSdbRaw *pRedoRaw = mndStbActionEncode(&stbObj); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto CREATE_STB_OVER; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); - SSdbRaw *pUndoRaw = mndStbActionEncode(&stbObj); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, &stbObj) != 0) { + mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); + goto CREATE_STB_OVER; } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); - SSdbRaw *pCommitRaw = mndStbActionEncode(&stbObj); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto CREATE_STB_OVER; + } + + if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_STB_OVER; + } + + if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, &stbObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_STB_OVER; } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -302,8 +453,11 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre return -1; } + code = 0; + +CREATE_STB_OVER: mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) { @@ -312,7 +466,7 @@ static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) { mDebug("stb:%s, start to create", pCreate->name); - if (mndCheckStbMsg(pCreate) != 0) { + if (mndCheckCreateStbMsg(pCreate) != 0) { mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } @@ -349,7 +503,10 @@ static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg) { + mndTransHandleActionRsp(pMsg); + return 0; +} static int32_t mndCheckAlterStbMsg(SAlterStbMsg *pAlter) { SSchema *pSchema = &pAlter->schema; @@ -410,9 +567,44 @@ static int32_t mndProcessAlterStbMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessAlterStbInRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessAlterStbInRsp(SMnodeMsg *pMsg) { + mndTransHandleActionRsp(pMsg); + return 0; +} + +static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { + SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropStbUndoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { + SSdbRaw *pUndoRaw = mndStbActionEncode(pStb); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { + SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; } + +static int32_t mndSetDropStbUndoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; } static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pStb) { + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { mError("stb:%s, failed to drop since %s", pStb->name, terrstr()); @@ -420,36 +612,39 @@ static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pStb) { } mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); - SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_STB_OVER; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); - SSdbRaw *pUndoRaw = mndStbActionEncode(pStb); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropStbUndoLogs(pMnode, pTrans, pStb) != 0) { + mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); + goto DROP_STB_OVER; } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); - SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_STB_OVER; + } + + if (mndSetDropStbRedoActions(pMnode, pTrans, pStb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_STB_OVER; + } + + if (mndSetDropStbUndoActions(pMnode, pTrans, pStb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_STB_OVER; } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + goto DROP_STB_OVER; } + code = 0; + +DROP_STB_OVER: mndTransDrop(pTrans); return 0; } @@ -484,7 +679,10 @@ static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) { + mndTransHandleActionRsp(pMsg); + return 0; +} static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 5caec6c78dc59bd061e96770c7817510c43a0f6f..54cd6ab501a0a9294b0f35e575c365d44e14b7bd 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -17,17 +17,10 @@ #include "mndTrans.h" #include "mndSync.h" -#define TSDB_TRANS_VER 1 +#define TSDB_TRANS_VER_NUMBER 1 #define TSDB_TRN_ARRAY_SIZE 8 #define TSDB_TRN_RESERVE_SIZE 64 -typedef struct { - SEpSet epSet; - int8_t msgType; - int32_t contLen; - void *pCont; -} STransAction; - static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); @@ -37,11 +30,11 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle); static void mndTransSendRpcRsp(STrans *pTrans, int32_t code); static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); -static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); +static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction); static void mndTransDropLogs(SArray *pArray); static void mndTransDropActions(SArray *pArray); static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray); -static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray); +static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray); static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans); @@ -100,7 +93,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { rawDataLen += (sizeof(STransAction) + pAction->contLen); } - SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER, rawDataLen); + SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER_NUMBER, rawDataLen); if (pRaw == NULL) { mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); return NULL; @@ -167,7 +160,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { return NULL; } - if (sver != TSDB_TRANS_VER) { + if (sver != TSDB_TRANS_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to get check soft ver from raw:%p since %s", pRaw, terrstr()); return NULL; @@ -324,8 +317,12 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOldTrans, STrans *pNewT } STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_TRANS, &transId); + SSdb *pSdb = pMnode->pSdb; + STrans *pTrans = sdbAcquire(pSdb, SDB_TRANS, &transId); + if (pTrans == NULL) { + terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; + } + return pTrans; } void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { @@ -343,10 +340,8 @@ char *mndTransStageStr(ETrnStage stage) { return "commit"; case TRN_STAGE_ROLLBACK: return "rollback"; - case TRN_STAGE_RETRY: - return "retry"; case TRN_STAGE_OVER: - return "stop"; + return "over"; default: return "undefined"; } @@ -388,7 +383,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { return NULL; } - mDebug("trans:%d, data:%p is created", pTrans->id, pTrans); + mDebug("trans:%d, is created", pTrans->id); return pTrans; } @@ -417,7 +412,7 @@ void mndTransDrop(STrans *pTrans) { mndTransDropActions(pTrans->redoActions); mndTransDropActions(pTrans->undoActions); - mDebug("trans:%d, data:%p is dropped", pTrans->id, pTrans); + // mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); tfree(pTrans); } @@ -459,10 +454,8 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) { return code; } -static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { - STransAction action = {.epSet = *pEpSet, .msgType = msgType, .contLen = contLen, .pCont = pCont}; - - void *ptr = taosArrayPush(pArray, &action); +static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction) { + void *ptr = taosArrayPush(pArray, pAction); if (ptr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -471,15 +464,15 @@ static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgTy return 0; } -int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { - int32_t code = mndTransAppendAction(pTrans->redoActions, pEpSet, msgType, contLen, pCont); - mTrace("trans:%d, msg:%s len:%d append to redo actions", pTrans->id, taosMsg[msgType], contLen); +int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction) { + int32_t code = mndTransAppendAction(pTrans->redoActions, pAction); + mTrace("trans:%d, msg:%s append to redo actions, code:0x%x", pTrans->id, taosMsg[pAction->msgType], code); return code; } -int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { - int32_t code = mndTransAppendAction(pTrans->undoActions, pEpSet, msgType, contLen, pCont); - mTrace("trans:%d, msg:%s len:%d append to undo actions", pTrans->id, taosMsg[msgType], contLen); +int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction) { + int32_t code = mndTransAppendAction(pTrans->undoActions, pAction); + mTrace("trans:%d, msg:%s append to undo actions, code:0x%x", pTrans->id, taosMsg[pAction->msgType], code); return code; } @@ -493,7 +486,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { } sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mTrace("trans:%d, start sync", pTrans->id); + mTrace("trans:%d, sync to other nodes", pTrans->id); int32_t code = mndSyncPropose(pMnode, pRaw); if (code != 0) { mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); @@ -515,7 +508,6 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { return -1; } - mDebug("trans:%d, prepare finished", pNewTrans->id); pNewTrans->rpcHandle = pTrans->rpcHandle; mndTransExecute(pMnode, pNewTrans); mndReleaseTrans(pMnode, pNewTrans); @@ -533,7 +525,7 @@ int32_t mndTransCommit(SMnode *pMnode, STrans *pTrans) { sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); if (taosArrayGetSize(pTrans->commitLogs) != 0) { - mTrace("trans:%d, start sync", pTrans->id); + mTrace("trans:%d, sync to other nodes", pTrans->id); int32_t code = mndSyncPropose(pMnode, pRaw); if (code != 0) { mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); @@ -563,7 +555,7 @@ int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) { } sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); - mTrace("trans:%d, start sync", pTrans->id); + mTrace("trans:%d, sync to other nodes", pTrans->id); int32_t code = mndSyncPropose(pMnode, pRaw); if (code != 0) { mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); @@ -596,6 +588,50 @@ void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code) // todo } +void mndTransHandleActionRsp(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + int64_t sig = (int64_t)(pMsg->rpcMsg.ahandle); + int32_t transId = (int32_t)(sig >> 32); + int32_t action = (int32_t)((sig << 32) >> 32); + + STrans *pTrans = mndAcquireTrans(pMnode, transId); + if (pTrans == NULL) { + mError("trans:%d, failed to get transId from vnode rsp since %s", transId, terrstr()); + goto HANDLE_ACTION_RSP_OVER; + } + + SArray *pArray = NULL; + if (pTrans->stage == TRN_STAGE_EXECUTE) { + pArray = pTrans->redoActions; + } else if (pTrans->stage == TRN_STAGE_ROLLBACK) { + pArray = pTrans->undoActions; + } else { + } + + if (pArray == NULL) { + mError("trans:%d, invalid trans stage:%s", transId, mndTransStageStr(pTrans->stage)); + goto HANDLE_ACTION_RSP_OVER; + } + + int32_t actionNum = taosArrayGetSize(pTrans->redoActions); + if (action < 0 || action > actionNum) { + mError("trans:%d, invalid action:%d", transId, action); + goto HANDLE_ACTION_RSP_OVER; + } + + STransAction *pAction = taosArrayGet(pArray, action); + if (pAction != NULL) { + pAction->msgReceived = 1; + pAction->errCode = pMsg->code; + } + + mDebug("trans:%d, action:%d response is received, code:0x%x", transId, action, pMsg->code); + mndTransExecute(pMnode, pTrans); + +HANDLE_ACTION_RSP_OVER: + mndReleaseTrans(pMnode, pTrans); +} + static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) { SSdb *pSdb = pMnode->pSdb; int32_t arraySize = taosArrayGetSize(pArray); @@ -618,7 +654,7 @@ static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) { if (code != 0) { mError("trans:%d, failed to execute redo logs since %s", pTrans->id, terrstr()) } else { - mTrace("trans:%d, execute redo logs finished", pTrans->id) + mDebug("trans:%d, execute redo logs finished", pTrans->id) } } @@ -632,7 +668,7 @@ static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) { if (code != 0) { mError("trans:%d, failed to execute undo logs since %s", pTrans->id, terrstr()) } else { - mTrace("trans:%d, execute undo logs finished", pTrans->id) + mDebug("trans:%d, execute undo logs finished", pTrans->id) } } @@ -646,47 +682,70 @@ static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) { if (code != 0) { mError("trans:%d, failed to execute commit logs since %s", pTrans->id, terrstr()) } else { - mTrace("trans:%d, execute commit logs finished", pTrans->id) + mDebug("trans:%d, execute commit logs finished", pTrans->id) } } return code; } -static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray) { -#if 0 - int32_t arraySize = taosArrayGetSize(pArray); - for (int32_t i = 0; i < arraySize; ++i) { - STransAction *pAction = taosArrayGet(pArray, i); +static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { + int32_t numOfActions = taosArrayGetSize(pArray); + if (numOfActions == 0) return 0; + + for (int32_t action = 0; action < numOfActions; ++action) { + STransAction *pAction = taosArrayGet(pArray, action); + if (pAction == NULL) continue; + if (pAction->msgSent) continue; - SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen}; + int64_t signature = pTrans->id; + signature = (signature << 32); + signature += action; + + SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .ahandle = (void *)signature}; rpcMsg.pCont = rpcMallocCont(pAction->contLen); if (rpcMsg.pCont == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); + + pAction->msgSent = 1; + pAction->msgReceived = 0; + pAction->errCode = 0; + + mDebug("trans:%d, action:%d is sent", pTrans->id, action); mndSendMsgToDnode(pMnode, &pAction->epSet, &rpcMsg); } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; -#else - return 0; -#endif + int32_t numOfReceivedMsgs = 0; + int32_t errorCode = 0; + for (int32_t action = 0; action < numOfActions; ++action) { + STransAction *pAction = taosArrayGet(pArray, action); + if (pAction == NULL) continue; + if (pAction->msgSent && pAction->msgReceived) { + numOfReceivedMsgs++; + if (pAction->errCode != 0) { + errorCode = pAction->errCode; + } + } + } + + if (numOfReceivedMsgs == numOfActions) { + mDebug("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errorCode); + terrno = errorCode; + return errorCode; + } else { + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } } static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { - if (taosArrayGetSize(pTrans->redoActions) <= 0) return 0; - - mTrace("trans:%d, start to execute redo actions", pTrans->id); - return mndTransExecuteActions(pMnode, pTrans->redoActions); + return mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); } static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { - if (taosArrayGetSize(pTrans->undoActions) <= 0) return 0; - - mTrace("trans:%d, start to execute undo actions", pTrans->id); - return mndTransExecuteActions(pMnode, pTrans->undoActions); + return mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); } static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { @@ -694,7 +753,7 @@ static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { if (code == 0) { pTrans->stage = TRN_STAGE_EXECUTE; - mTrace("trans:%d, stage from prepare to execute", pTrans->id); + mDebug("trans:%d, stage from prepare to execute", pTrans->id); } else { pTrans->stage = TRN_STAGE_ROLLBACK; mError("trans:%d, stage from prepare to rollback since %s", pTrans->id, terrstr()); @@ -708,17 +767,17 @@ static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans) { if (code == 0) { pTrans->stage = TRN_STAGE_COMMIT; - mTrace("trans:%d, stage from execute to commit", pTrans->id); + mDebug("trans:%d, stage from execute to commit", pTrans->id); } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { - mTrace("trans:%d, stage keep on execute since %s", pTrans->id, terrstr(code)); + mDebug("trans:%d, stage keep on execute since %s", pTrans->id, tstrerror(code)); return code; } else { if (pTrans->policy == TRN_POLICY_ROLLBACK) { pTrans->stage = TRN_STAGE_ROLLBACK; mError("trans:%d, stage from execute to rollback since %s", pTrans->id, terrstr()); } else { - pTrans->stage = TRN_STAGE_RETRY; - mError("trans:%d, stage from execute to retry since %s", pTrans->id, terrstr()); + pTrans->stage = TRN_STAGE_EXECUTE; + mError("trans:%d, stage keep on execute since %s", pTrans->id, terrstr()); } } @@ -726,29 +785,16 @@ static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans) { } static int32_t mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteCommitLogs(pMnode, pTrans); - - if (code == 0) { - pTrans->stage = TRN_STAGE_OVER; - mTrace("trans:%d, commit stage finished", pTrans->id); - } else { - if (pTrans->policy == TRN_POLICY_ROLLBACK) { - pTrans->stage = TRN_STAGE_ROLLBACK; - mError("trans:%d, stage from commit to rollback since %s", pTrans->id, terrstr()); - } else { - pTrans->stage = TRN_STAGE_RETRY; - mError("trans:%d, stage from commit to retry since %s", pTrans->id, terrstr()); - } - } - - return code; + mndTransExecuteCommitLogs(pMnode, pTrans); + pTrans->stage = TRN_STAGE_OVER; + return 0; } static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteUndoActions(pMnode, pTrans); if (code == 0) { - mTrace("trans:%d, rollbacked", pTrans->id); + mDebug("trans:%d, rollbacked", pTrans->id); } else { pTrans->stage = TRN_STAGE_ROLLBACK; mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr()); @@ -757,20 +803,6 @@ static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { return code; } -static int32_t mndTransPerformRetryStage(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteRedoActions(pMnode, pTrans); - - if (code == 0) { - pTrans->stage = TRN_STAGE_COMMIT; - mTrace("trans:%d, stage from retry to commit", pTrans->id); - } else { - pTrans->stage = TRN_STAGE_RETRY; - mError("trans:%d, stage keep on retry since %s", pTrans->id, terrstr()); - } - - return code; -} - static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { int32_t code = 0; @@ -785,7 +817,7 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { case TRN_STAGE_COMMIT: code = mndTransCommit(pMnode, pTrans); if (code == 0) { - code = mndTransPerformCommitStage(pMnode, pTrans); + mndTransPerformCommitStage(pMnode, pTrans); } break; case TRN_STAGE_ROLLBACK: @@ -794,9 +826,6 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { code = mndTransRollback(pMnode, pTrans); } break; - case TRN_STAGE_RETRY: - code = mndTransPerformRetryStage(pMnode, pTrans); - break; default: mndTransSendRpcRsp(pTrans, 0); return; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 58d687a14865ec7fe104457d287115e4bc0737f0..c96e6a80db3ec6b19f1ea609632a8022c65dad79 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -20,7 +20,7 @@ #include "mndTrans.h" #include "tkey.h" -#define TSDB_USER_VER 1 +#define TSDB_USER_VER_NUMBER 1 #define TSDB_USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); @@ -94,7 +94,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { } static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER, sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER_NUMBER, sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -114,7 +114,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_USER_VER) { + if (sver != TSDB_USER_VER_NUMBER) { mError("failed to decode user since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -175,8 +175,12 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNe } SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_USER, userName); + SSdb *pSdb = pMnode->pSdb; + SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + } + return pUser; } void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index e99fea200b944639aca3fb5bcb33623033c7915d..c65436efcb844d3b9387fdc9467673205d03d4dc 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -21,7 +21,7 @@ #include "mndShow.h" #include "mndTrans.h" -#define TSDB_VGROUP_VER_NUM 1 +#define TSDB_VGROUP_VER_NUMBER 1 #define TSDB_VGROUP_RESERVE_SIZE 64 static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); @@ -70,7 +70,7 @@ int32_t mndInitVgroup(SMnode *pMnode) { void mndCleanupVgroup(SMnode *pMnode) {} SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUMBER, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -98,7 +98,7 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_VGROUP_VER_NUM) { + if (sver != TSDB_VGROUP_VER_NUMBER) { mError("failed to decode vgroup since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -142,14 +142,20 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNe mTrace("vgId:%d, perform update action", pOldVgroup->vgId); pOldVgroup->updateTime = pNewVgroup->updateTime; pOldVgroup->version = pNewVgroup->version; + pOldVgroup->hashBegin = pNewVgroup->hashBegin; + pOldVgroup->hashEnd = pNewVgroup->hashEnd; pOldVgroup->replica = pNewVgroup->replica; memcpy(pOldVgroup->vnodeGid, pNewVgroup->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); return 0; } SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_VGROUP, &vgId); + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = sdbAcquire(pSdb, SDB_VGROUP, &vgId); + if (pVgroup == NULL) { + terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST; + } + return pVgroup; } void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { @@ -158,16 +164,17 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { } SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { - SCreateVnodeMsg *pCreate = malloc(sizeof(SCreateVnodeMsg)); + SCreateVnodeMsg *pCreate = calloc(1, sizeof(SCreateVnodeMsg)); if (pCreate == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pCreate->dnodeId = htonl(pDnode->id); pCreate->vgId = htonl(pVgroup->vgId); + pCreate->dnodeId = htonl(pDnode->id); memcpy(pCreate->db, pDb->name, TSDB_FULL_DB_NAME_LEN); pCreate->dbUid = htobe64(pDb->uid); + pCreate->vgVersion = htonl(pVgroup->version); pCreate->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); pCreate->totalBlocks = htonl(pDb->cfg.totalBlocks); pCreate->daysPerFile = htonl(pDb->cfg.daysPerFile); @@ -193,7 +200,6 @@ SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbOb SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pVgidDnode == NULL) { free(pCreate); - terrno = TSDB_CODE_MND_APP_ERROR; return NULL; } @@ -217,7 +223,7 @@ SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbOb } SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { - SDropVnodeMsg *pDrop = malloc(sizeof(SDropVnodeMsg)); + SDropVnodeMsg *pDrop = calloc(1, sizeof(SDropVnodeMsg)); if (pDrop == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -269,7 +275,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { return -1; } - int32_t alloceVgroups = 0; + int32_t allocedVgroups = 0; int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); uint32_t hashMin = 0; uint32_t hashMax = UINT32_MAX; @@ -281,7 +287,6 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { pVgroup->createdTime = taosGetTimestampMs(); pVgroup->updateTime = pVgroups->createdTime; pVgroup->version = 1; - pVgroup->dbUid = pDb->uid; pVgroup->hashBegin = hashMin + hashInterval * v; if (v == pDb->cfg.numOfVgroups - 1) { pVgroup->hashEnd = hashMax; @@ -290,6 +295,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { } memcpy(pVgroup->dbName, pDb->name, TSDB_FULL_DB_NAME_LEN); + pVgroup->dbUid = pDb->uid; pVgroup->replica = pDb->cfg.replications; if (mndGetAvailableDnode(pMnode, pVgroup) != 0) { @@ -298,22 +304,46 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { return -1; } - alloceVgroups++; + allocedVgroups++; } *ppVgroups = pVgroups; return 0; } -static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { return 0; } +SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup) { + SEpSet epset = {0}; + + for (int32_t v = 0; v < pVgroup->replica; ++v) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) continue; + + if (pVgid->role == TAOS_SYNC_STATE_LEADER) { + epset.inUse = epset.numOfEps; + } + + epset.port[epset.numOfEps] = pDnode->port; + memcpy(&epset.fqdn[epset.numOfEps], pDnode->fqdn, TSDB_FQDN_LEN); + epset.numOfEps++; + mndReleaseDnode(pMnode, pDnode); + } + + return epset; +} + +static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { + mndTransHandleActionRsp(pMsg); + return 0; +} + static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) { - SSdb *pSdb = pMnode->pSdb; - + SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -329,7 +359,7 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; - if (strcmp(pVgroup->dbName, dbName) == 0) { + if (pVgroup->dbUid == pDb->uid) { replica = MAX(replica, pVgroup->replica); numOfVgroups++; } @@ -441,11 +471,25 @@ static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter) { } static int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { - if (dnodeId == 0) { - return 0; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfVnodes = 0; + void *pIter = NULL; + + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + for (int32_t v = 0; v < pVgroup->replica; ++v) { + if (pVgroup->vnodeGid[v].dnodeId == dnodeId) { + numOfVnodes++; + } + } + + sdbRelease(pSdb, pVgroup); } - return 0; + return numOfVnodes; } static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 24155a4e5329f73fc950e25ef23b942b55a11bf0..fb0b95dc4a696f9a7b2751f3735d73d059838c60 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -225,7 +225,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { } return 0; -} +} SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { mDebug("start to open mnode in %s", path); @@ -237,6 +237,9 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { return NULL; } + char timestr[24] = "1970-01-01 00:00:00.00"; + (void)taosParseTime(timestr, &pMnode->checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + pMnode->pSteps = taosArrayInit(24, sizeof(SMnodeStep)); if (pMnode->pSteps == NULL) { free(pMnode); diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index f63f7236fbb56b0edfc49b61b6a2b00736454a65..1c8b82fd92cd3ac8ef536a744533e077c4a79588 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -240,6 +240,8 @@ void sdbRelease(SSdb *pSdb, void *pObj) { } void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { + *ppObj = NULL; + SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return NULL; diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index e37559808e2b5242354be6187c07fc49a732a940..5a0020199f0652376de73ec5c243301b4ac2371f 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -27,12 +27,12 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { pRaw->sver = sver; pRaw->dataLen = dataLen; - mTrace("raw:%p, is created, len:%d", pRaw, dataLen); + // mTrace("raw:%p, is created, len:%d", pRaw, dataLen); return pRaw; } void sdbFreeRaw(SSdbRaw *pRaw) { - mTrace("raw:%p, is freed", pRaw); + // mTrace("raw:%p, is freed", pRaw); free(pRaw); } diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index 3623516624e8ea31c3eedd0bcdfa52cf20d16ef4..6972605afd9c1dd93b7fd0b3198c3837b5351f88 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -19,5 +19,5 @@ target_link_libraries( # test if(${BUILD_TEST}) - #add_subdirectory(test) + add_subdirectory(test) endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index 56e07aca10704f7d8cb79a92c2ad80e4b25fee02..605557d4eace479f8cad786c658b2092b3e63d23 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -71,6 +71,7 @@ struct SVnode { SWal* pWal; SVnodeSync* pSync; SVnodeFS* pFs; + tsem_t canCommit; }; int vnodeScheduleTask(SVnodeTask* task); diff --git a/source/dnode/vnode/impl/src/vnodeCommit.c b/source/dnode/vnode/impl/src/vnodeCommit.c index 7213e31cb46d8bd75f5227e4b88bbd088dd7f340..f5bf60a7e3d56b66a8bf3804fbc2e7515671a4f8 100644 --- a/source/dnode/vnode/impl/src/vnodeCommit.c +++ b/source/dnode/vnode/impl/src/vnodeCommit.c @@ -38,9 +38,10 @@ int vnodeCommit(void *arg) { metaCommit(pVnode->pMeta); tqCommit(pVnode->pTq); - tsdbCommit(pVnode->pTq); + tsdbCommit(pVnode->pTsdb); vnodeBufPoolRecycle(pVnode); + tsem_post(&(pVnode->canCommit)); // TODO return 0; } diff --git a/source/dnode/vnode/impl/src/vnodeInt.c b/source/dnode/vnode/impl/src/vnodeInt.c index 8a6fc8bf5eb7da2545441adf932d5b4740f0ccaa..5deaffe6d220134ac6f8c5f0cfd4607be2ef304a 100644 --- a/source/dnode/vnode/impl/src/vnodeInt.c +++ b/source/dnode/vnode/impl/src/vnodeInt.c @@ -15,53 +15,31 @@ #define _DEFAULT_SOURCE #include "vnodeInt.h" -#include "tqueue.h" int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg) { return 0; } -SVnode *vnodeCreate(int32_t vgId, const char *path, const SVnodeCfg *pCfg) { return NULL; } -void vnodeDrop(SVnode *pVnode) {} + int32_t vnodeCompact(SVnode *pVnode) { return 0; } + int32_t vnodeSync(SVnode *pVnode) { return 0; } int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { return 0; } -SVnodeMsg *vnodeInitMsg(int32_t msgNum) { - SVnodeMsg *pMsg = taosAllocateQitem(msgNum * sizeof(SRpcMsg *) + sizeof(SVnodeMsg)); - if (pMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } else { - pMsg->allocNum = msgNum; - return pMsg; - } +int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + vInfo("query message is processed"); + return 0; } -int32_t vnodeAppendMsg(SVnodeMsg *pMsg, SRpcMsg *pRpcMsg) { - if (pMsg->curNum >= pMsg->allocNum) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pMsg->rpcMsg[pMsg->curNum++] = *pRpcMsg; +int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + vInfo("fetch message is processed"); + return 0; } -void vnodeCleanupMsg(SVnodeMsg *pMsg) { - for (int32_t i = 0; i < pMsg->curNum; ++i) { - rpcFreeCont(pMsg->rpcMsg[i].pCont); - } - taosFreeQitem(pMsg); +int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + vInfo("sync message is processed"); + return 0; } -void vnodeProcessMsg(SVnode *pVnode, SVnodeMsg *pMsg, EVnMsgType msgType) { - switch (msgType) { - case VN_MSG_TYPE_WRITE: - break; - case VN_MSG_TYPE_APPLY: - break; - case VN_MSG_TYPE_SYNC: - break; - case VN_MSG_TYPE_QUERY: - break; - case VN_MSG_TYPE_FETCH: - break; - } -} \ No newline at end of file +int vnodeProcessConsumeReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + vInfo("consume message is processed"); + return 0; +} diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index 70d9c7d4b09d899d91cecb5eaf7fc3159c495a82..c98f3e0800eaa0726c29708864040aa5d1a379ca 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -74,11 +74,14 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { pVnode->path = strdup(path); vnodeOptionsCopy(&(pVnode->config), pVnodeCfg); + tsem_init(&(pVnode->canCommit), 0, 1); + return pVnode; } static void vnodeFree(SVnode *pVnode) { if (pVnode) { + tsem_destroy(&(pVnode->canCommit)); tfree(pVnode->path); free(pVnode); } diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/impl/src/vnodeWrite.c index 85e044266ab368fac124c2f38d4008499e51b70e..5ec03f1fd3f7d7f4f2448d388146df311561dd35 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/impl/src/vnodeWrite.c @@ -16,29 +16,25 @@ #include "vnodeDef.h" int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { - SVnodeReq *pVnodeReq; - switch (pMsg->msgType) { - case TSDB_MSG_TYPE_MQ_SET: + case TSDB_MSG_TYPE_MQ_SET_CUR: if (tqSetCursor(pVnode->pTq, pMsg->pCont) < 0) { // TODO: handle error } break; } - - void *pBuf = pMsg->pCont; return 0; } int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { - SRpcMsg *pMsg; + SRpcMsg * pMsg; SVnodeReq *pVnodeReq; for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); // ser request version - void *pBuf = pMsg->pCont; + void * pBuf = pMsg->pCont; int64_t ver = pVnode->state.processed++; taosEncodeFixedU64(&pBuf, ver); @@ -49,68 +45,63 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { walFsync(pVnode->pWal, false); - // Apply each request now - for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { - pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); - SVnodeReq vReq; + // TODO: Integrate RAFT module here - // Apply the request - { - void *ptr = vnodeMalloc(pVnode, pMsg->contLen); - if (ptr == NULL) { - // TODO: handle error - } + return 0; +} + +int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + SVnodeReq vReq; + void * ptr = vnodeMalloc(pVnode, pMsg->contLen); + if (ptr == NULL) { + // TODO: handle error + } - // TODO: copy here need to be extended - memcpy(ptr, pMsg->pCont, pMsg->contLen); + // TODO: copy here need to be extended + memcpy(ptr, pMsg->pCont, pMsg->contLen); - // todo: change the interface here - uint64_t ver; - taosDecodeFixedU64(pMsg->pCont, &ver); - if (tqPushMsg(pVnode->pTq, ptr, ver) < 0) { + // todo: change the interface here + uint64_t ver; + taosDecodeFixedU64(pMsg->pCont, &ver); + if (tqPushMsg(pVnode->pTq, ptr, ver) < 0) { + // TODO: handle error + } + + vnodeParseReq(pMsg->pCont, &vReq, pMsg->msgType); + + switch (pMsg->msgType) { + case TSDB_MSG_TYPE_CREATE_STB_IN: + case TSDB_MSG_TYPE_CREATE_TABLE: + if (metaCreateTable(pVnode->pMeta, &(vReq.ctReq)) < 0) { // TODO: handle error } - vnodeParseReq(pMsg->pCont, &vReq, pMsg->msgType); - - switch (pMsg->msgType) { - case TSDB_MSG_TYPE_CREATE_TABLE: - if (metaCreateTable(pVnode->pMeta, &(vReq.ctReq)) < 0) { - // TODO: handle error - } - - // TODO: maybe need to clear the requst struct - break; - case TSDB_MSG_TYPE_DROP_TABLE: - if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) { - // TODO: handle error - } - break; - case TSDB_MSG_TYPE_SUBMIT: - if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr) < 0) { - // TODO: handle error - } - break; - default: - break; + // TODO: maybe need to clear the requst struct + break; + case TSDB_MSG_TYPE_DROP_STB_IN: + case TSDB_MSG_TYPE_DROP_TABLE: + if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) { + // TODO: handle error } - - pVnode->state.applied = ver; - } - - // Check if it needs to commit - if (vnodeShouldCommit(pVnode)) { - if (vnodeAsyncCommit(pVnode) < 0) { + break; + case TSDB_MSG_TYPE_SUBMIT: + if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr) < 0) { // TODO: handle error } - } + break; + default: + break; } - return 0; -} + pVnode->state.applied = ver; -int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - // TODO + // Check if it needs to commit + if (vnodeShouldCommit(pVnode)) { + tsem_wait(&(pVnode->canCommit)); + if (vnodeAsyncCommit(pVnode) < 0) { + // TODO: handle error + } + } return 0; } diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index 6f2e6f721ab335e7219e08e762a46041ddce73d7..aa9fc741d95122f8c60a44967e4b996d3b98deb5 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -81,7 +81,8 @@ static void vtBuildCreateStbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { pSchema = vtCreateBasicSchema(); pTagSchema = vtCreateBasicTagSchema(); - SVnodeReq vCreateSTbReq = VNODE_INIT_CREATE_STB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); + SVnodeReq vCreateSTbReq; + vnodeSetCreateStbReq(&vCreateSTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); zs = vnodeBuildReq(NULL, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs); @@ -104,7 +105,8 @@ static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { int tz; SKVRow pTag = vtCreateBasicTag(); - SVnodeReq vCreateCTbReq = VNODE_INIT_CREATE_CTB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pTag); + SVnodeReq vCreateCTbReq; + vnodeSetCreateCtbReq(&vCreateCTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pTag); tz = vnodeBuildReq(NULL, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz); @@ -166,6 +168,21 @@ static void vtClearMsgBatch(SArray *pMsgArr) { taosArrayClear(pMsgArr); } +static void vtProcessAndApplyReqs(SVnode *pVnode, SArray *pMsgArr) { + int rcode; + SRpcMsg *pReq; + SRpcMsg *pRsp; + + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + GTEST_ASSERT_EQ(rcode, 0); + + for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) { + pReq = *(SRpcMsg **)taosArrayGet(pMsgArr, i); + rcode = vnodeApplyWMsg(pVnode, pReq, NULL); + GTEST_ASSERT_EQ(rcode, 0); + } +} + TEST(vnodeApiTest, vnode_simple_create_table_test) { tb_uid_t suid = 1638166374163; SRpcMsg *pMsg; @@ -189,8 +206,7 @@ TEST(vnodeApiTest, vnode_simple_create_table_test) { sprintf(tbname, "st"); vtBuildCreateStbReq(suid, tbname, &pMsg); taosArrayPush(pMsgArr, &pMsg); - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - ASSERT_EQ(rcode, 0); + vtProcessAndApplyReqs(pVnode, pMsgArr); vtClearMsgBatch(pMsgArr); // CREATE A LOT OF CHILD TABLES @@ -203,8 +219,7 @@ TEST(vnodeApiTest, vnode_simple_create_table_test) { } // Process request batch - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - ASSERT_EQ(rcode, 0); + vtProcessAndApplyReqs(pVnode, pMsgArr); // Clear request batch vtClearMsgBatch(pMsgArr); @@ -242,16 +257,14 @@ TEST(vnodeApiTest, vnode_simple_insert_test) { sprintf(tbname, "st"); vtBuildCreateStbReq(suid, tbname, &pMsg); taosArrayPush(pMsgArr, &pMsg); - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - GTEST_ASSERT_EQ(rcode, 0); + vtProcessAndApplyReqs(pVnode, pMsgArr); vtClearMsgBatch(pMsgArr); // 2. CREATE A CHILD TABLE sprintf(tbname, "t0"); vtBuildCreateCtbReq(suid, tbname, &pMsg); taosArrayPush(pMsgArr, &pMsg); - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - GTEST_ASSERT_EQ(rcode, 0); + vtProcessAndApplyReqs(pVnode, pMsgArr); vtClearMsgBatch(pMsgArr); // 3. WRITE A LOT OF TIME-SERIES DATA @@ -260,8 +273,7 @@ TEST(vnodeApiTest, vnode_simple_insert_test) { vtBuildSubmitReq(&pMsg); taosArrayPush(pMsgArr, &pMsg); } - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - GTEST_ASSERT_EQ(rcode, 0); + vtProcessAndApplyReqs(pVnode, pMsgArr); vtClearMsgBatch(pMsgArr); } diff --git a/source/dnode/vnode/tq/CMakeLists.txt b/source/dnode/vnode/tq/CMakeLists.txt index 536e97d5f79b58442b2d0a6af5758bb4ec854fc1..8d59c7b07a9c3935187ba6b644d7f95f88179776 100644 --- a/source/dnode/vnode/tq/CMakeLists.txt +++ b/source/dnode/vnode/tq/CMakeLists.txt @@ -11,6 +11,7 @@ target_link_libraries( PUBLIC wal PUBLIC os PUBLIC util + PUBLIC common ) if(${BUILD_TEST}) diff --git a/source/dnode/vnode/tq/inc/tqInt.h b/source/dnode/vnode/tq/inc/tqInt.h index 022b5998162b907382643d1e01a8cf0b05596696..5685a29d03e7dc15a974d9728d0c6ec07d4cceb3 100644 --- a/source/dnode/vnode/tq/inc/tqInt.h +++ b/source/dnode/vnode/tq/inc/tqInt.h @@ -17,11 +17,20 @@ #define _TD_TQ_INT_H_ #include "tq.h" - +#include "tlog.h" #ifdef __cplusplus extern "C" { #endif +extern int32_t tqDebugFlag; + +#define tqFatal(...) { if (tqDebugFlag & DEBUG_FATAL) { taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); }} +#define tqError(...) { if (tqDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); }} +#define tqWarn(...) { if (tqDebugFlag & DEBUG_WARN) { taosPrintLog("TQ WARN ", 255, __VA_ARGS__); }} +#define tqInfo(...) { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", 255, __VA_ARGS__); }} +#define tqDebug(...) { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }} +#define tqTrace(...) { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }} + // create persistent storage for meta info such as consuming offset // return value > 0: cgId // return value <= 0: error code diff --git a/source/dnode/vnode/tq/inc/tqMetaStore.h b/source/dnode/vnode/tq/inc/tqMetaStore.h index 5bcedaed748d7d0668181183d09de7470ff630e2..ef71d8bf145ae535edf7740e864661e576078814 100644 --- a/source/dnode/vnode/tq/inc/tqMetaStore.h +++ b/source/dnode/vnode/tq/inc/tqMetaStore.h @@ -17,7 +17,7 @@ #define _TQ_META_STORE_H_ #include "os.h" -#include "tq.h" +#include "tqInt.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index 1a27870a1b738d52f1c142f010e8e7e44abce50d..667097e45f0ba482625afac818447c9489ae7f76 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -24,89 +24,82 @@ // handle management message // -int tqGetgHandleSSize(const STqGroupHandle* gHandle); -int tqBufHandleSSize(); -int tqBufItemSSize(); +int tqGroupSSize(const STqGroup* pGroup); +int tqTopicSSize(); +int tqItemSSize(); -STqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - STqGroupHandle* gHandle; - return NULL; -} +void* tqSerializeListHandle(STqList* listHandle, void* ptr); +void* tqSerializeTopic(STqTopic* pTopic, void* ptr); +void* tqSerializeItem(STqMsgItem* pItem, void* ptr); -void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr); -void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr); -void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr); - -const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle); -const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem); +const void* tqDeserializeTopic(const void* pBytes, STqTopic* pTopic); +const void* tqDeserializeItem(const void* pBytes, STqMsgItem* pItem); STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac) { STQ* pTq = malloc(sizeof(STQ)); if (pTq == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } pTq->path = strdup(path); pTq->tqConfig = tqConfig; pTq->tqLogReader = tqLogReader; +#if 0 pTq->tqMemRef.pAlloctorFactory = allocFac; - // pTq->tqMemRef.pAllocator = allocFac->create(allocFac); + pTq->tqMemRef.pAllocator = allocFac->create(allocFac); if (pTq->tqMemRef.pAllocator == NULL) { - // TODO + // TODO: error code of buffer pool } - pTq->tqMeta = - tqStoreOpen(path, (FTqSerialize)tqSerializeGroupHandle, (FTqDeserialize)tqDeserializeGroupHandle, free, 0); +#endif + pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeGroup, (FTqDeserialize)tqDeserializeGroup, free, 0); if (pTq->tqMeta == NULL) { // TODO: free STQ return NULL; } return pTq; } - -void tqClose(STQ*pTq) { +void tqClose(STQ* pTq) { // TODO } -static int tqProtoCheck(TmqMsgHead *pMsg) { - return pMsg->protoVer == 0; -} +static int tqProtoCheck(STqMsgHead* pMsg) { return pMsg->protoVer == 0; } -static int tqAckOneTopic(STqBufferHandle* bHandle, TmqOneAck* pAck, STqQueryMsg** ppQuery) { +static int tqAckOneTopic(STqTopic* pTopic, STqOneAck* pAck, STqQueryMsg** ppQuery) { // clean old item and move forward int32_t consumeOffset = pAck->consumeOffset; int idx = consumeOffset % TQ_BUFFER_SIZE; - ASSERT(bHandle->buffer[idx].content && bHandle->buffer[idx].executor); - tfree(bHandle->buffer[idx].content); + ASSERT(pTopic->buffer[idx].content && pTopic->buffer[idx].executor); + tfree(pTopic->buffer[idx].content); if (1 /* TODO: need to launch new query */) { STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg)); if (pNewQuery == NULL) { - // TODO: memory insufficient + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } // TODO: lock executor - pNewQuery->exec->executor = bHandle->buffer[idx].executor; // TODO: read from wal and assign to src - pNewQuery->exec->src = 0; - pNewQuery->exec->dest = &bHandle->buffer[idx]; - pNewQuery->next = *ppQuery; - *ppQuery = pNewQuery; + /*pNewQuery->exec->executor = pTopic->buffer[idx].executor;*/ + /*pNewQuery->exec->src = 0;*/ + /*pNewQuery->exec->dest = &pTopic->buffer[idx];*/ + /*pNewQuery->next = *ppQuery;*/ + /**ppQuery = pNewQuery;*/ } return 0; } -static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) { +static int tqAck(STqGroup* pGroup, STqAcks* pAcks) { int32_t ackNum = pAcks->ackNum; - TmqOneAck* acks = pAcks->acks; + STqOneAck* acks = pAcks->acks; // double ptr for acks and list - int i = 0; - STqListHandle* node = gHandle->head; - int ackCnt = 0; - STqQueryMsg* pQuery = NULL; + int i = 0; + STqList* node = pGroup->head; + int ackCnt = 0; + STqQueryMsg* pQuery = NULL; while (i < ackNum && node->next) { - if (acks[i].topicId == node->next->bufHandle.topicId) { + if (acks[i].topicId == node->next->topic.topicId) { ackCnt++; - tqAckOneTopic(&node->next->bufHandle, &acks[i], &pQuery); - } else if (acks[i].topicId < node->next->bufHandle.topicId) { + tqAckOneTopic(&node->next->topic, &acks[i], &pQuery); + } else if (acks[i].topicId < node->next->topic.topicId) { i++; } else { node = node->next; @@ -118,52 +111,56 @@ static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) { return ackCnt; } -static int tqCommitTCGroup(STqGroupHandle* handle) { +static int tqCommitGroup(STqGroup* pGroup) { // persist modification into disk return 0; } -int tqCreateTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroupHandle** handle) { +int tqCreateGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroup** ppGroup) { // create in disk - STqGroupHandle* gHandle = (STqGroupHandle*)malloc(sizeof(STqGroupHandle)); - if (gHandle == NULL) { + STqGroup* pGroup = (STqGroup*)malloc(sizeof(STqGroup)); + if (pGroup == NULL) { // TODO return -1; } - memset(gHandle, 0, sizeof(STqGroupHandle)); + *ppGroup = pGroup; + memset(pGroup, 0, sizeof(STqGroup)); return 0; } -STqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - STqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId); - if (gHandle == NULL) { - int code = tqCreateTCGroup(pTq, topicId, cgId, cId, &gHandle); - if (code != 0) { +STqGroup* tqOpenGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { + STqGroup* pGroup = tqHandleGet(pTq->tqMeta, cId); + if (pGroup == NULL) { + int code = tqCreateGroup(pTq, topicId, cgId, cId, &pGroup); + if (code < 0) { // TODO return NULL; } + tqHandleMovePut(pTq->tqMeta, cId, pGroup); } + ASSERT(pGroup); - // create - // open - return gHandle; + return pGroup; } -int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { return 0; } +int tqCloseGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { + // TODO + return 0; +} -int tqDropTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { +int tqDropGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { // delete from disk return 0; } -static int tqFetch(STqGroupHandle* gHandle, void** msg) { - STqListHandle* head = gHandle->head; - STqListHandle* node = head; - int totSize = 0; +static int tqFetch(STqGroup* pGroup, void** msg) { + STqList* head = pGroup->head; + STqList* node = head; + int totSize = 0; // TODO: make it a macro int sizeLimit = 4 * 1024; - TmqMsgContent* buffer = malloc(sizeLimit); + STqMsgContent* buffer = malloc(sizeLimit); if (buffer == NULL) { // TODO:memory insufficient return -1; @@ -172,25 +169,25 @@ static int tqFetch(STqGroupHandle* gHandle, void** msg) { // until all topic iterated or msgs over sizeLimit while (node->next) { node = node->next; - STqBufferHandle* bufHandle = &node->bufHandle; - int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE; - if (bufHandle->buffer[idx].content != NULL && bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset) { - totSize += bufHandle->buffer[idx].size; + STqTopic* topicHandle = &node->topic; + int idx = topicHandle->nextConsumeOffset % TQ_BUFFER_SIZE; + if (topicHandle->buffer[idx].content != NULL && topicHandle->buffer[idx].offset == topicHandle->nextConsumeOffset) { + totSize += topicHandle->buffer[idx].size; if (totSize > sizeLimit) { void* ptr = realloc(buffer, totSize); if (ptr == NULL) { - totSize -= bufHandle->buffer[idx].size; + totSize -= topicHandle->buffer[idx].size; // TODO:memory insufficient // return msgs already copied break; } } - *((int64_t*)buffer) = bufHandle->topicId; + *((int64_t*)buffer) = topicHandle->topicId; buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - *((int64_t*)buffer) = bufHandle->buffer[idx].size; + *((int64_t*)buffer) = topicHandle->buffer[idx].size; buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - memcpy(buffer, bufHandle->buffer[idx].content, bufHandle->buffer[idx].size); - buffer = POINTER_SHIFT(buffer, bufHandle->buffer[idx].size); + memcpy(buffer, topicHandle->buffer[idx].content, topicHandle->buffer[idx].size); + buffer = POINTER_SHIFT(buffer, topicHandle->buffer[idx].size); if (totSize > sizeLimit) { break; } @@ -199,11 +196,19 @@ static int tqFetch(STqGroupHandle* gHandle, void** msg) { return totSize; } -STqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { return NULL; } - -int tqLaunchQuery(STqGroupHandle* gHandle) { return 0; } +STqGroup* tqGetGroup(STQ* pTq, int64_t clientId) { return tqHandleGet(pTq->tqMeta, clientId); } -int tqSendLaunchQuery(STqGroupHandle* gHandle) { return 0; } +int tqSendLaunchQuery(STqMsgItem* bufItem, int64_t offset) { + if (tqQueryExecuting(bufItem->status)) { + return 0; + } + bufItem->status = 1; + // load data from wal or buffer pool + // put into exec + // send exec into non blocking queue + // when query finished, put into buffer pool + return 0; +} /*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/ /*return 0;*/ @@ -220,23 +225,96 @@ int tqCommit(STQ* pTq) { return 0; } -int tqSetCursor(STQ* pTq, void* msg) { +int tqBufferSetOffset(STqTopic* pTopic, int64_t offset) { + int code; + memset(pTopic->buffer, 0, sizeof(pTopic->buffer)); + // launch query + for (int i = offset; i < offset + TQ_BUFFER_SIZE; i++) { + int pos = i % TQ_BUFFER_SIZE; + code = tqSendLaunchQuery(&pTopic->buffer[pos], offset); + if (code < 0) { + // TODO: error handling + } + } + // set offset + pTopic->nextConsumeOffset = offset; + pTopic->floatingCursor = offset; + return 0; +} + +STqTopic* tqFindTopic(STqGroup* pGroup, int64_t topicId) { + // TODO + return NULL; +} + +int tqSetCursor(STQ* pTq, STqSetCurReq* pMsg) { + int code; + int64_t clientId = pMsg->head.clientId; + int64_t topicId = pMsg->topicId; + int64_t offset = pMsg->offset; + STqGroup* gHandle = tqGetGroup(pTq, clientId); + if (gHandle == NULL) { + // client not connect + return -1; + } + STqTopic* topicHandle = tqFindTopic(gHandle, topicId); + if (topicHandle == NULL) { + return -1; + } + if (pMsg->offset == topicHandle->nextConsumeOffset) { + return 0; + } + // TODO: check log last version + + code = tqBufferSetOffset(topicHandle, offset); + if (code < 0) { + // set error code + return -1; + } + + return 0; +} + +int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { + int64_t clientId = pMsg->head.clientId; + STqGroup* pGroup = tqGetGroup(pTq, clientId); + if (pGroup == NULL) { + terrno = TSDB_CODE_TQ_GROUP_NOT_SET; + return -1; + } + + STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg; + int numOfMsgs = tqFetch(pGroup, (void**)&pRsp->msgs); + if (numOfMsgs < 0) { + return -1; + } + if (numOfMsgs == 0) { + // most recent data has been fetched + + // enable timer for blocking wait + // once new data written during wait time + // launch query and response + } + + // fetched a num of msgs, rpc response + return 0; } +#if 0 int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { - if (!tqProtoCheck((TmqMsgHead*)pMsg)) { + if (!tqProtoCheck((STqMsgHead*)pMsg)) { // proto version invalid return -1; } - int64_t clientId = pMsg->head.clientId; - STqGroupHandle* gHandle = tqGetGroupHandle(pTq, clientId); - if (gHandle == NULL) { + int64_t clientId = pMsg->head.clientId; + STqGroup* pGroup = tqGetGroup(pTq, clientId); + if (pGroup == NULL) { // client not connect return -1; } if (pMsg->acks.ackNum != 0) { - if (tqAck(gHandle, &pMsg->acks) != 0) { + if (tqAck(pGroup, &pMsg->acks) != 0) { // ack not success return -1; } @@ -244,22 +322,23 @@ int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg; - if (tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) { + if (tqFetch(pGroup, (void**)&pRsp->msgs) <= 0) { // fetch error return -1; } // judge and launch new query - if (tqLaunchQuery(gHandle)) { - // launch query error - return -1; - } + /*if (tqSendLaunchQuery(gHandle)) {*/ + // launch query error + /*return -1;*/ + /*}*/ return 0; } +#endif -int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead) { +int tqSerializeGroup(const STqGroup* pGroup, STqSerializedHead** ppHead) { // calculate size - int sz = tqGetgHandleSSize(gHandle) + sizeof(STqSerializedHead); + int sz = tqGroupSSize(pGroup) + sizeof(STqSerializedHead); if (sz > (*ppHead)->ssize) { void* tmpPtr = realloc(*ppHead, sz); if (tmpPtr == NULL) { @@ -272,53 +351,53 @@ int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** pp } void* ptr = (*ppHead)->content; // do serialization - *(int64_t*)ptr = gHandle->cId; + *(int64_t*)ptr = pGroup->clientId; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int64_t*)ptr = gHandle->cgId; + *(int64_t*)ptr = pGroup->cgId; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int32_t*)ptr = gHandle->topicNum; + *(int32_t*)ptr = pGroup->topicNum; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - if (gHandle->topicNum > 0) { - tqSerializeListHandle(gHandle->head, ptr); + if (pGroup->topicNum > 0) { + tqSerializeListHandle(pGroup->head, ptr); } return 0; } -void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr) { - STqListHandle* node = listHandle; +void* tqSerializeListHandle(STqList* listHandle, void* ptr) { + STqList* node = listHandle; ASSERT(node != NULL); while (node) { - ptr = tqSerializeBufHandle(&node->bufHandle, ptr); + ptr = tqSerializeTopic(&node->topic, ptr); node = node->next; } return ptr; } -void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr) { - *(int64_t*)ptr = bufHandle->nextConsumeOffset; +void* tqSerializeTopic(STqTopic* pTopic, void* ptr) { + *(int64_t*)ptr = pTopic->nextConsumeOffset; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int64_t*)ptr = bufHandle->topicId; + *(int64_t*)ptr = pTopic->topicId; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int32_t*)ptr = bufHandle->head; + *(int32_t*)ptr = pTopic->head; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - *(int32_t*)ptr = bufHandle->tail; + *(int32_t*)ptr = pTopic->tail; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - ptr = tqSerializeBufItem(&bufHandle->buffer[i], ptr); + ptr = tqSerializeItem(&pTopic->buffer[i], ptr); } return ptr; } -void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr) { +void* tqSerializeItem(STqMsgItem* bufItem, void* ptr) { // TODO: do we need serialize this? // mainly for executor return ptr; } -const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** ppGHandle) { - STqGroupHandle* gHandle = *ppGHandle; - const void* ptr = pHead->content; - gHandle->cId = *(int64_t*)ptr; +const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup) { + STqGroup* gHandle = *ppGroup; + const void* ptr = pHead->content; + gHandle->clientId = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); gHandle->cgId = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); @@ -326,63 +405,63 @@ const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHan gHandle->topicNum = *(int32_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); gHandle->head = NULL; - STqListHandle* node = gHandle->head; + STqList* node = gHandle->head; for (int i = 0; i < gHandle->topicNum; i++) { if (gHandle->head == NULL) { - if ((node = malloc(sizeof(STqListHandle))) == NULL) { + if ((node = malloc(sizeof(STqList))) == NULL) { // TODO: error return NULL; } node->next = NULL; - ptr = tqDeserializeBufHandle(ptr, &node->bufHandle); + ptr = tqDeserializeTopic(ptr, &node->topic); gHandle->head = node; } else { - node->next = malloc(sizeof(STqListHandle)); + node->next = malloc(sizeof(STqList)); if (node->next == NULL) { // TODO: error return NULL; } node->next->next = NULL; - ptr = tqDeserializeBufHandle(ptr, &node->next->bufHandle); + ptr = tqDeserializeTopic(ptr, &node->next->topic); node = node->next; } } return ptr; } -const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle) { +const void* tqDeserializeTopic(const void* pBytes, STqTopic* topic) { const void* ptr = pBytes; - bufHandle->nextConsumeOffset = *(int64_t*)ptr; + topic->nextConsumeOffset = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - bufHandle->topicId = *(int64_t*)ptr; + topic->topicId = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - bufHandle->head = *(int32_t*)ptr; + topic->head = *(int32_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - bufHandle->tail = *(int32_t*)ptr; + topic->tail = *(int32_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - ptr = tqDeserializeBufItem(ptr, &bufHandle->buffer[i]); + ptr = tqDeserializeItem(ptr, &topic->buffer[i]); } return ptr; } -const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem) { return pBytes; } +const void* tqDeserializeItem(const void* pBytes, STqMsgItem* bufItem) { return pBytes; } // TODO: make this a macro -int tqGetgHandleSSize(const STqGroupHandle* gHandle) { +int tqGroupSSize(const STqGroup* gHandle) { return sizeof(int64_t) * 2 // cId + cgId + sizeof(int32_t) // topicNum - + gHandle->topicNum * tqBufHandleSSize(); + + gHandle->topicNum * tqTopicSSize(); } // TODO: make this a macro -int tqBufHandleSSize() { +int tqTopicSSize() { return sizeof(int64_t) * 2 // nextConsumeOffset + topicId + sizeof(int32_t) * 2 // head + tail - + TQ_BUFFER_SIZE * tqBufItemSSize(); + + TQ_BUFFER_SIZE * tqItemSSize(); } -int tqBufItemSSize() { +int tqItemSSize() { // TODO: do this need serialization? // mainly for executor return 0; diff --git a/source/dnode/vnode/tq/src/tqMetaStore.c b/source/dnode/vnode/tq/src/tqMetaStore.c index 082f0ad28e30356794d9cc12b69eac2668a6fb9c..57e20010e3b12b615875e1ecaba1eddd05161562 100644 --- a/source/dnode/vnode/tq/src/tqMetaStore.c +++ b/source/dnode/vnode/tq/src/tqMetaStore.c @@ -56,6 +56,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { int offset = tqSeekLastPage(fd); int nBytes; if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } if (nBytes == 0) { @@ -71,7 +72,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial int32_t tqConfigFlag) { STqMetaStore* pMeta = malloc(sizeof(STqMetaStore)); if (pMeta == NULL) { - // close + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } memset(pMeta, 0, sizeof(STqMetaStore)); @@ -79,8 +80,9 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial // concat data file name and index file name size_t pathLen = strlen(path); pMeta->dirPath = malloc(pathLen + 1); - if (pMeta->dirPath != NULL) { - // TODO: memory insufficient + if (pMeta->dirPath == NULL) { + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + return NULL; } strcpy(pMeta->dirPath, path); @@ -88,13 +90,14 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial strcpy(name, path); if (taosDirExist(name) != 0 && taosMkDir(name) != 0) { - ASSERT(false); + terrno = TSDB_CODE_TQ_FAILED_TO_CREATE_DIR; + tqError("failed to create dir:%s since %s ", name, terrstr()); } strcat(name, "/" TQ_IDX_NAME); int idxFd = open(name, O_RDWR | O_CREAT, 0755); if (idxFd < 0) { - ASSERT(false); - // close file + terrno = TAOS_SYSTEM_ERROR(errno); + tqError("failed to open file:%s since %s ", name, terrstr()); // free memory return NULL; } @@ -102,9 +105,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial pMeta->idxFd = idxFd; pMeta->unpersistHead = malloc(sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { - ASSERT(false); - // close file - // free memory + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } memset(pMeta->unpersistHead, 0, sizeof(STqMetaList)); @@ -114,7 +115,8 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial strcat(name, "/" TQ_META_NAME); int fileFd = open(name, O_RDWR | O_CREAT, 0755); if (fileFd < 0) { - ASSERT(false); + terrno = TAOS_SYSTEM_ERROR(errno); + tqError("failed to open file:%s since %s", name, terrstr()); return NULL; } @@ -129,7 +131,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial STqIdxPageBuf idxBuf; STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE); if (serializedObj == NULL) { - // TODO:memory insufficient + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; } int idxRead; int allocated = TQ_PAGE_SIZE; @@ -137,14 +139,16 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { if (idxRead == -1) { // TODO: handle error - ASSERT(false); + terrno = TAOS_SYSTEM_ERROR(errno); + tqError("failed to read tq index file since %s", terrstr()); } ASSERT(idxBuf.head.writeOffset == idxRead); // loop read every entry for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { STqMetaList* pNode = malloc(sizeof(STqMetaList)); if (pNode == NULL) { - // TODO: free memory and return error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + // TODO: free memory } memset(pNode, 0, sizeof(STqMetaList)); memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); @@ -153,7 +157,8 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial if (allocated < pNode->handle.serializedSize) { void* ptr = realloc(serializedObj, pNode->handle.serializedSize); if (ptr == NULL) { - // TODO: memory insufficient + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + // TODO: free memory } serializedObj = ptr; allocated = pNode->handle.serializedSize; @@ -292,7 +297,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { STqMetaList* pNode = pHead->unpersistNext; STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead)); if (pSHead == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } pSHead->ver = TQ_SVER; @@ -403,7 +408,6 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu STqMetaList* pNode = pMeta->bucket[bucketKey]; while (pNode) { if (pNode->handle.key == key) { - // TODO: think about thread safety if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInUse); } @@ -416,7 +420,7 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu } STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); if (pNewNode == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } memset(pNewNode, 0, sizeof(STqMetaList)); @@ -470,10 +474,10 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va STqMetaList* pNode = pMeta->bucket[bucketKey]; while (pNode) { if (pNode->handle.key == key) { - // TODO: think about thread safety if (pNode->handle.valueInTxn) { - if (TqDupIntxnReject(pMeta->tqConfigFlag)) { - return -2; + if (tqDupIntxnReject(pMeta->tqConfigFlag)) { + terrno = TSDB_CODE_TQ_META_KEY_DUP_IN_TXN; + return -1; } if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInTxn); @@ -488,7 +492,7 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va } STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); if (pNewNode == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } memset(pNewNode, 0, sizeof(STqMetaList)); @@ -505,7 +509,7 @@ int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { void* vmem = malloc(vsize); if (vmem == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } memcpy(vmem, value, vsize); @@ -535,6 +539,7 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) { while (pNode) { if (pNode->handle.key == key) { if (pNode->handle.valueInTxn == NULL) { + terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN; return -1; } if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { @@ -548,7 +553,8 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) { pNode = pNode->next; } } - return -2; + terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; + return -1; } int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { @@ -564,12 +570,14 @@ int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { tqLinkUnpersist(pMeta, pNode); return 0; } + terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN; return -1; } else { pNode = pNode->next; } } - return -2; + terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; + return -1; } int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { @@ -588,7 +596,7 @@ int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { pNode = pNode->next; } } - // no such key + terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; return -1; } diff --git a/source/dnode/vnode/tq/test/tqMetaTest.cpp b/source/dnode/vnode/tq/test/tqMetaTest.cpp index 58263efa71ec77db6a5fbfc973d4f7c9edd55714..d3c9b50e4abd884903f50b2bf5d10bd8ec71eb00 100644 --- a/source/dnode/vnode/tq/test/tqMetaTest.cpp +++ b/source/dnode/vnode/tq/test/tqMetaTest.cpp @@ -10,8 +10,8 @@ struct Foo { }; int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { - Foo* foo = (Foo*) pObj; - if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) { + Foo* foo = (Foo*)pObj; + if ((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) { *ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t)); (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t); } @@ -20,36 +20,28 @@ int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { } const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) { - if(*ppObj == NULL) { + if (*ppObj == NULL) { *ppObj = realloc(*ppObj, sizeof(int32_t)); } Foo* pFoo = *(Foo**)ppObj; - pFoo->a = *(int32_t*)pHead->content; + pFoo->a = *(int32_t*)pHead->content; return NULL; } -void FooDeleter(void* pObj) { - free(pObj); -} +void FooDeleter(void* pObj) { free(pObj); } class TqMetaUpdateAppendTest : public ::testing::Test { - protected: - - void SetUp() override { - taosRemoveDir(pathName); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); - ASSERT(pMeta); - } - - void TearDown() override { - tqStoreClose(pMeta); - } - - TqMetaStore* pMeta; - const char* pathName = "/tmp/tq_test"; + protected: + void SetUp() override { + taosRemoveDir(pathName); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); + ASSERT(pMeta); + } + + void TearDown() override { tqStoreClose(pMeta); } + + STqMetaStore* pMeta; + const char* pathName = "/tmp/tq_test"; }; TEST_F(TqMetaUpdateAppendTest, copyPutTest) { @@ -57,11 +49,11 @@ TEST_F(TqMetaUpdateAppendTest, copyPutTest) { foo.a = 3; tqHandleCopyPut(pMeta, 1, &foo, sizeof(Foo)); - Foo* pFoo = (Foo*) tqHandleGet(pMeta, 1); + Foo* pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); tqHandleCommit(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo->a, 3); } @@ -78,10 +70,7 @@ TEST_F(TqMetaUpdateAppendTest, persistTest) { EXPECT_EQ(pBar == NULL, true); tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); pBar = (Foo*)tqHandleGet(pMeta, 1); @@ -97,7 +86,7 @@ TEST_F(TqMetaUpdateAppendTest, uncommittedTest) { pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); } @@ -106,11 +95,11 @@ TEST_F(TqMetaUpdateAppendTest, abortTest) { pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); tqHandleAbort(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); } @@ -119,32 +108,29 @@ TEST_F(TqMetaUpdateAppendTest, deleteTest) { pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); tqHandleCommit(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); ASSERT_EQ(pFoo != NULL, true); EXPECT_EQ(pFoo->a, 3); tqHandleDel(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); ASSERT_EQ(pFoo != NULL, true); EXPECT_EQ(pFoo->a, 3); tqHandleCommit(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); } @@ -162,10 +148,7 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) { EXPECT_EQ(pFoo1->a, 3); tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); pFoo1 = (Foo*)tqHandleGet(pMeta, 1); @@ -177,10 +160,7 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) { EXPECT_EQ(pFoo1->a, 4); tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); pFoo1 = (Foo*)tqHandleGet(pMeta, 1); @@ -190,13 +170,13 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) { TEST_F(TqMetaUpdateAppendTest, multiplePage) { srand(0); std::vector v; - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { v.push_back(rand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 0; i < 500; i++) { + for (int i = 0; i < 500; i++) { tqHandleCommit(pMeta, i); Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; @@ -204,38 +184,34 @@ TEST_F(TqMetaUpdateAppendTest, multiplePage) { } tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); - - for(int i = 500; i < 1000; i++) { + + for (int i = 500; i < 1000; i++) { tqHandleCommit(pMeta, i); Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; EXPECT_EQ(pFoo->a, v[i]); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; EXPECT_EQ(pFoo->a, v[i]); } - } TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { srand(0); std::vector v; - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { v.push_back(rand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 0; i < 500; i++) { + for (int i = 0; i < 500; i++) { tqHandleCommit(pMeta, i); v[i] = rand(); Foo foo; @@ -243,25 +219,22 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 500; i < 1000; i++) { + for (int i = 500; i < 1000; i++) { v[i] = rand(); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { tqHandleCommit(pMeta, i); } tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); - - for(int i = 500; i < 1000; i++) { + + for (int i = 500; i < 1000; i++) { v[i] = rand(); Foo foo; foo.a = v[i]; @@ -269,40 +242,38 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { tqHandleCommit(pMeta, i); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; EXPECT_EQ(pFoo->a, v[i]); } - } TEST_F(TqMetaUpdateAppendTest, dupCommit) { srand(0); std::vector v; - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { v.push_back(rand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { int ret = tqHandleCommit(pMeta, i); EXPECT_EQ(ret, 0); ret = tqHandleCommit(pMeta, i); EXPECT_EQ(ret, -1); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { int ret = tqHandleCommit(pMeta, i); EXPECT_EQ(ret, -1); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; EXPECT_EQ(pFoo->a, v[i]); } - } diff --git a/source/dnode/vnode/tsdb/CMakeLists.txt b/source/dnode/vnode/tsdb/CMakeLists.txt index 870d75cd20ed3979cbb02a2aa2e96de131e64bf8..30e9d70f125c42990665948614ff570d1badc16b 100644 --- a/source/dnode/vnode/tsdb/CMakeLists.txt +++ b/source/dnode/vnode/tsdb/CMakeLists.txt @@ -1,10 +1,24 @@ aux_source_directory(src TSDB_SRC) -add_library(tsdb ${TSDB_SRC}) +if(0) + add_library(tsdb ${TSDB_SRC}) +else(0) + add_library(tsdb "") + target_sources(tsdb + PRIVATE + "src/tsdbCommit.c" + "src/tsdbMain.c" + "src/tsdbMemTable.c" + "src/tsdbOptions.c" + "src/tsdbWrite.c" + ) +endif(0) + target_include_directories( tsdb PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/tsdb" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + target_link_libraries( tsdb PUBLIC os diff --git a/src/tsdb/inc/tsdbCommit.h b/source/dnode/vnode/tsdb/inc/tsdbCommit.h similarity index 97% rename from src/tsdb/inc/tsdbCommit.h rename to source/dnode/vnode/tsdb/inc/tsdbCommit.h index cde728b1705cd1eead065772978631fb4b36246d..82ba1c9dffc98323d369fc6d8704ada0eb4210c7 100644 --- a/src/tsdb/inc/tsdbCommit.h +++ b/source/dnode/vnode/tsdb/inc/tsdbCommit.h @@ -16,6 +16,7 @@ #ifndef _TD_TSDB_COMMIT_H_ #define _TD_TSDB_COMMIT_H_ +#if 0 typedef struct { int minFid; int midFid; @@ -53,5 +54,6 @@ static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) { return -1; } } +#endif #endif /* _TD_TSDB_COMMIT_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbCompact.h b/source/dnode/vnode/tsdb/inc/tsdbCompact.h similarity index 94% rename from src/tsdb/inc/tsdbCompact.h rename to source/dnode/vnode/tsdb/inc/tsdbCompact.h index 5a382de5e0cc438dbd444d9a0895fd5f55094476..441dfda6ade603c2dd1240962543307ca3bb83f9 100644 --- a/src/tsdb/inc/tsdbCompact.h +++ b/source/dnode/vnode/tsdb/inc/tsdbCompact.h @@ -19,8 +19,12 @@ extern "C" { #endif +#if 0 + void *tsdbCompactImpl(STsdbRepo *pRepo); +#endif + #ifdef __cplusplus } #endif diff --git a/src/tsdb/inc/tsdbFS.h b/source/dnode/vnode/tsdb/inc/tsdbFS.h similarity index 99% rename from src/tsdb/inc/tsdbFS.h rename to source/dnode/vnode/tsdb/inc/tsdbFS.h index e89e10f7667e8aa5388ebfa4d2c5b54f1bf3e57f..03207567835c85cabd6ff8a97f3288d5481ff080 100644 --- a/src/tsdb/inc/tsdbFS.h +++ b/source/dnode/vnode/tsdb/inc/tsdbFS.h @@ -16,6 +16,8 @@ #ifndef _TD_TSDB_FS_H_ #define _TD_TSDB_FS_H_ +#if 0 + #define TSDB_FS_VERSION 0 // ================== TSDB global config @@ -113,4 +115,6 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS* pFs) { return 0; } +#endif + #endif /* _TD_TSDB_FS_H_ */ diff --git a/src/tsdb/inc/tsdbFile.h b/source/dnode/vnode/tsdb/inc/tsdbFile.h similarity index 99% rename from src/tsdb/inc/tsdbFile.h rename to source/dnode/vnode/tsdb/inc/tsdbFile.h index b9d5431de6bc3864a4a13ea30356033de76da178..73a7de02499bb316b27728de85b754cdc108c253 100644 --- a/src/tsdb/inc/tsdbFile.h +++ b/source/dnode/vnode/tsdb/inc/tsdbFile.h @@ -16,6 +16,8 @@ #ifndef _TS_TSDB_FILE_H_ #define _TS_TSDB_FILE_H_ +#if 0 + #define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_DELIMITER 0xF00AFA0F #define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF @@ -364,4 +366,5 @@ static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) { return true; } +#endif #endif /* _TS_TSDB_FILE_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbHealth.h b/source/dnode/vnode/tsdb/inc/tsdbHealth.h similarity index 98% rename from src/tsdb/inc/tsdbHealth.h rename to source/dnode/vnode/tsdb/inc/tsdbHealth.h index 324f4312e05fc0ca0200c319728bf692bf476bf6..d7b70ac053749863241fdd946842c7face2ed961 100644 --- a/src/tsdb/inc/tsdbHealth.h +++ b/source/dnode/vnode/tsdb/inc/tsdbHealth.h @@ -16,10 +16,14 @@ #ifndef _TD_TSDB_HEALTH_H_ #define _TD_TSDB_HEALTH_H_ +#if 0 + bool tsdbUrgeQueryFree(STsdbRepo* pRepo); int32_t tsdbInsertNewBlock(STsdbRepo* pRepo); bool tsdbIdleMemEnough(); bool tsdbAllowNewBlock(STsdbRepo* pRepo); +#endif + #endif /* _TD_TSDB_BUFFER_H_ */ diff --git a/source/dnode/vnode/tsdb/inc/tsdbIdx.h b/source/dnode/vnode/tsdb/inc/tsdbIdx.h deleted file mode 100644 index 73b2c5e6c5eaa02db5961d70e3ad0ec89b10a2c8..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/tsdb/inc/tsdbIdx.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_IDX_H_ -#define _TD_TSDB_IDX_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TSDB_IDX_H_*/ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbReadImpl.h b/source/dnode/vnode/tsdb/inc/tsdbReadImpl.h similarity index 99% rename from src/tsdb/inc/tsdbReadImpl.h rename to source/dnode/vnode/tsdb/inc/tsdbReadImpl.h index 814c4d130599768e8237145559c47e50e64db4db..a9bd76c2b162461fed199842533016b93a6a4f04 100644 --- a/src/tsdb/inc/tsdbReadImpl.h +++ b/source/dnode/vnode/tsdb/inc/tsdbReadImpl.h @@ -15,6 +15,7 @@ #ifndef _TD_TSDB_READ_IMPL_H_ #define _TD_TSDB_READ_IMPL_H_ +#if 0 #include "tfs.h" #include "tsdb.h" @@ -150,4 +151,6 @@ static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) { return 0; } +#endif + #endif /*_TD_TSDB_READ_IMPL_H_*/ diff --git a/src/tsdb/inc/tsdbRowMergeBuf.h b/source/dnode/vnode/tsdb/inc/tsdbRowMergeBuf.h similarity index 98% rename from src/tsdb/inc/tsdbRowMergeBuf.h rename to source/dnode/vnode/tsdb/inc/tsdbRowMergeBuf.h index cefa9b27fbd2a6116e2d7180f102440af9595e93..1531d532b9a2ba410747e833a8881289b5dd1370 100644 --- a/src/tsdb/inc/tsdbRowMergeBuf.h +++ b/source/dnode/vnode/tsdb/inc/tsdbRowMergeBuf.h @@ -16,6 +16,8 @@ #ifndef TSDB_ROW_MERGE_BUF_H #define TSDB_ROW_MERGE_BUF_H +#if 0 + #ifdef __cplusplus extern "C" { #endif @@ -42,4 +44,6 @@ static FORCE_INLINE void tsdbFreeMergeBuf(SMergeBuf buf) { } #endif +#endif + #endif /* ifndef TSDB_ROW_MERGE_BUF_H */ diff --git a/source/dnode/vnode/tsdb/inc/tsdbSMA.h b/source/dnode/vnode/tsdb/inc/tsdbSMA.h deleted file mode 100644 index 800a276bc85c1cc97cc84b9deb214bb66c154a0b..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/tsdb/inc/tsdbSMA.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_SMA_H_ -#define _TD_TSDB_SMA_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TSDB_SMA_H_*/ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbint.h b/source/dnode/vnode/tsdb/inc/tsdbint.h similarity index 99% rename from src/tsdb/inc/tsdbint.h rename to source/dnode/vnode/tsdb/inc/tsdbint.h index 2354269c6ee861632ff0d88c2efe52470a689a48..bdd1bd6f71f25e6c566ea3f7992d8b9197689855 100644 --- a/src/tsdb/inc/tsdbint.h +++ b/source/dnode/vnode/tsdb/inc/tsdbint.h @@ -16,6 +16,7 @@ #ifndef _TD_TSDB_INT_H_ #define _TD_TSDB_INT_H_ +#if 0 // // TODO: remove the include // #include // #include @@ -144,4 +145,5 @@ static FORCE_INLINE int tsdbGetNextMaxTables(int tid) { } #endif +#endif #endif /* _TD_TSDB_INT_H_ */ diff --git a/source/dnode/vnode/tsdb/src/tsdbCommit.c b/source/dnode/vnode/tsdb/src/tsdbCommit.c index a747c7333eabaabc17160943a7288b62927d34a8..1247dcd728117ee834ea28b951b5ebf7d4e29940 100644 --- a/source/dnode/vnode/tsdb/src/tsdbCommit.c +++ b/source/dnode/vnode/tsdb/src/tsdbCommit.c @@ -30,4 +30,1673 @@ int tsdbCommit(STsdb *pTsdb) { pTsdb->imem = NULL; // tsem_post(&(pTsdb->canCommit)); return 0; -} \ No newline at end of file +} + +#if 0 +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "tsdbint.h" + +extern int32_t tsTsdbMetaCompactRatio; + +#define TSDB_MAX_SUBBLOCKS 8 +static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { + if (key < 0) { + return (int)((key + 1) / tsTickPerDay[precision] / days - 1); + } else { + return (int)((key / tsTickPerDay[precision] / days)); + } +} + +typedef struct { + SRtn rtn; // retention snapshot + SFSIter fsIter; // tsdb file iterator + int niters; // memory iterators + SCommitIter *iters; + bool isRFileSet; // read and commit FSET + SReadH readh; + SDFileSet wSet; + bool isDFileSame; + bool isLFileSame; + TSKEY minKey; + TSKEY maxKey; + SArray * aBlkIdx; // SBlockIdx array + STable * pTable; + SArray * aSupBlk; // Table super-block array + SArray * aSubBlk; // table sub-block array + SDataCols * pDataCols; +} SCommitH; + +#define TSDB_COMMIT_REPO(ch) TSDB_READ_REPO(&(ch->readh)) +#define TSDB_COMMIT_REPO_ID(ch) REPO_ID(TSDB_READ_REPO(&(ch->readh))) +#define TSDB_COMMIT_WRITE_FSET(ch) (&((ch)->wSet)) +#define TSDB_COMMIT_TABLE(ch) ((ch)->pTable) +#define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD) +#define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA) +#define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST) +#define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh)) +#define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh)) +#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock) +#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) + +static int tsdbCommitMeta(STsdbRepo *pRepo); +static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact); +static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); +static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile); +static int tsdbCommitTSData(STsdbRepo *pRepo); +static void tsdbStartCommit(STsdbRepo *pRepo); +static void tsdbEndCommit(STsdbRepo *pRepo, int eno); +static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); +static int tsdbCreateCommitIters(SCommitH *pCommith); +static void tsdbDestroyCommitIters(SCommitH *pCommith); +static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key); +static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo); +static void tsdbDestroyCommitH(SCommitH *pCommith); +static int tsdbGetFidLevel(int fid, SRtn *pRtn); +static int tsdbNextCommitFid(SCommitH *pCommith); +static int tsdbCommitToTable(SCommitH *pCommith, int tid); +static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable); +static int tsdbComparKeyBlock(const void *arg1, const void *arg2); +static int tsdbWriteBlockInfo(SCommitH *pCommih); +static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData); +static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx); +static int tsdbMoveBlock(SCommitH *pCommith, int bidx); +static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks); +static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, + bool isLastOneBlock); +static void tsdbResetCommitFile(SCommitH *pCommith); +static void tsdbResetCommitTable(SCommitH *pCommith); +static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); +static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError); +static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo); +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, + TSKEY maxKey, int maxRows, int8_t update); + +void *tsdbCommitData(STsdbRepo *pRepo) { + if (pRepo->imem == NULL) { + return NULL; + } + tsdbStartCommit(pRepo); + + // Commit to update meta file + if (tsdbCommitMeta(pRepo) < 0) { + tsdbError("vgId:%d error occurs while committing META data since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + // Create the iterator to read from cache + if (tsdbCommitTSData(pRepo) < 0) { + tsdbError("vgId:%d error occurs while committing TS data since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS); + return NULL; + +_err: + ASSERT(terrno != TSDB_CODE_SUCCESS); + pRepo->code = terrno; + + tsdbEndCommit(pRepo, terrno); + return NULL; +} + +int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) { + SDiskID did; + SDFileSet nSet; + STsdbFS * pfs = REPO_FS(pRepo); + int level; + + ASSERT(pSet->fid >= pRtn->minFid); + + level = tsdbGetFidLevel(pSet->fid, pRtn); + + tfsAllocDisk(level, &(did.level), &(did.id)); + if (did.level == TFS_UNDECIDED_LEVEL) { + terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + return -1; + } + + if (did.level > TSDB_FSET_LEVEL(pSet)) { + // Need to move the FSET to higher level + tsdbInitDFileSet(&nSet, did, REPO_ID(pRepo), pSet->fid, FS_TXN_VERSION(pfs)); + + if (tsdbCopyDFileSet(pSet, &nSet) < 0) { + tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno)); + return -1; + } + + if (tsdbUpdateDFileSet(pfs, &nSet) < 0) { + return -1; + } + + tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id); + } else { + // On a correct level + if (tsdbUpdateDFileSet(pfs, pSet) < 0) { + return -1; + } + } + + return 0; +} + +int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, + SBlockIdx *pIdx) { + size_t nSupBlocks; + size_t nSubBlocks; + uint32_t tlen; + SBlockInfo *pBlkInfo; + int64_t offset; + SBlock * pBlock; + + memset(pIdx, 0, sizeof(*pIdx)); + + nSupBlocks = taosArrayGetSize(pSupA); + nSubBlocks = (pSubA == NULL) ? 0 : taosArrayGetSize(pSubA); + + if (nSupBlocks <= 0) { + // No data (data all deleted) + return 0; + } + + tlen = (uint32_t)(sizeof(SBlockInfo) + sizeof(SBlock) * (nSupBlocks + nSubBlocks) + sizeof(TSCKSUM)); + if (tsdbMakeRoom(ppBuf, tlen) < 0) return -1; + pBlkInfo = *ppBuf; + + pBlkInfo->delimiter = TSDB_FILE_DELIMITER; + pBlkInfo->tid = TABLE_TID(pTable); + pBlkInfo->uid = TABLE_UID(pTable); + + memcpy((void *)(pBlkInfo->blocks), taosArrayGet(pSupA, 0), nSupBlocks * sizeof(SBlock)); + if (nSubBlocks > 0) { + memcpy((void *)(pBlkInfo->blocks + nSupBlocks), taosArrayGet(pSubA, 0), nSubBlocks * sizeof(SBlock)); + + for (int i = 0; i < nSupBlocks; i++) { + pBlock = pBlkInfo->blocks + i; + + if (pBlock->numOfSubBlocks > 1) { + pBlock->offset += (sizeof(SBlockInfo) + sizeof(SBlock) * nSupBlocks); + } + } + } + + taosCalcChecksumAppend(0, (uint8_t *)pBlkInfo, tlen); + + if (tsdbAppendDFile(pHeadf, (void *)pBlkInfo, tlen, &offset) < 0) { + return -1; + } + + tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(pBlkInfo, tlen - sizeof(TSCKSUM))); + + // Set pIdx + pBlock = taosArrayGetLast(pSupA); + + pIdx->tid = TABLE_TID(pTable); + pIdx->uid = TABLE_UID(pTable); + pIdx->hasLast = pBlock->last ? 1 : 0; + pIdx->maxKey = pBlock->keyLast; + pIdx->numOfBlocks = (uint32_t)nSupBlocks; + pIdx->len = tlen; + pIdx->offset = (uint32_t)offset; + + return 0; +} + +int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { + SBlockIdx *pBlkIdx; + size_t nidx = taosArrayGetSize(pIdxA); + int tlen = 0, size; + int64_t offset; + + if (nidx <= 0) { + // All data are deleted + pHeadf->info.offset = 0; + pHeadf->info.len = 0; + return 0; + } + + for (size_t i = 0; i < nidx; i++) { + pBlkIdx = (SBlockIdx *)taosArrayGet(pIdxA, i); + + size = tsdbEncodeSBlockIdx(NULL, pBlkIdx); + if (tsdbMakeRoom(ppBuf, tlen + size) < 0) return -1; + + void *ptr = POINTER_SHIFT(*ppBuf, tlen); + tsdbEncodeSBlockIdx(&ptr, pBlkIdx); + + tlen += size; + } + + tlen += sizeof(TSCKSUM); + if (tsdbMakeRoom(ppBuf, tlen) < 0) return -1; + taosCalcChecksumAppend(0, (uint8_t *)(*ppBuf), tlen); + + if (tsdbAppendDFile(pHeadf, *ppBuf, tlen, &offset) < tlen) { + return -1; + } + + tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(*ppBuf, tlen - sizeof(TSCKSUM))); + pHeadf->info.offset = (uint32_t)offset; + pHeadf->info.len = tlen; + + return 0; +} + + +// =================== Commit Meta Data +static int tsdbInitCommitMetaFile(STsdbRepo *pRepo, SMFile* pMf, bool open) { + STsdbFS * pfs = REPO_FS(pRepo); + SMFile * pOMFile = pfs->cstatus->pmf; + SDiskID did; + + // Create/Open a meta file or open the existing file + if (pOMFile == NULL) { + // Create a new meta file + did.level = TFS_PRIMARY_LEVEL; + did.id = TFS_PRIMARY_ID; + tsdbInitMFile(pMf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); + + if (open && tsdbCreateMFile(pMf, true) < 0) { + tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMf)); + } else { + tsdbInitMFileEx(pMf, pOMFile); + if (open && tsdbOpenMFile(pMf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } + + return 0; +} + +static int tsdbCommitMeta(STsdbRepo *pRepo) { + STsdbFS * pfs = REPO_FS(pRepo); + SMemTable *pMem = pRepo->imem; + SMFile * pOMFile = pfs->cstatus->pmf; + SMFile mf; + SActObj * pAct = NULL; + SActCont * pCont = NULL; + SListNode *pNode = NULL; + + ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0); + + if (listNEles(pMem->actList) <= 0) { + // no meta data to commit, just keep the old meta file + tsdbUpdateMFile(pfs, pOMFile); + if (tsTsdbMetaCompactRatio > 0) { + if (tsdbInitCommitMetaFile(pRepo, &mf, false) < 0) { + return -1; + } + int ret = tsdbCompactMetaFile(pRepo, pfs, &mf); + if (ret < 0) tsdbError("compact meta file error"); + + return ret; + } + return 0; + } else { + if (tsdbInitCommitMetaFile(pRepo, &mf, true) < 0) { + return -1; + } + } + + // Loop to write + while ((pNode = tdListPopHead(pMem->actList)) != NULL) { + pAct = (SActObj *)pNode->data; + if (pAct->act == TSDB_UPDATE_META) { + pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); + if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len, false) < 0) { + tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, + tstrerror(terrno)); + tsdbCloseMFile(&mf); + (void)tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache + return -1; + } + } else if (pAct->act == TSDB_DROP_META) { + if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) { + tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, + tstrerror(terrno)); + tsdbCloseMFile(&mf); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache + return -1; + } + } else { + ASSERT(false); + } + } + + if (tsdbUpdateMFileHeader(&mf) < 0) { + tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno)); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache + return -1; + } + + TSDB_FILE_FSYNC(&mf); + tsdbCloseMFile(&mf); + tsdbUpdateMFile(pfs, &mf); + + if (tsTsdbMetaCompactRatio > 0 && tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) { + tsdbError("compact meta file error"); + } + + return 0; +} + +int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord) { + int tlen = 0; + tlen += taosEncodeFixedU64(buf, pRecord->uid); + tlen += taosEncodeFixedI64(buf, pRecord->offset); + tlen += taosEncodeFixedI64(buf, pRecord->size); + + return tlen; +} + +void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord) { + buf = taosDecodeFixedU64(buf, &(pRecord->uid)); + buf = taosDecodeFixedI64(buf, &(pRecord->offset)); + buf = taosDecodeFixedI64(buf, &(pRecord->size)); + + return buf; +} + +void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) { + STsdbCfg *pCfg = REPO_CFG(pRepo); + TSKEY minKey, midKey, maxKey, now; + + now = taosGetTimestamp(pCfg->precision); + minKey = now - pCfg->keep * tsTickPerDay[pCfg->precision]; + midKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision]; + maxKey = now - pCfg->keep1 * tsTickPerDay[pCfg->precision]; + + pRtn->minKey = minKey; + pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision)); + pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision)); + pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision)); + tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey, + pRtn->minFid, pRtn->midFid, pRtn->maxFid); +} + +static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact) { + char buf[64] = "\0"; + void * pBuf = buf; + SKVRecord rInfo; + int64_t offset; + + // Seek to end of meta file + offset = tsdbSeekMFile(pMFile, 0, SEEK_END); + if (offset < 0) { + return -1; + } + + rInfo.offset = offset; + rInfo.uid = uid; + rInfo.size = contLen; + + int tlen = tsdbEncodeKVRecord((void **)(&pBuf), &rInfo); + if (tsdbAppendMFile(pMFile, buf, tlen, NULL) < tlen) { + return -1; + } + + if (tsdbAppendMFile(pMFile, cont, contLen, NULL) < contLen) { + return -1; + } + + tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM))); + + SHashObj* cache = compact ? pfs->metaCacheComp : pfs->metaCache; + + pMFile->info.nRecords++; + + SKVRecord *pRecord = taosHashGet(cache, (void *)&uid, sizeof(uid)); + if (pRecord != NULL) { + pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord)); + } else { + pMFile->info.nRecords++; + } + taosHashPut(cache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); + + return 0; +} + +static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { + SKVRecord rInfo = {0}; + char buf[128] = "\0"; + + SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid)); + if (pRecord == NULL) { + tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid); + return -1; + } + + rInfo.offset = -pRecord->offset; + rInfo.uid = pRecord->uid; + rInfo.size = pRecord->size; + + void *pBuf = buf; + tsdbEncodeKVRecord(&pBuf, &rInfo); + + if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) { + return -1; + } + + pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord)); + pMFile->info.nDels++; + pMFile->info.nRecords--; + pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); + + taosHashRemove(pfs->metaCache, (void *)(&uid), sizeof(uid)); + return 0; +} + +static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { + float delPercent = (float)(pMFile->info.nDels) / (float)(pMFile->info.nRecords); + float tombPercent = (float)(pMFile->info.tombSize) / (float)(pMFile->info.size); + float compactRatio = (float)(tsTsdbMetaCompactRatio)/100; + + if (delPercent < compactRatio && tombPercent < compactRatio) { + return 0; + } + + if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) { + tsdbError("open meta file %s compact fail", pMFile->f.rname); + return -1; + } + + tsdbInfo("begin compact tsdb meta file, ratio:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 ",size:%" PRId64, + tsTsdbMetaCompactRatio, pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size); + + SMFile mf; + SDiskID did; + + // first create tmp meta file + did.level = TFS_PRIMARY_LEVEL; + did.id = TFS_PRIMARY_ID; + tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)) + 1); + + if (tsdbCreateMFile(&mf, true) < 0) { + tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d meta file %s is created to compact meta data", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); + + // second iterator metaCache + int code = -1; + int64_t maxBufSize = 1024; + SKVRecord *pRecord; + void *pBuf = NULL; + + pBuf = malloc((size_t)maxBufSize); + if (pBuf == NULL) { + goto _err; + } + + // init Comp + assert(pfs->metaCacheComp == NULL); + pfs->metaCacheComp = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (pfs->metaCacheComp == NULL) { + goto _err; + } + + pRecord = taosHashIterate(pfs->metaCache, NULL); + while (pRecord) { + if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { + tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), + tstrerror(terrno)); + goto _err; + } + if (pRecord->size > maxBufSize) { + maxBufSize = pRecord->size; + void* tmp = realloc(pBuf, (size_t)maxBufSize); + if (tmp == NULL) { + goto _err; + } + pBuf = tmp; + } + int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size); + if (nread < 0) { + tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), + tstrerror(terrno)); + goto _err; + } + + if (nread < pRecord->size) { + tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", + REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); + goto _err; + } + + if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, true) < 0) { + tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid, + tstrerror(terrno)); + goto _err; + } + + pRecord = taosHashIterate(pfs->metaCache, pRecord); + } + code = 0; + +_err: + if (code == 0) TSDB_FILE_FSYNC(&mf); + tsdbCloseMFile(&mf); + tsdbCloseMFile(pMFile); + + if (code == 0) { + // rename meta.tmp -> meta + tsdbInfo("vgId:%d meta file rename %s -> %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf), TSDB_FILE_FULL_NAME(pMFile)); + taosRename(mf.f.aname,pMFile->f.aname); + tstrncpy(mf.f.aname, pMFile->f.aname, TSDB_FILENAME_LEN); + tstrncpy(mf.f.rname, pMFile->f.rname, TSDB_FILENAME_LEN); + // update current meta file info + pfs->nstatus->pmf = NULL; + tsdbUpdateMFile(pfs, &mf); + + taosHashCleanup(pfs->metaCache); + pfs->metaCache = pfs->metaCacheComp; + pfs->metaCacheComp = NULL; + } else { + // remove meta.tmp file + remove(mf.f.aname); + taosHashCleanup(pfs->metaCacheComp); + pfs->metaCacheComp = NULL; + } + + tfree(pBuf); + + ASSERT(mf.info.nDels == 0); + ASSERT(mf.info.tombSize == 0); + + tsdbInfo("end compact tsdb meta file,code:%d,nRecords:%" PRId64 ",size:%" PRId64, + code,mf.info.nRecords,mf.info.size); + return code; +} + +// =================== Commit Time-Series Data +static int tsdbCommitTSData(STsdbRepo *pRepo) { + SMemTable *pMem = pRepo->imem; + SCommitH commith; + SDFileSet *pSet = NULL; + int fid; + + memset(&commith, 0, sizeof(commith)); + + if (pMem->numOfRows <= 0) { + // No memory data, just apply retention on each file on disk + if (tsdbApplyRtn(pRepo) < 0) { + return -1; + } + return 0; + } + + // Resource initialization + if (tsdbInitCommitH(&commith, pRepo) < 0) { + return -1; + } + + // Skip expired memory data and expired FSET + tsdbSeekCommitIter(&commith, commith.rtn.minKey); + while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { + if (pSet->fid < commith.rtn.minFid) { + tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + } else { + break; + } + } + + // Loop to commit to each file + fid = tsdbNextCommitFid(&(commith)); + while (true) { + // Loop over both on disk and memory + if (pSet == NULL && fid == TSDB_IVLD_FID) break; + + if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) { + // Only has existing FSET but no memory data to commit in this + // existing FSET, only check if file in correct retention + if (tsdbApplyRtnOnFSet(pRepo, pSet, &(commith.rtn)) < 0) { + tsdbDestroyCommitH(&commith); + return -1; + } + + pSet = tsdbFSIterNext(&(commith.fsIter)); + } else { + // Has memory data to commit + SDFileSet *pCSet; + int cfid; + + if (pSet == NULL || pSet->fid > fid) { + // Commit to a new FSET with fid: fid + pCSet = NULL; + cfid = fid; + } else { + // Commit to an existing FSET + pCSet = pSet; + cfid = pSet->fid; + pSet = tsdbFSIterNext(&(commith.fsIter)); + } + + if (tsdbCommitToFile(&commith, pCSet, cfid) < 0) { + tsdbDestroyCommitH(&commith); + return -1; + } + + fid = tsdbNextCommitFid(&commith); + } + } + + tsdbDestroyCommitH(&commith); + return 0; +} + +static void tsdbStartCommit(STsdbRepo *pRepo) { + SMemTable *pMem = pRepo->imem; + + ASSERT(pMem->numOfRows > 0 || listNEles(pMem->actList) > 0); + + tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64 " meta rows: %d", + REPO_ID(pRepo), pMem->keyFirst, pMem->keyLast, pMem->numOfRows, listNEles(pMem->actList)); + + tsdbStartFSTxn(pRepo, pMem->pointsAdd, pMem->storageAdd); + + pRepo->code = TSDB_CODE_SUCCESS; +} + +static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { + if (eno != TSDB_CODE_SUCCESS) { + tsdbEndFSTxnWithError(REPO_FS(pRepo)); + } else { + tsdbEndFSTxn(pRepo); + } + + tsdbInfo("vgId:%d commit over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); + + if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno); + + SMemTable *pIMem = pRepo->imem; + (void)tsdbLockRepo(pRepo); + pRepo->imem = NULL; + (void)tsdbUnlockRepo(pRepo); + tsdbUnRefMemTable(pRepo, pIMem); + tsem_post(&(pRepo->readyToCommit)); +} + +#if 0 +static bool tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { + for (int i = 0; i < nIters; i++) { + TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter); + if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return true; + } + return false; +} +#endif + +static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + + ASSERT(pSet == NULL || pSet->fid == fid); + + tsdbResetCommitFile(pCommith); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &(pCommith->minKey), &(pCommith->maxKey)); + + // Set and open files + if (tsdbSetAndOpenCommitFile(pCommith, pSet, fid) < 0) { + return -1; + } + + // Loop to commit each table data + for (int tid = 1; tid < pCommith->niters; tid++) { + SCommitIter *pIter = pCommith->iters + tid; + + if (pIter->pTable == NULL) continue; + + if (tsdbCommitToTable(pCommith, tid) < 0) { + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + } + + if (tsdbWriteBlockIdx(TSDB_COMMIT_HEAD_FILE(pCommith), pCommith->aBlkIdx, (void **)(&(TSDB_COMMIT_BUF(pCommith)))) < + 0) { + tsdbError("vgId:%d failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + + if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) { + tsdbError("vgId:%d failed to update FSET %d header since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + + // Close commit file + tsdbCloseCommitFile(pCommith, false); + + if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) { + return -1; + } + + return 0; +} + +static int tsdbCreateCommitIters(SCommitH *pCommith) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + SMemTable *pMem = pRepo->imem; + STsdbMeta *pMeta = pRepo->tsdbMeta; + + pCommith->niters = pMem->maxTables; + pCommith->iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter)); + if (pCommith->iters == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (tsdbRLockRepoMeta(pRepo) < 0) return -1; + + // reference all tables + for (int i = 0; i < pMem->maxTables; i++) { + if (pMeta->tables[i] != NULL) { + tsdbRefTable(pMeta->tables[i]); + pCommith->iters[i].pTable = pMeta->tables[i]; + } + } + + if (tsdbUnlockRepoMeta(pRepo) < 0) return -1; + + for (int i = 0; i < pMem->maxTables; i++) { + if ((pCommith->iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && + (TABLE_UID(pCommith->iters[i].pTable) == pMem->tData[i]->uid)) { + if ((pCommith->iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + tSkipListIterNext(pCommith->iters[i].pIter); + } + } + + return 0; +} + +static void tsdbDestroyCommitIters(SCommitH *pCommith) { + if (pCommith->iters == NULL) return; + + for (int i = 1; i < pCommith->niters; i++) { + if (pCommith->iters[i].pTable != NULL) { + tsdbUnRefTable(pCommith->iters[i].pTable); + tSkipListDestroyIter(pCommith->iters[i].pIter); + } + } + + free(pCommith->iters); + pCommith->iters = NULL; + pCommith->niters = 0; +} + +// Skip all keys until key (not included) +static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key) { + for (int i = 0; i < pCommith->niters; i++) { + SCommitIter *pIter = pCommith->iters + i; + if (pIter->pTable == NULL || pIter->pIter == NULL) continue; + + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, key - 1, INT32_MAX, NULL, NULL, 0, true, NULL); + } +} + +static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo) { + STsdbCfg *pCfg = REPO_CFG(pRepo); + + memset(pCommith, 0, sizeof(*pCommith)); + tsdbGetRtnSnap(pRepo, &(pCommith->rtn)); + + TSDB_FSET_SET_CLOSED(TSDB_COMMIT_WRITE_FSET(pCommith)); + + // Init read handle + if (tsdbInitReadH(&(pCommith->readh), pRepo) < 0) { + return -1; + } + + // Init file iterator + tsdbFSIterInit(&(pCommith->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD); + + if (tsdbCreateCommitIters(pCommith) < 0) { + tsdbDestroyCommitH(pCommith); + return -1; + } + + pCommith->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx)); + if (pCommith->aBlkIdx == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; + } + + pCommith->aSupBlk = taosArrayInit(1024, sizeof(SBlock)); + if (pCommith->aSupBlk == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; + } + + pCommith->aSubBlk = taosArrayInit(1024, sizeof(SBlock)); + if (pCommith->aSubBlk == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; + } + + pCommith->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); + if (pCommith->pDataCols == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; + } + + return 0; +} + +static void tsdbDestroyCommitH(SCommitH *pCommith) { + pCommith->pDataCols = tdFreeDataCols(pCommith->pDataCols); + pCommith->aSubBlk = taosArrayDestroy(pCommith->aSubBlk); + pCommith->aSupBlk = taosArrayDestroy(pCommith->aSupBlk); + pCommith->aBlkIdx = taosArrayDestroy(pCommith->aBlkIdx); + tsdbDestroyCommitIters(pCommith); + tsdbDestroyReadH(&(pCommith->readh)); + tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); +} + +static int tsdbNextCommitFid(SCommitH *pCommith) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + int fid = TSDB_IVLD_FID; + + for (int i = 0; i < pCommith->niters; i++) { + SCommitIter *pIter = pCommith->iters + i; + if (pIter->pTable == NULL || pIter->pIter == NULL) continue; + + TSKEY nextKey = tsdbNextIterKey(pIter->pIter); + if (nextKey == TSDB_DATA_TIMESTAMP_NULL) { + continue; + } else { + int tfid = (int)(TSDB_KEY_FID(nextKey, pCfg->daysPerFile, pCfg->precision)); + if (fid == TSDB_IVLD_FID || fid > tfid) { + fid = tfid; + } + } + } + + return fid; +} + +static int tsdbCommitToTable(SCommitH *pCommith, int tid) { + SCommitIter *pIter = pCommith->iters + tid; + TSKEY nextKey = tsdbNextIterKey(pIter->pIter); + + tsdbResetCommitTable(pCommith); + + TSDB_RLOCK_TABLE(pIter->pTable); + + // Set commit table + if (tsdbSetCommitTable(pCommith, pIter->pTable) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + + // No disk data and no memory data, just return + if (pCommith->readh.pBlkIdx == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return 0; + } + + // Must has disk data or has memory data + int nBlocks; + int bidx = 0; + SBlock *pBlock; + + if (pCommith->readh.pBlkIdx) { + if (tsdbLoadBlockInfo(&(pCommith->readh), NULL) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + + nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; + } else { + nBlocks = 0; + } + + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + + while (true) { + if (pBlock == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) break; + + if ((nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey) || + (pBlock && (!pBlock->last) && tsdbComparKeyBlock((void *)(&nextKey), pBlock) > 0)) { + if (tsdbMoveBlock(pCommith, bidx) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + + bidx++; + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + } else if (pBlock && (pBlock->last || tsdbComparKeyBlock((void *)(&nextKey), pBlock) == 0)) { + // merge pBlock data and memory data + if (tsdbMergeMemData(pCommith, pIter, bidx) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + + bidx++; + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + nextKey = tsdbNextIterKey(pIter->pIter); + } else { + // Only commit memory data + if (pBlock == NULL) { + if (tsdbCommitMemData(pCommith, pIter, pCommith->maxKey, false) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + } else { + if (tsdbCommitMemData(pCommith, pIter, pBlock->keyFirst - 1, true) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + } + nextKey = tsdbNextIterKey(pIter->pIter); + } + } + + TSDB_RUNLOCK_TABLE(pIter->pTable); + + if (tsdbWriteBlockInfo(pCommith) < 0) { + tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), + TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); + return -1; + } + + return 0; +} + +static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) { + STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); + + pCommith->pTable = pTable; + + if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (pCommith->isRFileSet) { + if (tsdbSetReadTable(&(pCommith->readh), pTable) < 0) { + return -1; + } + } else { + pCommith->readh.pBlkIdx = NULL; + } + return 0; +} + +static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { + TSKEY key = *(TSKEY *)arg1; + SBlock *pBlock = (SBlock *)arg2; + + if (key < pBlock->keyFirst) { + return -1; + } else if (key > pBlock->keyLast) { + return 1; + } else { + return 0; + } +} + +int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, + bool isLast, bool isSuper, void **ppBuf, void **ppCBuf) { + STsdbCfg * pCfg = REPO_CFG(pRepo); + SBlockData *pBlockData; + int64_t offset = 0; + int rowsToWrite = pDataCols->numOfRows; + + ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock); + ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock); + + // Make buffer space + if (tsdbMakeRoom(ppBuf, TSDB_BLOCK_STATIS_SIZE(pDataCols->numOfCols)) < 0) { + return -1; + } + pBlockData = (SBlockData *)(*ppBuf); + + // Get # of cols not all NULL(not including key column) + int nColsNotAllNull = 0; + for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column + SDataCol * pDataCol = pDataCols->cols + ncol; + SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; + + if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it + continue; + } + + memset(pBlockCol, 0, sizeof(*pBlockCol)); + + pBlockCol->colId = pDataCol->colId; + pBlockCol->type = pDataCol->type; + if (tDataTypes[pDataCol->type].statisFunc) { + (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), + &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), + &(pBlockCol->numOfNull)); + } + nColsNotAllNull++; + } + + ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols); + + // Compress the data if neccessary + int tcol = 0; // counter of not all NULL and written columns + uint32_t toffset = 0; + int32_t tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull); + int32_t lsize = tsize; + int32_t keyLen = 0; + for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { + // All not NULL columns finish + if (ncol != 0 && tcol >= nColsNotAllNull) break; + + SDataCol * pDataCol = pDataCols->cols + ncol; + SBlockCol *pBlockCol = pBlockData->cols + tcol; + + if (ncol != 0 && (pDataCol->colId != pBlockCol->colId)) continue; + + int32_t flen; // final length + int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite); + void * tptr; + + // Make room + if (tsdbMakeRoom(ppBuf, lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { + return -1; + } + pBlockData = (SBlockData *)(*ppBuf); + pBlockCol = pBlockData->cols + tcol; + tptr = POINTER_SHIFT(pBlockData, lsize); + + if (pCfg->compression == TWO_STAGE_COMP && + tsdbMakeRoom(ppCBuf, tlen + COMP_OVERFLOW_BYTES) < 0) { + return -1; + } + + // Compress or just copy + if (pCfg->compression) { + flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, + tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, + tlen + COMP_OVERFLOW_BYTES); + } else { + flen = tlen; + memcpy(tptr, pDataCol->pData, flen); + } + + // Add checksum + ASSERT(flen > 0); + flen += sizeof(TSCKSUM); + taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); + tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM))); + + if (ncol != 0) { + tsdbSetBlockColOffset(pBlockCol, toffset); + pBlockCol->len = flen; + tcol++; + } else { + keyLen = flen; + } + + toffset += flen; + lsize += flen; + } + + pBlockData->delimiter = TSDB_FILE_DELIMITER; + pBlockData->uid = TABLE_UID(pTable); + pBlockData->numOfCols = nColsNotAllNull; + + taosCalcChecksumAppend(0, (uint8_t *)pBlockData, tsize); + tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(pBlockData, tsize - sizeof(TSCKSUM))); + + // Write the whole block to file + if (tsdbAppendDFile(pDFile, (void *)pBlockData, lsize, &offset) < lsize) { + return -1; + } + + // Update pBlock membership vairables + pBlock->last = isLast; + pBlock->offset = offset; + pBlock->algorithm = pCfg->compression; + pBlock->numOfRows = rowsToWrite; + pBlock->len = lsize; + pBlock->keyLen = keyLen; + pBlock->numOfSubBlocks = isSuper ? 1 : 0; + pBlock->numOfCols = nColsNotAllNull; + pBlock->keyFirst = dataColsKeyFirst(pDataCols); + pBlock->keyLast = dataColsKeyLast(pDataCols); + + tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 + " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, + REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, + pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); + + return 0; +} + +static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, + bool isSuper) { + return tsdbWriteBlockImpl(TSDB_COMMIT_REPO(pCommith), TSDB_COMMIT_TABLE(pCommith), pDFile, pDataCols, pBlock, isLast, + isSuper, (void **)(&(TSDB_COMMIT_BUF(pCommith))), + (void **)(&(TSDB_COMMIT_COMP_BUF(pCommith)))); +} + + +static int tsdbWriteBlockInfo(SCommitH *pCommih) { + SDFile * pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); + SBlockIdx blkIdx; + STable * pTable = TSDB_COMMIT_TABLE(pCommih); + + if (tsdbWriteBlockInfoImpl(pHeadf, pTable, pCommih->aSupBlk, pCommih->aSubBlk, (void **)(&(TSDB_COMMIT_BUF(pCommih))), + &blkIdx) < 0) { + return -1; + } + + if (blkIdx.numOfBlocks == 0) { + return 0; + } + + if (taosArrayPush(pCommih->aBlkIdx, (void *)(&blkIdx)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + SMergeInfo mInfo; + int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); + SDFile * pDFile; + bool isLast; + SBlock block; + + while (true) { + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, defaultRows, pCommith->pDataCols, NULL, 0, + pCfg->update, &mInfo); + + if (pCommith->pDataCols->numOfRows <= 0) break; + + if (toData || pCommith->pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; + } else { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isLast = true; + } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; + + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) { + return -1; + } + } + + return 0; +} + +static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + int nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; + SBlock * pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + TSKEY keyLimit; + int16_t colId = 0; + SMergeInfo mInfo; + SBlock subBlocks[TSDB_MAX_SUBBLOCKS]; + SBlock block, supBlock; + SDFile * pDFile; + + if (bidx == nBlocks - 1) { + keyLimit = pCommith->maxKey; + } else { + keyLimit = pBlock[1].keyFirst - 1; + } + + SSkipListIterator titer = *(pIter->pIter); + if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1) < 0) return -1; + + tsdbLoadDataFromCache(pIter->pTable, &titer, keyLimit, INT32_MAX, NULL, pCommith->readh.pDCols[0]->cols[0].pData, + pCommith->readh.pDCols[0]->numOfRows, pCfg->update, &mInfo); + + if (mInfo.nOperations == 0) { + // no new data to insert (all updates denied) + if (tsdbMoveBlock(pCommith, bidx) < 0) { + return -1; + } + *(pIter->pIter) = titer; + } else if (pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed == 0) { + // Ignore the block + ASSERT(0); + *(pIter->pIter) = titer; + } else if (tsdbCanAddSubBlock(pCommith, pBlock, &mInfo)) { + // Add a sub-block + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, INT32_MAX, pCommith->pDataCols, + pCommith->readh.pDCols[0]->cols[0].pData, pCommith->readh.pDCols[0]->numOfRows, pCfg->update, + &mInfo); + if (pBlock->last) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, pBlock->last, false) < 0) return -1; + + if (pBlock->numOfSubBlocks == 1) { + subBlocks[0] = *pBlock; + subBlocks[0].numOfSubBlocks = 0; + } else { + memcpy(subBlocks, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), + sizeof(SBlock) * pBlock->numOfSubBlocks); + } + subBlocks[pBlock->numOfSubBlocks] = block; + supBlock = *pBlock; + supBlock.keyFirst = mInfo.keyFirst; + supBlock.keyLast = mInfo.keyLast; + supBlock.numOfSubBlocks++; + supBlock.numOfRows = pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed; + supBlock.offset = taosArrayGetSize(pCommith->aSubBlk) * sizeof(SBlock); + + if (tsdbCommitAddBlock(pCommith, &supBlock, subBlocks, supBlock.numOfSubBlocks) < 0) return -1; + } else { + if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; + if (tsdbMergeBlockData(pCommith, pIter, pCommith->readh.pDCols[0], keyLimit, bidx == (nBlocks - 1)) < 0) return -1; + } + + return 0; +} + +static int tsdbMoveBlock(SCommitH *pCommith, int bidx) { + SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + SDFile *pDFile; + SBlock block; + bool isSameFile; + + ASSERT(pBlock->numOfSubBlocks > 0); + + if (pBlock->last) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isSameFile = pCommith->isLFileSame; + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isSameFile = pCommith->isDFileSame; + } + + if (isSameFile) { + if (pBlock->numOfSubBlocks == 1) { + if (tsdbCommitAddBlock(pCommith, pBlock, NULL, 0) < 0) { + return -1; + } + } else { + block = *pBlock; + block.offset = sizeof(SBlock) * taosArrayGetSize(pCommith->aSubBlk); + + if (tsdbCommitAddBlock(pCommith, &block, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), + pBlock->numOfSubBlocks) < 0) { + return -1; + } + } + } else { + if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; + if (tsdbWriteBlock(pCommith, pDFile, pCommith->readh.pDCols[0], &block, pBlock->last, true) < 0) return -1; + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; + } + + return 0; +} + +static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks) { + if (taosArrayPush(pCommith->aSupBlk, pSupBlock) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (pSubBlocks && taosArrayAddBatch(pCommith->aSubBlk, pSubBlocks, nSubBlocks) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, bool isLastOneBlock) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + SBlock block; + SDFile * pDFile; + bool isLast; + int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); + + int biter = 0; + while (true) { + tsdbLoadAndMergeFromCache(pCommith->readh.pDCols[0], &biter, pIter, pCommith->pDataCols, keyLimit, defaultRows, + pCfg->update); + + if (pCommith->pDataCols->numOfRows == 0) break; + + if (isLastOneBlock) { + if (pCommith->pDataCols->numOfRows < pCfg->minRowsPerFileBlock) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isLast = true; + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; + } + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; + } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; + } + + return 0; +} + +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, + TSKEY maxKey, int maxRows, int8_t update) { + TSKEY key1 = INT64_MAX; + TSKEY key2 = INT64_MAX; + STSchema *pSchema = NULL; + + ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); + tdResetDataCols(pTarget); + + while (true) { + key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); + SMemRow row = tsdbNextIterRow(pCommitIter->pIter); + if (row == NULL || memRowKey(row) > maxKey) { + key2 = INT64_MAX; + } else { + key2 = memRowKey(row); + } + + if (key1 == INT64_MAX && key2 == INT64_MAX) break; + + if (key1 < key2) { + for (int i = 0; i < pDataCols->numOfCols; i++) { + //TODO: dataColAppendVal may fail + dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, + pTarget->maxPoints); + } + + pTarget->numOfRows++; + (*iter)++; + } else if (key1 > key2) { + if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row)); + ASSERT(pSchema != NULL); + } + + tdAppendMemRowToDataCol(row, pSchema, pTarget, true); + + tSkipListIterNext(pCommitIter->pIter); + } else { + if (update != TD_ROW_OVERWRITE_UPDATE) { + //copy disk data + for (int i = 0; i < pDataCols->numOfCols; i++) { + //TODO: dataColAppendVal may fail + dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, + pTarget->maxPoints); + } + + if(update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++; + } + if (update != TD_ROW_DISCARD_UPDATE) { + //copy mem data + if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row)); + ASSERT(pSchema != NULL); + } + + tdAppendMemRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE); + } + (*iter)++; + tSkipListIterNext(pCommitIter->pIter); + } + + if (pTarget->numOfRows >= maxRows) break; + } +} + +static void tsdbResetCommitFile(SCommitH *pCommith) { + pCommith->isRFileSet = false; + pCommith->isDFileSame = false; + pCommith->isLFileSame = false; + taosArrayClear(pCommith->aBlkIdx); +} + +static void tsdbResetCommitTable(SCommitH *pCommith) { + taosArrayClear(pCommith->aSubBlk); + taosArrayClear(pCommith->aSupBlk); + pCommith->pTable = NULL; +} + +static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { + SDiskID did; + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith); + + tfsAllocDisk(tsdbGetFidLevel(fid, &(pCommith->rtn)), &(did.level), &(did.id)); + if (did.level == TFS_UNDECIDED_LEVEL) { + terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + return -1; + } + + // Open read FSET + if (pSet) { + if (tsdbSetAndOpenReadFSet(&(pCommith->readh), pSet) < 0) { + return -1; + } + + pCommith->isRFileSet = true; + + if (tsdbLoadBlockIdx(&(pCommith->readh)) < 0) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + + tsdbDebug("vgId:%d FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet), + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + } else { + pCommith->isRFileSet = false; + } + + // Set and open commit FSET + if (pSet == NULL || did.level > TSDB_FSET_LEVEL(pSet)) { + // Create a new FSET to write data + tsdbInitDFileSet(pWSet, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo))); + + if (tsdbCreateDFileSet(pWSet, true) < 0) { + tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), + TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno)); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + } + return -1; + } + + pCommith->isDFileSame = false; + pCommith->isLFileSame = false; + + tsdbDebug("vgId:%d FSET %d at level %d disk id %d is created to commit", REPO_ID(pRepo), TSDB_FSET_FID(pWSet), + TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet)); + } else { + did.level = TSDB_FSET_LEVEL(pSet); + did.id = TSDB_FSET_ID(pSet); + + pCommith->wSet.fid = fid; + pCommith->wSet.state = 0; + + // TSDB_FILE_HEAD + SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); + tsdbInitDFile(pWHeadf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); + if (tsdbCreateDFile(pWHeadf, true) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), + tstrerror(terrno)); + + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + + // TSDB_FILE_DATA + SDFile *pRDataf = TSDB_READ_DATA_FILE(&(pCommith->readh)); + SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith); + tsdbInitDFileEx(pWDataf, pRDataf); + if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + pCommith->isDFileSame = true; + + // TSDB_FILE_LAST + SDFile *pRLastf = TSDB_READ_LAST_FILE(&(pCommith->readh)); + SDFile *pWLastf = TSDB_COMMIT_LAST_FILE(pCommith); + if (pRLastf->info.size < 32 * 1024) { + tsdbInitDFileEx(pWLastf, pRLastf); + pCommith->isLFileSame = true; + + if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } else { + tsdbInitDFile(pWLastf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST); + pCommith->isLFileSame = false; + + if (tsdbCreateDFile(pWLastf, true) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + (void)tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } + } + + return 0; +} + +static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError) { + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + } + + if (!hasError) { + TSDB_FSET_FSYNC(TSDB_COMMIT_WRITE_FSET(pCommith)); + } + tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); +} + +static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + int mergeRows = pBlock->numOfRows + pInfo->rowsInserted - pInfo->rowsDeleteSucceed; + + ASSERT(mergeRows > 0); + + if (pBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pInfo->nOperations <= pCfg->maxRowsPerFileBlock) { + if (pBlock->last) { + if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true; + } else { + if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true; + } + } + + return false; +} + +int tsdbApplyRtn(STsdbRepo *pRepo) { + SRtn rtn; + SFSIter fsiter; + STsdbFS * pfs = REPO_FS(pRepo); + SDFileSet *pSet; + + // Get retention snapshot + tsdbGetRtnSnap(pRepo, &rtn); + + tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); + while ((pSet = tsdbFSIterNext(&fsiter))) { + if (pSet->fid < rtn.minFid) { + tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + continue; + } + + if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) { + return -1; + } + } + + return 0; +} +#endif \ No newline at end of file diff --git a/src/tsdb/src/tsdbCompact.c b/source/dnode/vnode/tsdb/src/tsdbCompact.c similarity index 100% rename from src/tsdb/src/tsdbCompact.c rename to source/dnode/vnode/tsdb/src/tsdbCompact.c diff --git a/src/tsdb/src/tsdbFS.c b/source/dnode/vnode/tsdb/src/tsdbFS.c similarity index 100% rename from src/tsdb/src/tsdbFS.c rename to source/dnode/vnode/tsdb/src/tsdbFS.c diff --git a/src/tsdb/src/tsdbFile.c b/source/dnode/vnode/tsdb/src/tsdbFile.c similarity index 100% rename from src/tsdb/src/tsdbFile.c rename to source/dnode/vnode/tsdb/src/tsdbFile.c diff --git a/src/tsdb/src/tsdbHealth.c b/source/dnode/vnode/tsdb/src/tsdbHealth.c similarity index 100% rename from src/tsdb/src/tsdbHealth.c rename to source/dnode/vnode/tsdb/src/tsdbHealth.c diff --git a/source/dnode/vnode/tsdb/src/tsdbIdx.c b/source/dnode/vnode/tsdb/src/tsdbIdx.c deleted file mode 100644 index 6dea4a4e57392be988126c579648f39a8270b9bf..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/tsdb/src/tsdbIdx.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/src/tsdbMain.c b/source/dnode/vnode/tsdb/src/tsdbMain.c index d67d45660da5a02636a71dcb6d4461a3ddb8af41..c8bcfc69062cdac8e140edd6143a5c9c39568ab7 100644 --- a/source/dnode/vnode/tsdb/src/tsdbMain.c +++ b/source/dnode/vnode/tsdb/src/tsdbMain.c @@ -92,4 +92,1014 @@ static int tsdbOpenImpl(STsdb *pTsdb) { static void tsdbCloseImpl(STsdb *pTsdb) { // TODO -} \ No newline at end of file +} +#if 0 +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// no test file errors here +#include "taosdef.h" +#include "tsdbint.h" +#include "ttimer.h" +#include "tthread.h" + +#define IS_VALID_PRECISION(precision) \ + (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO)) +#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP +#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) + +static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); +static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH); +static void tsdbFreeRepo(STsdbRepo *pRepo); +static void tsdbStartStream(STsdbRepo *pRepo); +static void tsdbStopStream(STsdbRepo *pRepo); +static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh); +static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, SBlockIdx *pIdx); + +// Function declaration +int32_t tsdbCreateRepo(int repoid) { + char tsdbDir[TSDB_FILENAME_LEN] = "\0"; + char dataDir[TSDB_FILENAME_LEN] = "\0"; + + tsdbGetRootDir(repoid, tsdbDir); + if (tfsMkdir(tsdbDir) < 0) { + goto _err; + } + + tsdbGetDataDir(repoid, dataDir); + if (tfsMkdir(dataDir) < 0) { + goto _err; + } + + // TODO: need to create current file with nothing in + + return 0; + +_err: + tsdbError("vgId:%d failed to create TSDB repository since %s", repoid, tstrerror(terrno)); + return -1; +} + +int32_t tsdbDropRepo(int repoid) { + char tsdbDir[TSDB_FILENAME_LEN] = "\0"; + + tsdbGetRootDir(repoid, tsdbDir); + return tfsRmdir(tsdbDir); +} + +STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { + STsdbRepo *pRepo; + STsdbCfg config = *pCfg; + + terrno = TSDB_CODE_SUCCESS; + + // Check and set default configurations + if (tsdbCheckAndSetDefaultCfg(&config) < 0) { + tsdbError("vgId:%d failed to open TSDB repository since %s", config.tsdbId, tstrerror(terrno)); + return NULL; + } + + // Create new TSDB object + if ((pRepo = tsdbNewRepo(&config, pAppH)) == NULL) { + tsdbError("vgId:%d failed to open TSDB repository while creating TSDB object since %s", config.tsdbId, + tstrerror(terrno)); + return NULL; + } + + // Open meta + if (tsdbOpenMeta(pRepo) < 0) { + tsdbError("vgId:%d failed to open TSDB repository while opening Meta since %s", config.tsdbId, tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; + } + + if (tsdbOpenBufPool(pRepo) < 0) { + tsdbError("vgId:%d failed to open TSDB repository while opening buffer pool since %s", config.tsdbId, + tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; + } + + if (tsdbOpenFS(pRepo) < 0) { + tsdbError("vgId:%d failed to open TSDB repository while opening FS since %s", config.tsdbId, tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; + } + + // TODO: Restore information from data + if ((!(pRepo->state & TSDB_STATE_BAD_DATA)) && tsdbRestoreInfo(pRepo) < 0) { + tsdbError("vgId:%d failed to open TSDB repository while restore info since %s", config.tsdbId, tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; + } + + pRepo->mergeBuf = NULL; + + tsdbStartStream(pRepo); + + tsdbDebug("vgId:%d, TSDB repository opened", REPO_ID(pRepo)); + + return pRepo; +} + +// Note: all working thread and query thread must stopped when calling this function +int tsdbCloseRepo(STsdbRepo *repo, int toCommit) { + if (repo == NULL) return 0; + + STsdbRepo *pRepo = repo; + int vgId = REPO_ID(pRepo); + + terrno = TSDB_CODE_SUCCESS; + + tsdbStopStream(pRepo); + if(pRepo->pthread){ + taosDestoryThread(pRepo->pthread); + pRepo->pthread = NULL; + } + + if (toCommit) { + tsdbSyncCommit(repo); + } + + tsem_wait(&(pRepo->readyToCommit)); + + tsdbUnRefMemTable(pRepo, pRepo->mem); + tsdbUnRefMemTable(pRepo, pRepo->imem); + pRepo->mem = NULL; + pRepo->imem = NULL; + + tsdbCloseFS(pRepo); + tsdbCloseBufPool(pRepo); + tsdbCloseMeta(pRepo); + tsdbFreeRepo(pRepo); + tsdbDebug("vgId:%d repository is closed", vgId); + + if (terrno != TSDB_CODE_SUCCESS) { + return -1; + } else { + return 0; + } +} + +STsdbCfg *tsdbGetCfg(const STsdbRepo *repo) { + ASSERT(repo != NULL); + return &((STsdbRepo *)repo)->config; +} + +int tsdbLockRepo(STsdbRepo *pRepo) { + int code = pthread_mutex_lock(&pRepo->mutex); + if (code != 0) { + tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + pRepo->repoLocked = true; + return 0; +} + +int tsdbUnlockRepo(STsdbRepo *pRepo) { + ASSERT(IS_REPO_LOCKED(pRepo)); + pRepo->repoLocked = false; + int code = pthread_mutex_unlock(&pRepo->mutex); + if (code != 0) { + tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +int tsdbCheckCommit(STsdbRepo *pRepo) { + ASSERT(pRepo->mem != NULL); + STsdbCfg *pCfg = &(pRepo->config); + + STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); + ASSERT(pBufBlock != NULL); + if ((pRepo->mem->extraBuffList != NULL) || + ((listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 3) && (pBufBlock->remain < TSDB_BUFFER_RESERVE))) { + // trigger commit + if (tsdbAsyncCommit(pRepo) < 0) return -1; + } + + return 0; +} + +STsdbMeta *tsdbGetMeta(STsdbRepo *pRepo) { return pRepo->tsdbMeta; } + +STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo) { return NULL; } + +int tsdbGetState(STsdbRepo *repo) { return repo->state; } + +int8_t tsdbGetCompactState(STsdbRepo *repo) { return (int8_t)(repo->compactState); } + +void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) { + ASSERT(repo != NULL); + STsdbRepo *pRepo = repo; + *totalPoints = pRepo->stat.pointsWritten; + *totalStorage = pRepo->stat.totalStorage; + *compStorage = pRepo->stat.compStorage; +} + +int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { + // TODO: think about multithread cases + if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; + + STsdbCfg * pRCfg = &repo->config; + + ASSERT(pRCfg->tsdbId == pCfg->tsdbId); + ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); + ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); + ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); + ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); + ASSERT(pRCfg->precision == pCfg->precision); + + bool configChanged = false; + if (pRCfg->compression != pCfg->compression) { + configChanged = true; + } + if (pRCfg->keep != pCfg->keep) { + configChanged = true; + } + if (pRCfg->keep1 != pCfg->keep1) { + configChanged = true; + } + if (pRCfg->keep2 != pCfg->keep2) { + configChanged = true; + } + if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { + configChanged = true; + } + if (pRCfg->totalBlocks != pCfg->totalBlocks) { + configChanged = true; + } + + if (!configChanged) { + tsdbError("vgId:%d no config changed", REPO_ID(repo)); + } + + int code = pthread_mutex_lock(&repo->save_mutex); + if (code != 0) { + tsdbError("vgId:%d failed to lock tsdb save config mutex since %s", REPO_ID(repo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + + STsdbCfg * pSaveCfg = &repo->save_config; + *pSaveCfg = repo->config; + + pSaveCfg->compression = pCfg->compression; + pSaveCfg->keep = pCfg->keep; + pSaveCfg->keep1 = pCfg->keep1; + pSaveCfg->keep2 = pCfg->keep2; + pSaveCfg->cacheLastRow = pCfg->cacheLastRow; + pSaveCfg->totalBlocks = pCfg->totalBlocks; + + tsdbInfo("vgId:%d old config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d),totalBlocks(%d)", + REPO_ID(repo), + pRCfg->compression, pRCfg->keep, pRCfg->keep1,pRCfg->keep2, + pRCfg->cacheLastRow, pRCfg->totalBlocks); + tsdbInfo("vgId:%d new config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d),totalBlocks(%d)", + REPO_ID(repo), + pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, + pSaveCfg->cacheLastRow,pSaveCfg->totalBlocks); + + repo->config_changed = true; + + pthread_mutex_unlock(&repo->save_mutex); + + // schedule a commit msg and wait for the new config applied + tsdbSyncCommitConfig(repo); + + return 0; +#if 0 + STsdbRepo *pRepo = (STsdbRepo *)repo; + STsdbCfg config = pRepo->config; + STsdbCfg * pRCfg = &pRepo->config; + + if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; + + ASSERT(pRCfg->tsdbId == pCfg->tsdbId); + ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); + ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); + ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); + ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); + ASSERT(pRCfg->precision == pCfg->precision); + + bool configChanged = false; + if (pRCfg->compression != pCfg->compression) { + tsdbAlterCompression(pRepo, pCfg->compression); + config.compression = pCfg->compression; + configChanged = true; + } + if (pRCfg->keep != pCfg->keep) { + if (tsdbAlterKeep(pRepo, pCfg->keep) < 0) { + tsdbError("vgId:%d failed to configure repo when alter keep since %s", REPO_ID(pRepo), tstrerror(terrno)); + config.keep = pCfg->keep; + return -1; + } + configChanged = true; + } + if (pRCfg->totalBlocks != pCfg->totalBlocks) { + tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks); + config.totalBlocks = pCfg->totalBlocks; + configChanged = true; + } + if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { + config.cacheLastRow = pCfg->cacheLastRow; + configChanged = true; + } + + if (configChanged) { + if (tsdbSaveConfig(pRepo->rootDir, &config) < 0) { + tsdbError("vgId:%d failed to configure repository while save config since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } + + return 0; +#endif +} + +uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) { + // TODO + return 0; +#if 0 + STsdbRepo *pRepo = (STsdbRepo *)repo; + // STsdbMeta *pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + uint32_t magic = 0; + char * fname = NULL; + + struct stat fState; + + tsdbDebug("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex); + ASSERT(*index <= eindex); + + if (name[0] == 0) { // get the file from index or after, but not larger than eindex + int fid = (*index) / TSDB_FILE_TYPE_MAX; + + if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { + if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { + fname = tsdbGetMetaFileName(pRepo->rootDir); + *index = TSDB_META_FILE_INDEX; + magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); + sprintf(name, "tsdb/%s", TSDB_META_FILE_NAME); + } else { + return 0; + } + } else { + SFileGroup *pFGroup = + taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); + if (pFGroup->fileId == fid) { + SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX]; + fname = strdup(TSDB_FILE_NAME(pFile)); + magic = pFile->info.magic; + char *tfname = strdup(fname); + sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); + tfree(tfname); + } else { + if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) { + SFile *pFile = &pFGroup->files[0]; + fname = strdup(TSDB_FILE_NAME(pFile)); + *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; + magic = pFile->info.magic; + char *tfname = strdup(fname); + sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); + tfree(tfname); + } else { + return 0; + } + } + } + } else { // get the named file at the specified index. If not there, return 0 + fname = malloc(256); + sprintf(fname, "%s/vnode/vnode%d/%s", TFS_PRIMARY_PATH(), REPO_ID(pRepo), name); + if (access(fname, F_OK) != 0) { + tfree(fname); + return 0; + } + if (*index == TSDB_META_FILE_INDEX) { // get meta file + tsdbGetStoreInfo(fname, &magic, size); + } else { + char tfname[TSDB_FILENAME_LEN] = "\0"; + sprintf(tfname, "vnode/vnode%d/tsdb/%s/%s", REPO_ID(pRepo), TSDB_DATA_DIR_NAME, basename(name)); + tsdbGetFileInfoImpl(tfname, &magic, size); + } + tfree(fname); + return magic; + } + + if (stat(fname, &fState) < 0) { + tfree(fname); + return 0; + } + + *size = fState.st_size; + // magic = *size; + + tfree(fname); + return magic; +#endif +} + +void tsdbGetRootDir(int repoid, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb", repoid); +} + +void tsdbGetDataDir(int repoid, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data", repoid); +} + +static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { + // Check tsdbId + if (pCfg->tsdbId < 0) { + tsdbError("vgId:%d invalid vgroup ID", pCfg->tsdbId); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + + // Check precision + if (pCfg->precision == -1) { + pCfg->precision = TSDB_DEFAULT_PRECISION; + } else { + if (!IS_VALID_PRECISION(pCfg->precision)) { + tsdbError("vgId:%d invalid precision configuration %d", pCfg->tsdbId, pCfg->precision); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + } + + // Check compression + if (pCfg->compression == -1) { + pCfg->compression = TSDB_DEFAULT_COMPRESSION; + } else { + if (!IS_VALID_COMPRESSION(pCfg->compression)) { + tsdbError("vgId:%d invalid compression configuration %d", pCfg->tsdbId, pCfg->precision); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + } + + // Check daysPerFile + if (pCfg->daysPerFile == -1) { + pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; + } else { + if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { + tsdbError( + "vgId:%d invalid daysPerFile configuration! daysPerFile %d TSDB_MIN_DAYS_PER_FILE %d TSDB_MAX_DAYS_PER_FILE " + "%d", + pCfg->tsdbId, pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + } + + // Check minRowsPerFileBlock and maxRowsPerFileBlock + if (pCfg->minRowsPerFileBlock == -1) { + pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; + } else { + if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { + tsdbError( + "vgId:%d invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MIN_ROW_FBLOCK %d " + "TSDB_MAX_MIN_ROW_FBLOCK %d", + pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + } + + if (pCfg->maxRowsPerFileBlock == -1) { + pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; + } else { + if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { + tsdbError( + "vgId:%d invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAX_ROW_FBLOCK %d " + "TSDB_MAX_MAX_ROW_FBLOCK %d", + pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + } + + if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { + tsdbError("vgId:%d invalid configuration! minRowsPerFileBlock %d maxRowsPerFileBlock %d", pCfg->tsdbId, + pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + + // Check keep + if (pCfg->keep == -1) { + pCfg->keep = TSDB_DEFAULT_KEEP; + } else { + if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) { + tsdbError( + "vgId:%d invalid keep configuration! keep %d TSDB_MIN_KEEP %d " + "TSDB_MAX_KEEP %d", + pCfg->tsdbId, pCfg->keep, TSDB_MIN_KEEP, TSDB_MAX_KEEP); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + } + + if (pCfg->keep1 == 0) { + pCfg->keep1 = pCfg->keep; + } + + if (pCfg->keep2 == 0) { + pCfg->keep2 = pCfg->keep; + } + + // update check + if (pCfg->update < TD_ROW_DISCARD_UPDATE || pCfg->update > TD_ROW_PARTIAL_UPDATE) + pCfg->update = TD_ROW_DISCARD_UPDATE; + + // update cacheLastRow + if (pCfg->cacheLastRow != 0) { + if (pCfg->cacheLastRow > 3) + pCfg->cacheLastRow = 1; + } + return 0; +} + +static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { + STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(*pRepo)); + if (pRepo == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + + pRepo->state = TSDB_STATE_OK; + pRepo->code = TSDB_CODE_SUCCESS; + pRepo->compactState = 0; + pRepo->config = *pCfg; + if (pAppH) { + pRepo->appH = *pAppH; + } + pRepo->repoLocked = false; + pRepo->pthread = NULL; + + int code = pthread_mutex_init(&(pRepo->mutex), NULL); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + tsdbFreeRepo(pRepo); + return NULL; + } + + code = pthread_mutex_init(&(pRepo->save_mutex), NULL); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + tsdbFreeRepo(pRepo); + return NULL; + } + pRepo->config_changed = false; + atomic_store_8(&pRepo->hasCachedLastColumn, 0); + + code = tsem_init(&(pRepo->readyToCommit), 0, 1); + if (code != 0) { + code = errno; + terrno = TAOS_SYSTEM_ERROR(code); + tsdbFreeRepo(pRepo); + return NULL; + } + + pRepo->tsdbMeta = tsdbNewMeta(pCfg); + if (pRepo->tsdbMeta == NULL) { + tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbFreeRepo(pRepo); + return NULL; + } + + pRepo->pPool = tsdbNewBufPool(pCfg); + if (pRepo->pPool == NULL) { + tsdbError("vgId:%d failed to create buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbFreeRepo(pRepo); + return NULL; + } + + pRepo->fs = tsdbNewFS(pCfg); + if (pRepo->fs == NULL) { + tsdbError("vgId:%d failed to TSDB file system since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbFreeRepo(pRepo); + return NULL; + } + + return pRepo; +} + +static void tsdbFreeRepo(STsdbRepo *pRepo) { + if (pRepo) { + tsdbFreeFS(pRepo->fs); + tsdbFreeBufPool(pRepo->pPool); + tsdbFreeMeta(pRepo->tsdbMeta); + tsdbFreeMergeBuf(pRepo->mergeBuf); + // tsdbFreeMemTable(pRepo->mem); + // tsdbFreeMemTable(pRepo->imem); + tsem_destroy(&(pRepo->readyToCommit)); + pthread_mutex_destroy(&pRepo->mutex); + free(pRepo); + } +} + +static void tsdbStartStream(STsdbRepo *pRepo) { + STsdbMeta *pMeta = pRepo->tsdbMeta; + + for (int i = 0; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable && pTable->type == TSDB_STREAM_TABLE) { + pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, + tsdbGetTableSchemaImpl(pTable, false, false, -1), 0); + } + } +} + +static void tsdbStopStream(STsdbRepo *pRepo) { + STsdbMeta *pMeta = pRepo->tsdbMeta; + + for (int i = 0; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable && pTable->type == TSDB_STREAM_TABLE) { + (*pRepo->appH.cqDropFunc)(pTable->cqhandle); + } + } +} + +static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh) { + //tsdbInfo("tsdbRestoreLastColumns of table %s", pTable->name->data); + + STSchema *pSchema = tsdbGetTableLatestSchema(pTable); + if (pSchema == NULL) { + tsdbError("tsdbGetTableLatestSchema of table %s fail", pTable->name->data); + return 0; + } + + SBlock* pBlock; + int numColumns; + int32_t blockIdx; + SDataStatis* pBlockStatis = NULL; + SMemRow row = NULL; + // restore last column data with last schema + + int err = 0; + + numColumns = schemaNCols(pSchema); + if (numColumns <= pTable->restoreColumnNum) { + pTable->hasRestoreLastColumn = true; + return 0; + } + if (pTable->lastColSVersion != schemaVersion(pSchema)) { + if (tsdbInitColIdCacheWithSchema(pTable, pSchema) < 0) { + return -1; + } + } + + row = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); + if (row == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + err = -1; + goto out; + } + + memRowSetType(row, SMEM_ROW_DATA); + tdInitDataRow(memRowDataBody(row), pSchema); + + // first load block index info + if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { + err = -1; + goto out; + } + + pBlockStatis = calloc(numColumns, sizeof(SDataStatis)); + if (pBlockStatis == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + err = -1; + goto out; + } + memset(pBlockStatis, 0, numColumns * sizeof(SDataStatis)); + for(int32_t i = 0; i < numColumns; ++i) { + STColumn *pCol = schemaColAt(pSchema, i); + pBlockStatis[i].colId = pCol->colId; + } + + // load block from backward + SBlockIdx *pIdx = pReadh->pBlkIdx; + blockIdx = (int32_t)(pIdx->numOfBlocks - 1); + + while (numColumns > pTable->restoreColumnNum && blockIdx >= 0) { + bool loadStatisData = false; + pBlock = pReadh->pBlkInfo->blocks + blockIdx; + blockIdx -= 1; + + // load block data + if (tsdbLoadBlockData(pReadh, pBlock, NULL) < 0) { + err = -1; + goto out; + } + + // file block with sub-blocks has no statistics data + if (pBlock->numOfSubBlocks <= 1) { + tsdbLoadBlockStatis(pReadh, pBlock); + tsdbGetBlockStatis(pReadh, pBlockStatis, (int)numColumns); + loadStatisData = true; + } + + for (int16_t i = 0; i < numColumns && numColumns > pTable->restoreColumnNum; ++i) { + STColumn *pCol = schemaColAt(pSchema, i); + // ignore loaded columns + if (pTable->lastCols[i].bytes != 0) { + continue; + } + + // ignore block which has no not-null colId column + if (loadStatisData && pBlockStatis[i].numOfNull == pBlock->numOfRows) { + continue; + } + + // OK,let's load row from backward to get not-null column + for (int32_t rowId = pBlock->numOfRows - 1; rowId >= 0; rowId--) { + SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; + const void* pColData = tdGetColDataOfRow(pDataCol, rowId); + tdAppendColVal(memRowDataBody(row), pColData, pCol->type, pCol->offset); + //SDataCol *pDataCol = readh.pDCols[0]->cols + j; + void *value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); + if (isNull(value, pCol->type)) { + continue; + } + + int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pCol->colId); + if (idx == -1) { + tsdbError("tsdbRestoreLastColumns restore vgId:%d,table:%s cache column %d fail", REPO_ID(pRepo), pTable->name->data, pCol->colId); + continue; + } + // save not-null column + uint16_t bytes = IS_VAR_DATA_TYPE(pCol->type) ? varDataTLen(pColData) : pCol->bytes; + SDataCol *pLastCol = &(pTable->lastCols[idx]); + pLastCol->pData = malloc(bytes); + pLastCol->bytes = bytes; + pLastCol->colId = pCol->colId; + memcpy(pLastCol->pData, value, bytes); + + // save row ts(in column 0) + pDataCol = pReadh->pDCols[0]->cols + 0; + pCol = schemaColAt(pSchema, 0); + tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->offset); + pLastCol->ts = memRowKey(row); + + pTable->restoreColumnNum += 1; + + tsdbDebug("tsdbRestoreLastColumns restore vgId:%d,table:%s cache column %d, %" PRId64, REPO_ID(pRepo), pTable->name->data, pLastCol->colId, pLastCol->ts); + break; + } + } + } + +out: + taosTZfree(row); + tfree(pBlockStatis); + + if (err == 0 && numColumns <= pTable->restoreColumnNum) { + pTable->hasRestoreLastColumn = true; + } + + return err; +} + +static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, SBlockIdx *pIdx) { + ASSERT(pTable->lastRow == NULL); + if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { + return -1; + } + + SBlock* pBlock = pReadh->pBlkInfo->blocks + pIdx->numOfBlocks - 1; + + if (tsdbLoadBlockData(pReadh, pBlock, NULL) < 0) { + return -1; + } + + // Get the data in row + + STSchema *pSchema = tsdbGetTableSchema(pTable); + pTable->lastRow = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); + if (pTable->lastRow == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + memRowSetType(pTable->lastRow, SMEM_ROW_DATA); + tdInitDataRow(memRowDataBody(pTable->lastRow), pSchema); + for (int icol = 0; icol < schemaNCols(pSchema); icol++) { + STColumn *pCol = schemaColAt(pSchema, icol); + SDataCol *pDataCol = pReadh->pDCols[0]->cols + icol; + tdAppendColVal(memRowDataBody(pTable->lastRow), tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, + pCol->offset); + } + + return 0; +} + +int tsdbRestoreInfo(STsdbRepo *pRepo) { + SFSIter fsiter; + SReadH readh; + SDFileSet *pSet; + STsdbMeta *pMeta = pRepo->tsdbMeta; + STsdbCfg * pCfg = REPO_CFG(pRepo); + + if (tsdbInitReadH(&readh, pRepo) < 0) { + return -1; + } + + tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); + + if (CACHE_LAST_NULL_COLUMN(pCfg)) { + for (int i = 1; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + pTable->restoreColumnNum = 0; + pTable->hasRestoreLastColumn = false; + } + } + + while ((pSet = tsdbFSIterNext(&fsiter)) != NULL) { + if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + if (tsdbLoadBlockIdx(&readh) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + for (int i = 1; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + + //tsdbInfo("tsdbRestoreInfo restore vgId:%d,table:%s", REPO_ID(pRepo), pTable->name->data); + + if (tsdbSetReadTable(&readh, pTable) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); + SBlockIdx *pIdx = readh.pBlkIdx; + if (pIdx && lastKey < pIdx->maxKey) { + pTable->lastKey = pIdx->maxKey; + + if (CACHE_LAST_ROW(pCfg) && tsdbRestoreLastRow(pRepo, pTable, &readh, pIdx) != 0) { + tsdbDestroyReadH(&readh); + return -1; + } + } + + // restore NULL columns + if (pIdx && CACHE_LAST_NULL_COLUMN(pCfg) && !pTable->hasRestoreLastColumn) { + if (tsdbRestoreLastColumns(pRepo, pTable, &readh) != 0) { + tsdbDestroyReadH(&readh); + return -1; + } + } + } + } + + tsdbDestroyReadH(&readh); + + if (CACHE_LAST_NULL_COLUMN(pCfg)) { + atomic_store_8(&pRepo->hasCachedLastColumn, 1); + } + + return 0; +} + +int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) { + bool cacheLastRow = false, cacheLastCol = false; + SFSIter fsiter; + SReadH readh; + SDFileSet *pSet; + STsdbMeta *pMeta = pRepo->tsdbMeta; + int tableNum = 0; + int maxTableIdx = 0; + int cacheLastRowTableNum = 0; + int cacheLastColTableNum = 0; + + bool need_free_last_row = CACHE_LAST_ROW(oldCfg) && !CACHE_LAST_ROW(&(pRepo->config)); + bool need_free_last_col = CACHE_LAST_NULL_COLUMN(oldCfg) && !CACHE_LAST_NULL_COLUMN(&(pRepo->config)); + + if (CACHE_LAST_ROW(&(pRepo->config)) || CACHE_LAST_NULL_COLUMN(&(pRepo->config))) { + tsdbInfo("tsdbCacheLastData cache last data since cacheLast option changed"); + cacheLastRow = !CACHE_LAST_ROW(oldCfg) && CACHE_LAST_ROW(&(pRepo->config)); + cacheLastCol = !CACHE_LAST_NULL_COLUMN(oldCfg) && CACHE_LAST_NULL_COLUMN(&(pRepo->config)); + } + + // calc max table idx and table num + for (int i = 1; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + tableNum += 1; + maxTableIdx = i; + if (cacheLastCol) { + pTable->restoreColumnNum = 0; + pTable->hasRestoreLastColumn = false; + } + } + + // if close last option,need to free data + if (need_free_last_row || need_free_last_col) { + if (need_free_last_col) { + atomic_store_8(&pRepo->hasCachedLastColumn, 0); + } + tsdbInfo("free cache last data since cacheLast option changed"); + for (int i = 1; i <= maxTableIdx; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + if (need_free_last_row) { + taosTZfree(pTable->lastRow); + pTable->lastRow = NULL; + } + if (need_free_last_col) { + tsdbFreeLastColumns(pTable); + pTable->hasRestoreLastColumn = false; + } + } + } + + if (!cacheLastRow && !cacheLastCol) { + return 0; + } + + cacheLastRowTableNum = cacheLastRow ? tableNum : 0; + cacheLastColTableNum = cacheLastCol ? tableNum : 0; + + if (tsdbInitReadH(&readh, pRepo) < 0) { + return -1; + } + + tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); + + while ((pSet = tsdbFSIterNext(&fsiter)) != NULL && (cacheLastRowTableNum > 0 || cacheLastColTableNum > 0)) { + if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + if (tsdbLoadBlockIdx(&readh) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + for (int i = 1; i <= maxTableIdx; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + + //tsdbInfo("tsdbRestoreInfo restore vgId:%d,table:%s", REPO_ID(pRepo), pTable->name->data); + + if (tsdbSetReadTable(&readh, pTable) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + SBlockIdx *pIdx = readh.pBlkIdx; + + if (pIdx && cacheLastRowTableNum > 0 && pTable->lastRow == NULL) { + pTable->lastKey = pIdx->maxKey; + + if (tsdbRestoreLastRow(pRepo, pTable, &readh, pIdx) != 0) { + tsdbDestroyReadH(&readh); + return -1; + } + cacheLastRowTableNum -= 1; + } + + // restore NULL columns + if (pIdx && cacheLastColTableNum > 0 && !pTable->hasRestoreLastColumn) { + if (tsdbRestoreLastColumns(pRepo, pTable, &readh) != 0) { + tsdbDestroyReadH(&readh); + return -1; + } + if (pTable->hasRestoreLastColumn) { + cacheLastColTableNum -= 1; + } + } + } + } + + tsdbDestroyReadH(&readh); + + if (cacheLastCol) { + atomic_store_8(&pRepo->hasCachedLastColumn, 1); + } + + return 0; +} +#endif \ No newline at end of file diff --git a/src/tsdb/src/tsdbRead.c b/source/dnode/vnode/tsdb/src/tsdbRead.c similarity index 100% rename from src/tsdb/src/tsdbRead.c rename to source/dnode/vnode/tsdb/src/tsdbRead.c diff --git a/src/tsdb/src/tsdbReadImpl.c b/source/dnode/vnode/tsdb/src/tsdbReadImpl.c similarity index 100% rename from src/tsdb/src/tsdbReadImpl.c rename to source/dnode/vnode/tsdb/src/tsdbReadImpl.c diff --git a/src/tsdb/src/tsdbRowMergeBuf.c b/source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c similarity index 100% rename from src/tsdb/src/tsdbRowMergeBuf.c rename to source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c diff --git a/source/dnode/vnode/tsdb/src/tsdbSMA.c b/source/dnode/vnode/tsdb/src/tsdbSMA.c deleted file mode 100644 index 6dea4a4e57392be988126c579648f39a8270b9bf..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/tsdb/src/tsdbSMA.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/src/tsdb/src/tsdbScan.c b/source/dnode/vnode/tsdb/src/tsdbScan.c similarity index 100% rename from src/tsdb/src/tsdbScan.c rename to source/dnode/vnode/tsdb/src/tsdbScan.c diff --git a/src/tsdb/src/tsdbSync.c b/source/dnode/vnode/tsdb/src/tsdbSync.c similarity index 100% rename from src/tsdb/src/tsdbSync.c rename to source/dnode/vnode/tsdb/src/tsdbSync.c diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index e14c58d4122f1f830bcac17c2c358ef6632ef15c..29c80ae9ec05453dd23605c39d43fa562c2aa2d9 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -61,8 +61,8 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep2 = htonl(3650); - pReq->minRowsPerFileBlock = htonl(100); - pReq->maxRowsPerFileBlock = htonl(4096); + pReq->minRows = htonl(100); + pReq->maxRows = htonl(4096); pReq->commitTime = htonl(3600); pReq->fsyncPeriod = htonl(3000); pReq->walLevel = 1; diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index a258cb834dcbe4a15845c690b706d46b09674c78..72415a971674790f1b8ce5d9f21261c82eb16b89 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -18,87 +18,112 @@ #include "index.h" #include "index_fst.h" -#include "tlog.h" -#include "thash.h" #include "taos.h" +#include "thash.h" +#include "tlog.h" #ifdef USE_LUCENE #include #endif - #ifdef __cplusplus extern "C" { #endif -typedef enum {kTypeValue, kTypeDeletion} STermValueType ; +typedef enum { kTypeValue, kTypeDeletion } STermValueType; typedef struct SIndexStat { - int32_t totalAdded; // - int32_t totalDeled; // - int32_t totalUpdated; // - int32_t totalTerms; // - int32_t distinctCol; // distinct column -} SIndexStat; + int32_t totalAdded; // + int32_t totalDeled; // + int32_t totalUpdated; // + int32_t totalTerms; // + int32_t distinctCol; // distinct column +} SIndexStat; struct SIndex { -#ifdef USE_LUCENE - index_t *index; -#endif - void *cache; - void *tindex; - SHashObj *colObj;// < field name, field id> - - int64_t suid; // current super table id, -1 is normal table - int colId; // field id allocated to cache - int32_t cVersion; // current version allocated to cache - - SIndexStat stat; - pthread_mutex_t mtx; -}; +#ifdef USE_LUCENE + index_t *index; +#endif + void * cache; + void * tindex; + SHashObj *colObj; // < field name, field id> + + int64_t suid; // current super table id, -1 is normal table + int colId; // field id allocated to cache + int32_t cVersion; // current version allocated to cache + + SIndexStat stat; + pthread_mutex_t mtx; +}; struct SIndexOpts { -#ifdef USE_LUCENE - void *opts; -#endif +#ifdef USE_LUCENE + void *opts; +#endif #ifdef USE_INVERTED_INDEX - int32_t cacheSize; // MB + int32_t cacheSize; // MB // add cache module later #endif - }; struct SIndexMultiTermQuery { - EIndexOperatorType opera; - SArray *query; + EIndexOperatorType opera; + SArray * query; }; // field and key; typedef struct SIndexTerm { - int64_t suid; - SIndexOperOnColumn operType; // oper type, add/del/update - uint8_t colType; // term data type, str/interger/json - char *colName; - int32_t nColName; - char *colVal; - int32_t nColVal; + int64_t suid; + SIndexOperOnColumn operType; // oper type, add/del/update + uint8_t colType; // term data type, str/interger/json + char * colName; + int32_t nColName; + char * colVal; + int32_t nColVal; } SIndexTerm; typedef struct SIndexTermQuery { - SIndexTerm* term; + SIndexTerm * term; EIndexQueryType qType; } SIndexTermQuery; - - -#define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); }} while(0) -#define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); }} while(0) -#define indexWarn(...) do { if (sDebugFlag & DEBUG_WARN) { taosPrintLog("index WARN ", 255, __VA_ARGS__); }} while(0) -#define indexInfo(...) do { if (sDebugFlag & DEBUG_INFO) { taosPrintLog("index ", 255, __VA_ARGS__); }} while(0) -#define indexDebug(...) do { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); }} while(0) -#define indexTrace(...) do { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); }} while(0) - +#define indexFatal(...) \ + do { \ + if (sDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("index FATAL ", 255, __VA_ARGS__); \ + } \ + } while (0) +#define indexError(...) \ + do { \ + if (sDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("index ERROR ", 255, __VA_ARGS__); \ + } \ + } while (0) +#define indexWarn(...) \ + do { \ + if (sDebugFlag & DEBUG_WARN) { \ + taosPrintLog("index WARN ", 255, __VA_ARGS__); \ + } \ + } while (0) +#define indexInfo(...) \ + do { \ + if (sDebugFlag & DEBUG_INFO) { \ + taosPrintLog("index ", 255, __VA_ARGS__); \ + } \ + } while (0) +#define indexDebug(...) \ + do { \ + if (sDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ + } \ + } while (0) +#define indexTrace(...) \ + do { \ + if (sDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ + } \ + } while (0) #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 97a7b835f6a5afb348847276ba427a11b2113c83..9d5083d23d14c3e2490f3228b28672cdf8417f08 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -22,8 +22,10 @@ // ----------------- key structure in skiplist --------------------- /* A data row, the format is like below: - * content: |<--totalLen-->|<-- fieldid-->|<--field type -->|<-- value len--->|<-- value -->|<-- uid -->|<--version--->|<-- itermType -->| - * len : |<--int32_t -->|<-- int16_t-->|<-- int8_t --->|<--- int32_t --->|<--valuelen->|<--uint64_t->|<-- int32_t-->|<-- int8_t --->| + * content: |<--totalLen-->|<-- fieldid-->|<--field type-->|<-- value len--->| + * |<-- value -->|<--uid -->|<--version--->|<-- itermType -->| + * len : |<--int32_t -->|<-- int16_t-->|<-- int8_t --->|<--- int32_t --->| + * <--valuelen->|<--uint64_t->| * <-- int32_t-->|<-- int8_t --->| */ #ifdef __cplusplus @@ -31,25 +33,23 @@ extern "C" { #endif typedef struct IndexCache { - T_REF_DECLARE() + T_REF_DECLARE() SSkipList *skiplist; } IndexCache; - -// +// IndexCache *indexCacheCreate(); void indexCacheDestroy(void *cache); -int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid); +int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid); -//int indexCacheGet(void *cache, uint64_t *rst); -int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s); +// int indexCacheGet(void *cache, uint64_t *rst); +int indexCacheSearch( + void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s); #ifdef __cplusplus } #endif - - #endif diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 0dcc25831c6bb3940e19d8541653e47a24736143..67438c092b3a99df876e445e2abf71d1b1a126c1 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -20,29 +20,29 @@ extern "C" { #endif -#include "tarray.h" -#include "index_fst_util.h" -#include "index_fst_registry.h" -#include "index_fst_counting_writer.h" #include "index_fst_automation.h" +#include "index_fst_counting_writer.h" +#include "index_fst_registry.h" +#include "index_fst_util.h" +#include "tarray.h" -#define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a) +#define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a) -typedef struct Fst Fst; -typedef struct FstNode FstNode; +typedef struct Fst Fst; +typedef struct FstNode FstNode; typedef struct StreamWithState StreamWithState; -typedef enum { Included, Excluded, Unbounded} FstBound; +typedef enum { Included, Excluded, Unbounded } FstBound; typedef struct FstBoundWithData { - FstSlice data; + FstSlice data; FstBound type; } FstBoundWithData; typedef struct FstStreamBuilder { - Fst *fst; - AutomationCtx *aut; - FstBoundWithData *min; + Fst * fst; + AutomationCtx * aut; + FstBoundWithData *min; FstBoundWithData *max; } FstStreamBuilder, FstStreamWithStateBuilder; @@ -51,17 +51,14 @@ typedef struct FstRange { uint64_t end; } FstRange; +typedef enum { GE, GT, LE, LT } RangeType; +typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State; +typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType; -typedef enum {GE, GT, LE, LT} RangeType; -typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal} State; -typedef enum {Ordered, OutOfOrdered, DuplicateKey} OrderType; - - -FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data); -bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice); -bool fstBoundWithDataIsEmpty(FstBoundWithData *bound); -bool fstBoundWithDataIsIncluded(FstBoundWithData *bound); - +FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data); +bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice); +bool fstBoundWithDataIsEmpty(FstBoundWithData *bound); +bool fstBoundWithDataIsIncluded(FstBoundWithData *bound); typedef struct FstOutput { bool null; @@ -69,110 +66,105 @@ typedef struct FstOutput { } FstOutput; /* - * + * * UnFinished node and helper function - * TODO: simple function name + * TODO: simple function name */ typedef struct FstUnFinishedNodes { - SArray *stack; // } FstUnFinishedNodes; + SArray *stack; // } FstUnFinishedNodes; } FstUnFinishedNodes; -#define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack) - -FstUnFinishedNodes *fstUnFinishedNodesCreate(); -void fstUnFinishedNodesDestroy(FstUnFinishedNodes *node); -void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal); -FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes); -FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr); -FstBuilderNode *fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes); -void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *node, Output out); -void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *node, CompiledAddr addr); -void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *node, FstSlice bs, Output out); -uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs); -uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out); +#define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack) +FstUnFinishedNodes *fstUnFinishedNodesCreate(); +void fstUnFinishedNodesDestroy(FstUnFinishedNodes *node); +void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal); +void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *node, Output out); +void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *node, CompiledAddr addr); +void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *node, FstSlice bs, Output out); +uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs); +FstBuilderNode * fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes); +FstBuilderNode * fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr); +FstBuilderNode * fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes); + +uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out); typedef struct FstBuilder { - FstCountingWriter *wrt; // The FST raw data is written directly to `wtr`. - FstUnFinishedNodes *unfinished; // The stack of unfinished nodes - FstRegistry* registry; // A map of finished nodes. - FstSlice last; // The last word added - CompiledAddr lastAddr; // The address of the last compiled node - uint64_t len; // num of keys added + FstCountingWriter * wrt; // The FST raw data is written directly to `wtr`. + FstUnFinishedNodes *unfinished; // The stack of unfinished nodes + FstRegistry * registry; // A map of finished nodes. + FstSlice last; // The last word added + CompiledAddr lastAddr; // The address of the last compiled node + uint64_t len; // num of keys added } FstBuilder; - FstBuilder *fstBuilderCreate(void *w, FstType ty); - -void fstBuilderDestroy(FstBuilder *b); -void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in); -bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in); -OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup); -void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate); +void fstBuilderDestroy(FstBuilder *b); +void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in); +bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in); +void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate); +void * fstBuilerIntoInner(FstBuilder *b); +void fstBuilderFinish(FstBuilder *b); +OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup); CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn); -void* fstBuilerIntoInner(FstBuilder *b); -void fstBuilderFinish(FstBuilder *b); - - - typedef struct FstTransitions { - FstNode *node; - FstRange range; + FstNode *node; + FstRange range; } FstTransitions; -//FstState and relation function +// FstState and relation function typedef struct FstState { - State state; + State state; uint8_t val; } FstState; -FstState fstStateCreateFrom(FstSlice* data, CompiledAddr addr); +FstState fstStateCreateFrom(FstSlice *data, CompiledAddr addr); FstState fstStateCreate(State state); -//compile +// compile void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uint8_t inp); void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTransition *trn); void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node); // set_comm_input -void fstStateSetCommInput(FstState* state, uint8_t inp); +void fstStateSetCommInput(FstState *state, uint8_t inp); // comm_input -uint8_t fstStateCommInput(FstState* state, bool *null); +uint8_t fstStateCommInput(FstState *state, bool *null); // input_len -uint64_t fstStateInputLen(FstState* state); +uint64_t fstStateInputLen(FstState *state); - -// end_addr -uint64_t fstStateEndAddrForOneTransNext(FstState* state, FstSlice *data); +// end_addr +uint64_t fstStateEndAddrForOneTransNext(FstState *state, FstSlice *data); uint64_t fstStateEndAddrForOneTrans(FstState *state, FstSlice *data, PackSizes sizes); -uint64_t fstStateEndAddrForAnyTrans(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans); -// input -uint8_t fstStateInput(FstState *state, FstNode *node); -uint8_t fstStateInputForAnyTrans(FstState *state, FstNode *node, uint64_t i); +uint64_t fstStateEndAddrForAnyTrans( + FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans); +// input +uint8_t fstStateInput(FstState *state, FstNode *node); +uint8_t fstStateInputForAnyTrans(FstState *state, FstNode *node, uint64_t i); // trans_addr CompiledAddr fstStateTransAddr(FstState *state, FstNode *node); CompiledAddr fstStateTransAddrForAnyTrans(FstState *state, FstNode *node, uint64_t i); -// sizes +// sizes PackSizes fstStateSizes(FstState *state, FstSlice *data); -// Output +// Output Output fstStateOutput(FstState *state, FstNode *node); Output fstStateOutputForAnyTrans(FstState *state, FstNode *node, uint64_t i); // anyTrans specify function void fstStateSetFinalState(FstState *state, bool yes); -bool fstStateIsFinalState(FstState *state); +bool fstStateIsFinalState(FstState *state); void fstStateSetStateNtrans(FstState *state, uint8_t n); // state_ntrans -uint8_t fstStateStateNtrans(FstState *state, bool *null); +uint8_t fstStateStateNtrans(FstState *state, bool *null); uint64_t fstStateTotalTransSize(FstState *state, uint64_t version, PackSizes size, uint64_t nTrans); uint64_t fstStateTransIndexSize(FstState *state, uint64_t version, uint64_t nTrans); uint64_t fstStateNtransLen(FstState *state); @@ -180,72 +172,72 @@ uint64_t fstStateNtrans(FstState *state, FstSlice *slice); Output fstStateFinalOutput(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans); uint64_t fstStateFindInput(FstState *state, FstNode *node, uint8_t b, bool *null); - - -#define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext) +#define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext) #define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) #define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans) -#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal) - +#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal) typedef struct FstLastTransition { uint8_t inp; Output out; } FstLastTransition; -/* +/* * FstBuilderNodeUnfinished and helper function - * TODO: simple function name + * TODO: simple function name */ typedef struct FstBuilderNodeUnfinished { - FstBuilderNode *node; - FstLastTransition* last; + FstBuilderNode * node; + FstLastTransition *last; } FstBuilderNodeUnfinished; - - void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *node, CompiledAddr addr); + void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *node, Output out); /* - * FstNode and helper function + * FstNode and helper function */ typedef struct FstNode { FstSlice data; - uint64_t version; + uint64_t version; FstState state; - CompiledAddr start; - CompiledAddr end; + CompiledAddr start; + CompiledAddr end; bool isFinal; uint64_t nTrans; PackSizes sizes; - Output finalOutput; + Output finalOutput; } FstNode; -// If this node is final and has a terminal output value, then it is, returned. Otherwise, a zero output is returned +// If this node is final and has a terminal output value, then it is, returned. +// Otherwise, a zero output is returned #define FST_NODE_FINAL_OUTPUT(node) node->finalOutput -// Returns true if and only if this node corresponds to a final or "match", state in the finite state transducer. +// Returns true if and only if this node corresponds to a final or "match", +// state in the finite state transducer. #define FST_NODE_IS_FINAL(node) node->isFinal -// Returns the number of transitions in this node, The maximum number of transitions is 256. +// Returns the number of transitions in this node, The maximum number of +// transitions is 256. #define FST_NODE_LEN(node) node->nTrans // Returns true if and only if this node has zero transitions. #define FST_NODE_IS_EMPTYE(node) (node->nTrans == 0) // Return the address of this node. -#define FST_NODE_ADDR(node) node->start - +#define FST_NODE_ADDR(node) node->start FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *data); -void fstNodeDestroy(FstNode *fstNode); +void fstNodeDestroy(FstNode *fstNode); -FstTransitions fstNodeTransitionIter(FstNode *node); -FstTransitions* fstNodeTransitions(FstNode *node); -bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *res); -bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res); -bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res); -bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode); -FstSlice fstNodeAsSlice(FstNode *node); +FstTransitions fstNodeTransitionIter(FstNode *node); +FstTransitions *fstNodeTransitions(FstNode *node); +bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *res); +bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res); +bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res); -// ops +bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode); + +FstSlice fstNodeAsSlice(FstNode *node); + +// ops typedef struct FstIndexedValue { uint64_t index; @@ -253,87 +245,87 @@ typedef struct FstIndexedValue { } FstIndexedValue; FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out); -void fstLastTransitionDestroy(FstLastTransition *trn); - +void fstLastTransitionDestroy(FstLastTransition *trn); typedef struct FstMeta { uint64_t version; - CompiledAddr rootAddr; + CompiledAddr rootAddr; FstType ty; uint64_t len; uint32_t checkSum; } FstMeta; typedef struct Fst { - FstMeta *meta; - FstSlice *data; // - FstNode *root; // + FstMeta * meta; + FstSlice *data; // + FstNode * root; // } Fst; -// refactor simple function +// refactor simple function -Fst* fstCreate(FstSlice *data); -void fstDestroy(Fst *fst); +Fst *fstCreate(FstSlice *data); +void fstDestroy(Fst *fst); -bool fstGet(Fst *fst, FstSlice *b, Output *out); -FstNode* fstGetNode(Fst *fst, CompiledAddr); -FstNode* fstGetRoot(Fst *fst); -FstType fstGetType(Fst *fst); -CompiledAddr fstGetRootAddr(Fst *fst); -Output fstEmptyFinalOutput(Fst *fst, bool *null); +bool fstGet(Fst *fst, FstSlice *b, Output *out); +FstNode * fstGetNode(Fst *fst, CompiledAddr); +FstNode * fstGetRoot(Fst *fst); +FstType fstGetType(Fst *fst); +CompiledAddr fstGetRootAddr(Fst *fst); +Output fstEmptyFinalOutput(Fst *fst, bool *null); FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx); -FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx); - -// into stream to expand later -StreamWithState* streamBuilderIntoStream(FstStreamBuilder *sb); -bool fstVerify(Fst *fst); +FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx); +// into stream to expand later +StreamWithState *streamBuilderIntoStream(FstStreamBuilder *sb); +bool fstVerify(Fst *fst); -//refactor this function -bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); +// refactor this function +bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); typedef struct StreamState { - FstNode *node; + FstNode * node; uint64_t trans; - FstOutput out; - void *autState; -} StreamState; + FstOutput out; + void * autState; +} StreamState; void streamStateDestroy(void *s); typedef struct StreamWithState { - Fst *fst; - AutomationCtx *aut; - SArray *inp; - FstOutput emptyOutput; - SArray *stack; // + Fst * fst; + AutomationCtx * aut; + SArray * inp; + FstOutput emptyOutput; + SArray * stack; // FstBoundWithData *endAt; } StreamWithState; typedef struct StreamWithStateResult { - FstSlice data; + FstSlice data; FstOutput out; - void *state; + void * state; } StreamWithStateResult; StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state); -void swsResultDestroy(StreamWithStateResult *result); +void swsResultDestroy(StreamWithStateResult *result); + +typedef void *(*StreamCallback)(void *); +StreamWithState *streamWithStateCreate( + Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max); -typedef void* (*StreamCallback)(void *); -StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) ; void streamWithStateDestroy(StreamWithState *sws); -bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min); -StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); +bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min); -FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut); +StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); + +FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut); // set up bound range -// refator, simple code by marco +// refator, simple code by marco FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type); - #ifdef __cplusplus } #endif diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index 8050b85b0878e378bbe40098a4e2fbd0b1535b8a..4e5309bf07abfb653a1ff69d0ef789b5bfce6739 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -21,62 +21,56 @@ extern "C" { #include "index_fst_util.h" - typedef struct AutomationCtx AutomationCtx; -typedef enum AutomationType { - AUTOMATION_PREFIX, - AUTMMATION_MATCH -} AutomationType; +typedef enum AutomationType { AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType; typedef struct StartWith { - AutomationCtx *autoSelf; + AutomationCtx *autoSelf; } StartWith; typedef struct Complement { AutomationCtx *autoSelf; } Complement; -// automation +// automation typedef struct AutomationCtx { - AutomationType type; - void *stdata; - char *data; + AutomationType type; + void * stdata; + char * data; } AutomationCtx; - -typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY} ValueType; -typedef enum StartWithStateKind { Done, Running } StartWithStateKind; +typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType; +typedef enum StartWithStateKind { Done, Running } StartWithStateKind; typedef struct StartWithStateValue { StartWithStateKind kind; - ValueType type; + ValueType type; union { - int val; - char *ptr; + int val; + char * ptr; SArray *arr; // add more type - } ; + }; } StartWithStateValue; StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val); -StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv); -void startWithStateValueDestroy(void *sv); - +StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv); +void startWithStateValueDestroy(void *sv); typedef struct AutomationFunc { - void* (*start)(AutomationCtx *ctx) ; + void *(*start)(AutomationCtx *ctx); bool (*isMatch)(AutomationCtx *ctx, void *); bool (*canMatch)(AutomationCtx *ctx, void *data); - bool (*willAlwaysMatch)(AutomationCtx *ctx, void *state); - void* (*accept)(AutomationCtx *ctx, void *state, uint8_t byte); - void* (*acceptEof)(AutomationCtx *ct, void *state); -} AutomationFunc; + bool (*willAlwaysMatch)(AutomationCtx *ctx, void *state); + void *(*accept)(AutomationCtx *ctx, void *state, uint8_t byte); + void *(*acceptEof)(AutomationCtx *ct, void *state); +} AutomationFunc; AutomationCtx *automCtxCreate(void *data, AutomationType atype); -void automCtxDestroy(AutomationCtx *ctx); +void automCtxDestroy(AutomationCtx *ctx); -extern AutomationFunc automFuncs[]; +extern AutomationFunc automFuncs[]; #ifdef __cplusplus } #endif diff --git a/source/libs/index/inc/index_fst_common.h b/source/libs/index/inc/index_fst_common.h index 9c802faa33f52982a34e9d236956ec2e71c845f0..8335e437fb148b5f83f3b0db8fc2c6f63a4caa72 100644 --- a/source/libs/index/inc/index_fst_common.h +++ b/source/libs/index/inc/index_fst_common.h @@ -1,9 +1,9 @@ #ifndef __INDEX_FST_COMM_H__ #define __INDEX_FST_COMM_H__ - +#include "tutil.h" extern const uint8_t COMMON_INPUTS[]; -extern char const COMMON_INPUTS_INV[]; +extern const char COMMON_INPUTS_INV[]; #ifdef __cplusplus extern "C" { diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index ea090389bba1ff1a12b91ebcac2d15827916286a..b8a0384fb87584bbd8ad7da9193b19cf7165b772 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -22,20 +22,25 @@ extern "C" { #include "tfile.h" - -#define DefaultMem 1024*1024 +#define DefaultMem 1024 * 1024 static char tmpFile[] = "./index"; -typedef enum WriterType {TMemory, TFile} WriterType; +typedef enum WriterType { TMemory, TFile } WriterType; typedef struct WriterCtx { int (*write)(struct WriterCtx *ctx, uint8_t *buf, int len); int (*read)(struct WriterCtx *ctx, uint8_t *buf, int len); int (*flush)(struct WriterCtx *ctx); - WriterType type; + WriterType type; union { - int fd; - void *mem; + struct { + int fd; + bool readOnly; + } file; + struct { + int32_t capa; + char * buf; + } mem; }; int32_t offset; int32_t limit; @@ -45,35 +50,31 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len); static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len); static int writeCtxDoFlush(WriterCtx *ctx); -WriterCtx* writerCtxCreate(WriterType type, bool readOnly); -void writerCtxDestroy(WriterCtx *w); +WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity); +void writerCtxDestroy(WriterCtx *w); typedef uint32_t CheckSummer; - typedef struct FstCountingWriter { - void* wrt; // wrap any writer that counts and checksum bytes written - uint64_t count; - CheckSummer summer; + void * wrt; // wrap any writer that counts and checksum bytes written + uint64_t count; + CheckSummer summer; } FstCountingWriter; -int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len); +int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len); int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len); int fstCountingWriterFlush(FstCountingWriter *write); - uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write); -FstCountingWriter *fstCountingWriterCreate(void *wtr, bool readOnly); -void fstCountingWriterDestroy(FstCountingWriter *w); - +FstCountingWriter *fstCountingWriterCreate(void *wtr); +void fstCountingWriterDestroy(FstCountingWriter *w); -void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes); +void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes); uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n); - #define FST_WRITER_COUNT(writer) (writer->count) #define FST_WRITER_INTER_WRITER(writer) (writer->wtr) #define FST_WRITE_CHECK_SUMMER(writer) (writer->summer) @@ -83,5 +84,3 @@ uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n); #endif #endif - - diff --git a/source/libs/index/inc/index_fst_node.h b/source/libs/index/inc/index_fst_node.h index 87eb7cb746e8650f45014d4a7a354de71b32bba6..a2041fad408745dbc502bd1c206dcaff29f64de7 100644 --- a/source/libs/index/inc/index_fst_node.h +++ b/source/libs/index/inc/index_fst_node.h @@ -20,24 +20,24 @@ extern "C" { #endif -#include "index_fst_util.h" #include "index_fst_counting_writer.h" +#include "index_fst_util.h" -#define FST_BUILDER_NODE_IS_FINAL(bn) (bn->isFinal) -#define FST_BUILDER_NODE_TRANS_ISEMPTY(bn) (taosArrayGetSize(bn->trans) == 0) +#define FST_BUILDER_NODE_IS_FINAL(bn) (bn->isFinal) +#define FST_BUILDER_NODE_TRANS_ISEMPTY(bn) (taosArrayGetSize(bn->trans) == 0) #define FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn) (bn->finalOutput == 0) typedef struct FstTransition { - uint8_t inp; //The byte input associated with this transition. - Output out; //The output associated with this transition - CompiledAddr addr; //The address of the node that this transition points to + uint8_t inp; // The byte input associated with this transition. + Output out; // The output associated with this transition + CompiledAddr addr; // The address of the node that this transition points to } FstTransition; typedef struct FstBuilderNode { - bool isFinal; - Output finalOutput; + bool isFinal; + Output finalOutput; SArray *trans; // -} FstBuilderNode; +} FstBuilderNode; FstBuilderNode *fstBuilderNodeDefault(); @@ -45,8 +45,9 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src); void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src); -//bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); -bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2); +// bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, +// CompiledAddr lastAddr, CompiledAddr startAddr); +bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2); void fstBuilderNodeDestroy(FstBuilderNode *node); diff --git a/source/libs/index/inc/index_fst_registry.h b/source/libs/index/inc/index_fst_registry.h index 1d89e57e525ea4b5b30242481b1ddad962a954e9..1b0922c7241cfa1c5f92863f459b5cf999baac99 100644 --- a/source/libs/index/inc/index_fst_registry.h +++ b/source/libs/index/inc/index_fst_registry.h @@ -19,49 +19,48 @@ extern "C" { #endif +#include "index_fst_node.h" #include "index_fst_util.h" #include "tarray.h" -#include "index_fst_node.h" typedef struct FstRegistryCell { - CompiledAddr addr; - FstBuilderNode *node; + CompiledAddr addr; + FstBuilderNode *node; } FstRegistryCell; #define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS) -#define FST_REGISTRY_CELL_INSERT(cell, tAddr) do {cell->addr = tAddr;} while(0) - +#define FST_REGISTRY_CELL_INSERT(cell, tAddr) \ + do { \ + cell->addr = tAddr; \ + } while (0) -//typedef struct FstRegistryCache { -// SArray *cells; +// typedef struct FstRegistryCache { +// SArray *cells; // uint32_t start; // uint32_t end; //} FstRegistryCache; -typedef enum {FOUND, NOTFOUND, REJECTED} FstRegistryEntryState; +typedef enum { FOUND, NOTFOUND, REJECTED } FstRegistryEntryState; typedef struct FstRegistryEntry { FstRegistryEntryState state; - CompiledAddr addr; - FstRegistryCell *cell; -} FstRegistryEntry; + CompiledAddr addr; + FstRegistryCell * cell; +} FstRegistryEntry; - - -// Registry relation function +// Registry relation function typedef struct FstRegistry { - SArray *table; // - uint64_t tableSize; // num of rows - uint64_t mruSize; // num of columns -} FstRegistry; - -// -FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize); -void fstRegistryDestroy(FstRegistry *registry); + SArray * table; // + uint64_t tableSize; // num of rows + uint64_t mruSize; // num of columns +} FstRegistry; +// +FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize); +void fstRegistryDestroy(FstRegistry *registry); -FstRegistryEntry* fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode); -void fstRegistryEntryDestroy(FstRegistryEntry *entry); +FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode); +void fstRegistryEntryDestroy(FstRegistryEntry *entry); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_fst_util.h b/source/libs/index/inc/index_fst_util.h index 24b2508678e4664704ffe4425f779d84a876c296..c41977e6377f97a007c221cc63bf28182cae40ba 100644 --- a/source/libs/index/inc/index_fst_util.h +++ b/source/libs/index/inc/index_fst_util.h @@ -13,7 +13,6 @@ * along with this program. If not, see . */ - #ifndef __INDEX_FST_UTIL_H__ #define __INDEX_FST_UTIL_H__ @@ -21,16 +20,15 @@ extern "C" { #endif -#include "tarray.h" #include "index_fst_common.h" +#include "tarray.h" typedef uint64_t FstType; -typedef uint64_t CompiledAddr; -typedef uint64_t Output; -typedef uint8_t PackSizes; +typedef uint64_t CompiledAddr; +typedef uint64_t Output; +typedef uint8_t PackSizes; - -//A sentinel value used to indicate an empty final state +// A sentinel value used to indicate an empty final state extern const CompiledAddr EMPTY_ADDRESS; /// A sentinel value used to indicate an invalid state. extern const CompiledAddr NONE_ADDRESS; @@ -38,9 +36,9 @@ extern const CompiledAddr NONE_ADDRESS; // This version number is written to every finite state transducer created by // this version When a finite state transducer is read, its version number is // checked against this value. -extern const uint64_t VERSION; -// The threshold (in number of transitions) at which an index is created for -// a node's transitions. This speeds up lookup time at the expense of FST size +extern const uint64_t VERSION; +// The threshold (in number of transitions) at which an index is created for +// a node's transitions. This speeds up lookup time at the expense of FST size extern const uint64_t TRANS_INDEX_THRESHOLD; // high 4 bits is transition address packed size. @@ -48,73 +46,75 @@ extern const uint64_t TRANS_INDEX_THRESHOLD; // // `0` is a legal value which means there are no transitions/outputs - -#define FST_SET_TRANSITION_PACK_SIZE(v, sz) do {v = (v & 0b00001111) | (sz << 4); } while(0) -#define FST_GET_TRANSITION_PACK_SIZE(v) (((v) & 0b11110000) >> 4) -#define FST_SET_OUTPUT_PACK_SIZE(v, sz) do { v = (v & 0b11110000) | sz; } while(0) -#define FST_GET_OUTPUT_PACK_SIZE(v) ((v) & 0b00001111) - -#define COMMON_INPUT(idx) COMMON_INPUTS_INV[(idx) - 1] - -#define COMMON_INDEX(v, max, val) do { \ - val = ((uint16_t)COMMON_INPUTS[v] + 1)%256; \ - val = val > max ? 0: val; \ -} while(0) - - -//uint8_t commonInput(uint8_t idx); -//uint8_t commonIdx(uint8_t v, uint8_t max); - -uint8_t packSize(uint64_t n); +#define FST_SET_TRANSITION_PACK_SIZE(v, sz) \ + do { \ + v = (v & 0b00001111) | (sz << 4); \ + } while (0) +#define FST_GET_TRANSITION_PACK_SIZE(v) (((v)&0b11110000) >> 4) +#define FST_SET_OUTPUT_PACK_SIZE(v, sz) \ + do { \ + v = (v & 0b11110000) | sz; \ + } while (0) +#define FST_GET_OUTPUT_PACK_SIZE(v) ((v)&0b00001111) + +#define COMMON_INPUT(idx) COMMON_INPUTS_INV[(idx)-1] + +#define COMMON_INDEX(v, max, val) \ + do { \ + val = ((uint16_t)COMMON_INPUTS[v] + 1) % 256; \ + val = val > max ? 0 : val; \ + } while (0) + +// uint8_t commonInput(uint8_t idx); +// uint8_t commonIdx(uint8_t v, uint8_t max); + +uint8_t packSize(uint64_t n); uint64_t unpackUint64(uint8_t *ch, uint8_t sz); uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr); CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr); - typedef struct FstString { - uint8_t *data; + uint8_t *data; uint32_t len; - int32_t ref; + int32_t ref; } FstString; typedef struct FstSlice { - FstString *str; - int32_t start; - int32_t end; + FstString *str; + int32_t start; + int32_t end; } FstSlice; FstSlice fstSliceCreate(uint8_t *data, uint64_t len); FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end); FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end); -bool fstSliceIsEmpty(FstSlice *s); -int fstSliceCompare(FstSlice *s1, FstSlice *s2); -void fstSliceDestroy(FstSlice *s); -uint8_t *fstSliceData(FstSlice *s, int32_t *sz); +bool fstSliceIsEmpty(FstSlice *s); +int fstSliceCompare(FstSlice *s1, FstSlice *s2); +void fstSliceDestroy(FstSlice *s); +uint8_t *fstSliceData(FstSlice *s, int32_t *sz); #define FST_SLICE_LEN(s) (s->end - s->start + 1) -//// stack +//// stack // -//typedef (*StackFreeElemFn)(void *elem); +// typedef (*StackFreeElemFn)(void *elem); // -//typedef struct FstStack { -// void *first; -// void *end; -// size_t elemSize; +// typedef struct FstStack { +// void *first; +// void *end; +// size_t elemSize; // size_t nElem; // StackFreeElemFn fn; //} FstStack; // // -//FstStack* fstStackCreate(size_t elemSize, stackFreeElem); -//void *fstStackPush(FstStack *s, void *elem); -//void *fstStackTop(FstStack *s); -//size_t fstStackLen(FstStack *s); -//void fstStackDestory(FstStack *); +// FstStack* fstStackCreate(size_t elemSize, stackFreeElem); +// void *fstStackPush(FstStack *s, void *elem); +// void *fstStackTop(FstStack *s); +// size_t fstStackLen(FstStack *s); +// void fstStackDestory(FstStack *); // - - #ifdef __cplusplus } #endif diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index c3f4bd25e5e6a792120f1ac8880bfbc57915dfef..7c6261aacf7ad14d58c883443cfadc16d50269d4 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -17,30 +17,97 @@ #include "index.h" #include "indexInt.h" +#include "index_fst.h" +#include "index_fst_counting_writer.h" +#include "index_tfile.h" #include "tlockfree.h" -#include "tskiplist.h" #ifdef __cplusplus extern "C" { #endif +// tfile header +// |<---suid--->|<---version--->|<--colLen-->|<-colName->|<---type-->| +// |<-uint64_t->|<---int32_t--->|<--int32_t->|<-colLen-->|<-uint8_t->| + +typedef struct TFileReadHeader { + uint64_t suid; + int32_t version; + char colName[128]; // + uint8_t colType; +} TFileReadHeader; + +#define TFILE_HEADER_SIZE (sizeof(TFILE_HEADER_SIZE) + sizeof(uint32_t)); +#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t)) + +typedef struct TFileCacheKey { + uint64_t suid; + uint8_t colType; + int32_t version; + const char *colName; + int32_t nColName; +} TFileCacheKey; + +// table cache +// refactor to LRU cache later +typedef struct TFileCache { + SHashObj *tableCache; + int16_t capacity; + // add more param +} TFileCache; + +typedef struct TFileWriter { + FstBuilder *fb; + WriterCtx * ctx; +} TFileWriter; + +typedef struct TFileReader { + T_REF_DECLARE() + Fst * fst; + WriterCtx * ctx; + TFileReadHeader header; +} TFileReader; + typedef struct IndexTFile { - T_REF_DECLARE() + char * path; + TFileCache * cache; + TFileWriter *tw; } IndexTFile; +typedef struct TFileWriterOpt { + uint64_t suid; + int8_t colType; + char * colName; + int32_t nColName; + int32_t version; +} TFileWriterOpt; + +typedef struct TFileReaderOpt { + uint64_t suid; + char * colName; + int32_t nColName; +} TFileReaderOpt; +// tfile cache, manage tindex reader +TFileCache * tfileCacheCreate(const char *path); +void tfileCacheDestroy(TFileCache *tcache); +TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key); +void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader); -IndexTFile *indexTFileCreate(); +TFileReader *tfileReaderCreate(); +void TFileReaderDestroy(TFileReader *reader); +TFileWriter *tfileWriterCreate(const char *suid, const char *colName); +void tfileWriterDestroy(TFileWriter *tw); -int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result); +// +IndexTFile *indexTFileCreate(const char *path); +int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid); +int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result); #ifdef __cplusplus } - #endif - - #endif diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h new file mode 100644 index 0000000000000000000000000000000000000000..f708e71f57364a673d3f1c7327f932464702350c --- /dev/null +++ b/source/libs/index/inc/index_util.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef __INDEX_UTIL_H__ +#define __INDEX_UTIL_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ + do { \ + memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ + buf += sizeof(key->mem); \ + } while (0) + +#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ + do { \ + memcpy((void *)buf, (void *)key->mem, len); \ + buf += len; \ + } while (0) + +#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ + do { \ + type c = var; \ + assert(sizeof(var) == sizeof(type)); \ + memcpy((void *)buf, (void *)&c, sizeof(c)); \ + buf += sizeof(c); \ + } while (0) + +#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ + do { \ + memcpy((void *)buf, (void *)var, len); \ + buf += len; \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index ec83e84a3b22541fb1ba7906551d94a906917ea0..cc68324017f44960d14537c782806a8351c65f89 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -26,105 +26,108 @@ static int uidCompare(const void *a, const void *b) { uint64_t u1 = *(uint64_t *)a; uint64_t u2 = *(uint64_t *)b; - if (u1 == u2) { return 0; } - else { return u1 < u2 ? -1 : 1; } + if (u1 == u2) { + return 0; + } else { + return u1 < u2 ? -1 : 1; + } } typedef struct SIdxColInfo { - int colId; // generated by index internal + int colId; // generated by index internal int cVersion; -} SIdxColInfo; +} SIdxColInfo; static pthread_once_t isInit = PTHREAD_ONCE_INIT; -static void indexInit(); - +static void indexInit(); static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *term, SArray **result); static int indexMergeCacheIntoTindex(SIndex *sIdx); static void indexInterResultsDestroy(SArray *results); -static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *finalResult); +static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *finalResult); int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) { pthread_once(&isInit, indexInit); SIndex *sIdx = calloc(1, sizeof(SIndex)); - if (sIdx == NULL) { return -1; } + if (sIdx == NULL) { + return -1; + } -#ifdef USE_LUCENE - index_t *index = index_open(path); +#ifdef USE_LUCENE + index_t *index = index_open(path); sIdx->index = index; #endif - sIdx->cache = (void*)indexCacheCreate(); - sIdx->tindex = NULL; - sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - sIdx->colId = 1; - sIdx->cVersion = 1; + sIdx->cache = (void *)indexCacheCreate(); + sIdx->tindex = NULL; + sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + sIdx->colId = 1; + sIdx->cVersion = 1; pthread_mutex_init(&sIdx->mtx, NULL); - *index = sIdx; - return 0; + *index = sIdx; + return 0; } void indexClose(SIndex *sIdx) { -#ifdef USE_LUCENE - index_close(sIdex->index); +#ifdef USE_LUCENE + index_close(sIdex->index); sIdx->index = NULL; #endif #ifdef USE_INVERTED_INDEX indexCacheDestroy(sIdx->cache); - taosHashCleanup(sIdx->colObj); + taosHashCleanup(sIdx->colObj); pthread_mutex_destroy(&sIdx->mtx); #endif - free(sIdx); + free(sIdx); return; } -int indexPut(SIndex *index, SIndexMultiTerm * fVals, uint64_t uid) { - -#ifdef USE_LUCENE - index_document_t *doc = index_document_create(); - - char buf[16] = {0}; - sprintf(buf, "%d", uid); - - for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm *p = taosArrayGetP(fVals, i); - index_document_add(doc, (const char *)(p->key), p->nKey, (const char *)(p->val), p->nVal, 1); - } - index_document_add(doc, NULL, 0, buf, strlen(buf), 0); +int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) { +#ifdef USE_LUCENE + index_document_t *doc = index_document_create(); + + char buf[16] = {0}; + sprintf(buf, "%d", uid); - index_put(index->index, doc); - index_document_destroy(doc); + for (int i = 0; i < taosArrayGetSize(fVals); i++) { + SIndexTerm *p = taosArrayGetP(fVals, i); + index_document_add(doc, (const char *)(p->key), p->nKey, (const char *)(p->val), p->nVal, 1); + } + index_document_add(doc, NULL, 0, buf, strlen(buf), 0); + + index_put(index->index, doc); + index_document_destroy(doc); #endif #ifdef USE_INVERTED_INDEX - - //TODO(yihao): reduce the lock range - pthread_mutex_lock(&index->mtx); + + // TODO(yihao): reduce the lock range + pthread_mutex_lock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm *p = taosArrayGetP(fVals, i); + SIndexTerm * p = taosArrayGetP(fVals, i); SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); if (fi == NULL) { SIdxColInfo tfi = {.colId = index->colId}; - index->cVersion++; + index->cVersion++; index->colId++; - taosHashPut(index->colObj, p->colName, p->nColName, &tfi, sizeof(tfi)); + taosHashPut(index->colObj, p->colName, p->nColName, &tfi, sizeof(tfi)); } else { - //TODO, del + // TODO, del } - } + } pthread_mutex_unlock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm *p = taosArrayGetP(fVals, i); + SIndexTerm * p = taosArrayGetP(fVals, i); SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); - assert(fi != NULL); - int32_t colId = fi->colId; + assert(fi != NULL); + int32_t colId = fi->colId; int32_t version = index->cVersion; - int ret = indexCachePut(index->cache, p, colId, version, uid); + int ret = indexCachePut(index->cache, p, colId, version, uid); if (ret != 0) { - return ret; + return ret; } } #endif @@ -132,29 +135,29 @@ int indexPut(SIndex *index, SIndexMultiTerm * fVals, uint64_t uid) { return 0; } int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) { -#ifdef USE_LUCENE - EIndexOperatorType opera = multiQuerys->opera; +#ifdef USE_LUCENE + EIndexOperatorType opera = multiQuerys->opera; - int nQuery = taosArrayGetSize(multiQuerys->query); + int nQuery = taosArrayGetSize(multiQuerys->query); char **fields = malloc(sizeof(char *) * nQuery); - char **keys = malloc(sizeof(char *) * nQuery); - int *types = malloc(sizeof(int) * nQuery); + char **keys = malloc(sizeof(char *) * nQuery); + int * types = malloc(sizeof(int) * nQuery); for (int i = 0; i < nQuery; i++) { - SIndexTermQuery *p = taosArrayGet(multiQuerys->query, i); - SIndexTerm *term = p->field_value; - - fields[i] = calloc(1, term->nKey + 1); - keys[i] = calloc(1, term->nVal + 1); - - memcpy(fields[i], term->key, term->nKey); - memcpy(keys[i], term->val, term->nVal); - types[i] = (int)(p->type); - } - int *tResult = NULL; - int tsz= 0; + SIndexTermQuery *p = taosArrayGet(multiQuerys->query, i); + SIndexTerm * term = p->field_value; + + fields[i] = calloc(1, term->nKey + 1); + keys[i] = calloc(1, term->nVal + 1); + + memcpy(fields[i], term->key, term->nKey); + memcpy(keys[i], term->val, term->nVal); + types[i] = (int)(p->type); + } + int *tResult = NULL; + int tsz = 0; index_multi_search(index->index, (const char **)fields, (const char **)keys, types, nQuery, opera, &tResult, &tsz); - + for (int i = 0; i < tsz; i++) { taosArrayPush(result, &tResult[i]); } @@ -169,57 +172,55 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result #endif #ifdef USE_INVERTED_INDEX - EIndexOperatorType opera = multiQuerys->opera; // relation of querys - + EIndexOperatorType opera = multiQuerys->opera; // relation of querys + SArray *interResults = taosArrayInit(4, POINTER_BYTES); - int nQuery = taosArrayGetSize(multiQuerys->query); + int nQuery = taosArrayGetSize(multiQuerys->query); for (size_t i = 0; i < nQuery; i++) { - SIndexTermQuery *qTerm = taosArrayGet(multiQuerys->query, i); - SArray *tResult = NULL; - indexTermSearch(index, qTerm, &tResult); - taosArrayPush(interResults, (void *)&tResult); - } + SIndexTermQuery *qTerm = taosArrayGet(multiQuerys->query, i); + SArray * tResult = NULL; + indexTermSearch(index, qTerm, &tResult); + taosArrayPush(interResults, (void *)&tResult); + } indexMergeFinalResults(interResults, opera, result); indexInterResultsDestroy(interResults); - + #endif return 1; } - - int indexDelete(SIndex *index, SIndexMultiTermQuery *query) { #ifdef USE_INVERTED_INDEX #endif - + return 1; } -int indexRebuild(SIndex *index, SIndexOpts *opts) { +int indexRebuild(SIndex *index, SIndexOpts *opts){ #ifdef USE_INVERTED_INDEX #endif } - SIndexOpts *indexOptsCreate() { -#ifdef USE_LUCENE +#ifdef USE_LUCENE #endif -return NULL; + return NULL; } -void indexOptsDestroy(SIndexOpts *opts) { -#ifdef USE_LUCENE +void indexOptsDestroy(SIndexOpts *opts){ +#ifdef USE_LUCENE #endif -} -/* - * @param: oper - * -*/ +} /* + * @param: oper + * + */ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery *p = (SIndexMultiTermQuery *)malloc(sizeof(SIndexMultiTermQuery)); - if (p == NULL) { return NULL; } - p->opera = opera; - p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); + if (p == NULL) { + return NULL; + } + p->opera = opera; + p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); return p; } void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) { @@ -227,25 +228,27 @@ void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) { SIndexTermQuery *p = (SIndexTermQuery *)taosArrayGet(pQuery->query, i); indexTermDestroy(p->term); } - taosArrayDestroy(pQuery->query); + taosArrayDestroy(pQuery->query); free(pQuery); }; -int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType){ - SIndexTermQuery q = {.qType = qType, .term = term}; +int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType) { + SIndexTermQuery q = {.qType = qType, .term = term}; taosArrayPush(pQuery->query, &q); return 0; } +SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, + int32_t nColName, const char *colVal, int32_t nColVal) { + SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm))); + if (t == NULL) { + return NULL; + } -SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, int32_t nColName, const char *colVal, int32_t nColVal) { - SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm))); - if (t == NULL) { return NULL; } - - t->suid = suid; - t->operType= oper; + t->suid = suid; + t->operType = oper; t->colType = colType; - t->colName = (char *)calloc(1, nColName + 1); + t->colName = (char *)calloc(1, nColName + 1); memcpy(t->colName, colName, nColName); t->nColName = nColName; @@ -258,15 +261,13 @@ void indexTermDestroy(SIndexTerm *p) { free(p->colName); free(p->colVal); free(p); -} - -SIndexMultiTerm *indexMultiTermCreate() { - return taosArrayInit(4, sizeof(SIndexTerm *)); } +SIndexMultiTerm *indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm *)); } + int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) { - taosArrayPush(terms, &term); - return 0; + taosArrayPush(terms, &term); + return 0; } void indexMultiTermDestroy(SIndexMultiTerm *terms) { for (int32_t i = 0; i < taosArrayGetSize(terms); i++) { @@ -277,40 +278,40 @@ void indexMultiTermDestroy(SIndexMultiTerm *terms) { } void indexInit() { - //do nothing + // do nothing } static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result) { - int32_t version = -1; - int16_t colId = -1; + int32_t version = -1; + int16_t colId = -1; SIdxColInfo *colInfo = NULL; SIndexTerm *term = query->term; const char *colName = term->colName; - int32_t nColName = term->nColName; + int32_t nColName = term->nColName; - pthread_mutex_lock(&sIdx->mtx); - colInfo = taosHashGet(sIdx->colObj, colName, nColName); + pthread_mutex_lock(&sIdx->mtx); + colInfo = taosHashGet(sIdx->colObj, colName, nColName); if (colInfo == NULL) { - pthread_mutex_unlock(&sIdx->mtx); - return -1; + pthread_mutex_unlock(&sIdx->mtx); + return -1; } - colId = colInfo->colId; + colId = colInfo->colId; version = colInfo->cVersion; - pthread_mutex_unlock(&sIdx->mtx); - + pthread_mutex_unlock(&sIdx->mtx); + *result = taosArrayInit(4, sizeof(uint64_t)); - //TODO: iterator mem and tidex - STermValueType s; + // TODO: iterator mem and tidex + STermValueType s; if (0 == indexCacheSearch(sIdx->cache, query, colId, version, *result, &s)) { if (s == kTypeDeletion) { indexInfo("col: %s already drop by other opera", term->colName); - // coloum already drop by other oper, no need to query tindex + // coloum already drop by other oper, no need to query tindex return 0; } else { if (0 != indexTFileSearch(sIdx->tindex, query, *result)) { - indexError("corrupt at index(TFile) col:%s val: %s", term->colName, term->colVal); - return -1; - } + indexError("corrupt at index(TFile) col:%s val: %s", term->colName, term->colVal); + return -1; + } } } else { indexError("corrupt at index(cache) col:%s val: %s", term->colName, term->colVal); @@ -319,35 +320,40 @@ static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result return 0; } static void indexInterResultsDestroy(SArray *results) { - if (results == NULL) { return; } + if (results == NULL) { + return; + } size_t sz = taosArrayGetSize(results); for (size_t i = 0; i < sz; i++) { SArray *p = taosArrayGetP(results, i); - taosArrayDestroy(p); - } + taosArrayDestroy(p); + } taosArrayDestroy(results); - } static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *fResults) { - //refactor, merge interResults into fResults by oType - SArray *first = taosArrayGetP(interResults, 0); - taosArraySort(first, uidCompare); + // refactor, merge interResults into fResults by oType + SArray *first = taosArrayGetP(interResults, 0); + taosArraySort(first, uidCompare); + taosArrayRemoveDuplicate(first, uidCompare, NULL); if (oType == MUST) { - + // just one column index, enhance later + taosArrayAddAll(fResults, first); } else if (oType == SHOULD) { - + // just one column index, enhance later + taosArrayAddAll(fResults, first); // tag1 condistion || tag2 condition } else if (oType == NOT) { - - // not use currently + // just one column index, enhance later + taosArrayAddAll(fResults, first); + // not use currently } return 0; } static int indexMergeCacheIntoTindex(SIndex *sIdx) { if (sIdx == NULL) { - return -1; + return -1; } - indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); + indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); return 0; } diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 3c52275a4c96f2f0f12f6fae012c9681f2a37c99..dd8a8bcbb6e31031429e7181793fafc65cc004c8 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -14,160 +14,154 @@ */ #include "index_cache.h" +#include "index_util.h" #include "tcompare.h" -#define MAX_INDEX_KEY_LEN 256// test only, change later +#define MAX_INDEX_KEY_LEN 256 // test only, change later -// ref index_cache.h:22 -#define CACHE_KEY_LEN(p) (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType)) +// ref index_cache.h:22 +#define CACHE_KEY_LEN(p) \ + (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + \ + sizeof(p->operType)) -static char* getIndexKey(const void *pData) { - return NULL; -} +static char * getIndexKey(const void *pData) { return NULL; } static int32_t compareKey(const void *l, const void *r) { char *lp = (char *)l; char *rp = (char *)r; // skip total len, not compare - int32_t ll, rl; // len + int32_t ll, rl; // len memcpy(&ll, lp, sizeof(int32_t)); memcpy(&rl, rp, sizeof(int32_t)); - lp += sizeof(int32_t); + lp += sizeof(int32_t); rp += sizeof(int32_t); - + // compare field id - int16_t lf, rf; // field id + int16_t lf, rf; // field id memcpy(&lf, lp, sizeof(lf)); memcpy(&rf, rp, sizeof(rf)); if (lf != rf) { - return lf < rf ? -1: 1; + return lf < rf ? -1 : 1; } lp += sizeof(lf); rp += sizeof(rf); // compare field type - int8_t lft, rft; + int8_t lft, rft; memcpy(&lft, lp, sizeof(lft)); memcpy(&rft, rp, sizeof(rft)); lp += sizeof(lft); rp += sizeof(rft); assert(rft == rft); - - // skip value len + + // skip value len int32_t lfl, rfl; - memcpy(&lfl, lp, sizeof(lfl)); - memcpy(&rfl, rp, sizeof(rfl)); + memcpy(&lfl, lp, sizeof(lfl)); + memcpy(&rfl, rp, sizeof(rfl)); lp += sizeof(lfl); rp += sizeof(rfl); - - // compare value + + // compare value int32_t i, j; for (i = 0, j = 0; i < lfl && j < rfl; i++, j++) { - if (lp[i] == rp[j]) { continue; } - else { return lp[i] < rp[j] ? -1 : 1;} + if (lp[i] == rp[j]) { + continue; + } else { + return lp[i] < rp[j] ? -1 : 1; + } + } + if (i < lfl) { + return 1; + } else if (j < rfl) { + return -1; } - if (i < lfl) { return 1;} - else if (j < rfl) { return -1; } lp += lfl; - rp += rfl; + rp += rfl; - // skip uid + // skip uid uint64_t lu, ru; - memcpy(&lu, lp, sizeof(lu)); + memcpy(&lu, lp, sizeof(lu)); memcpy(&ru, rp, sizeof(ru)); lp += sizeof(lu); rp += sizeof(ru); - + // compare version, desc order int32_t lv, rv; memcpy(&lv, lp, sizeof(lv)); memcpy(&rv, rp, sizeof(rv)); if (lv != rv) { - return lv > rv ? -1 : 1; - } + return lv > rv ? -1 : 1; + } lp += sizeof(lv); rp += sizeof(rv); // not care item type - return 0; - -} + return 0; +} IndexCache *indexCacheCreate() { IndexCache *cache = calloc(1, sizeof(IndexCache)); - cache->skiplist = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); + cache->skiplist = tSkipListCreate( + MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); return cache; - } void indexCacheDestroy(void *cache) { - IndexCache *pCache = cache; - if (pCache == NULL) { return; } + IndexCache *pCache = cache; + if (pCache == NULL) { + return; + } tSkipListDestroy(pCache->skiplist); free(pCache); } int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid) { - if (cache == NULL) { return -1;} + if (cache == NULL) { + return -1; + } IndexCache *pCache = cache; - // encode data - int32_t total = CACHE_KEY_LEN(term); - - char *buf = calloc(1, total); - char *p = buf; - - memcpy(p, &total, sizeof(total)); - p += sizeof(total); + int32_t total = CACHE_KEY_LEN(term); + char * buf = calloc(1, total); + char * p = buf; - memcpy(p, &colId, sizeof(colId)); - p += sizeof(colId); + SERIALIZE_VAR_TO_BUF(p, total, int32_t); + SERIALIZE_VAR_TO_BUF(p, colId, int16_t); - memcpy(p, &term->colType, sizeof(term->colType)); - p += sizeof(term->colType); - - memcpy(p, &term->nColVal, sizeof(term->nColVal)); - p += sizeof(term->nColVal); - memcpy(p, term->colVal, term->nColVal); - p += term->nColVal; + SERIALIZE_MEM_TO_BUF(p, term, colType); + SERIALIZE_MEM_TO_BUF(p, term, nColVal); + SERIALIZE_STR_MEM_TO_BUF(p, term, colVal, term->nColVal); - memcpy(p, &version, sizeof(version)); - p += sizeof(version); + SERIALIZE_VAR_TO_BUF(p, version, int32_t); + SERIALIZE_VAR_TO_BUF(p, uid, uint64_t); - memcpy(p, &uid, sizeof(uid)); - p += sizeof(uid); + SERIALIZE_MEM_TO_BUF(p, term, operType); - memcpy(p, &term->operType, sizeof(term->operType)); - p += sizeof(term->operType); - - tSkipListPut(pCache->skiplist, (void *)buf); + tSkipListPut(pCache->skiplist, (void *)buf); return 0; // encode end - } int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { IndexCache *pCache = cache; return 0; } -int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) { - if (cache == NULL) { return -1; } - IndexCache *pCache = cache; - SIndexTerm *term = query->term; - EIndexQueryType qtype = query->qType; - - int32_t keyLen = CACHE_KEY_LEN(term); +int indexCacheSearch( + void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) { + if (cache == NULL) { + return -1; + } + IndexCache * pCache = cache; + SIndexTerm * term = query->term; + EIndexQueryType qtype = query->qType; + + int32_t keyLen = CACHE_KEY_LEN(term); char *buf = calloc(1, keyLen); if (qtype == QUERY_TERM) { - } else if (qtype == QUERY_PREFIX) { - } else if (qtype == QUERY_SUFFIX) { - } else if (qtype == QUERY_REGEX) { - } - + return 0; } - diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 7aaa498864fff74dccccdb2446c190b24272fcb2..adb002c2b74301403e599a71935a66c7d5a6759b 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -14,83 +14,84 @@ */ #include "index_fst.h" -#include "tcoding.h" -#include "tchecksum.h" -#include "indexInt.h" +#include "indexInt.h" #include "index_fst_automation.h" - +#include "tchecksum.h" +#include "tcoding.h" static void fstPackDeltaIn(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr; - fstCountingWriterPackUintIn(wrt, deltaAddr, nBytes); + fstCountingWriterPackUintIn(wrt, deltaAddr, nBytes); } static uint8_t fstPackDetla(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) { uint8_t nBytes = packDeltaSize(nodeAddr, transAddr); - fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes); + fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes); return nBytes; } FstUnFinishedNodes *fstUnFinishedNodesCreate() { FstUnFinishedNodes *nodes = malloc(sizeof(FstUnFinishedNodes)); - if (nodes == NULL) { return NULL; } + if (nodes == NULL) { + return NULL; + } nodes->stack = (SArray *)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); fstUnFinishedNodesPushEmpty(nodes, false); return nodes; } -void unFinishedNodeDestroyElem(void* elem) { - FstBuilderNodeUnfinished *b = (FstBuilderNodeUnfinished*)elem; - fstBuilderNodeDestroy(b->node); - free(b->last); +void unFinishedNodeDestroyElem(void *elem) { + FstBuilderNodeUnfinished *b = (FstBuilderNodeUnfinished *)elem; + fstBuilderNodeDestroy(b->node); + free(b->last); b->last = NULL; -} +} void fstUnFinishedNodesDestroy(FstUnFinishedNodes *nodes) { - if (nodes == NULL) { return; } + if (nodes == NULL) { + return; + } - taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); + taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); free(nodes); } void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) { FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); - node->isFinal = isFinal; + node->isFinal = isFinal; node->finalOutput = 0; - node->trans = taosArrayInit(16, sizeof(FstTransition)); + node->trans = taosArrayInit(16, sizeof(FstTransition)); - FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; + FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; taosArrayPush(nodes->stack, &un); - } FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes) { assert(taosArrayGetSize(nodes->stack) == 1); FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); - assert(un->last == NULL); - return un->node; + assert(un->last == NULL); + return un->node; } FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr) { FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); fstBuilderNodeUnfinishedLastCompiled(un, addr); - //free(un->last); // TODO add func FstLastTransitionFree() - //un->last = NULL; - return un->node; + // free(un->last); // TODO add func FstLastTransitionFree() + // un->last = NULL; + return un->node; } FstBuilderNode *fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes) { FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); - assert(un->last == NULL); - return un->node; - + assert(un->last == NULL); + return un->node; } void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *nodes, Output out) { FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, 0); - un->node->isFinal = true; + un->node->isFinal = true; un->node->finalOutput = out; - //un->node->trans = NULL; -} + // un->node->trans = NULL; +} void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr) { - size_t sz = taosArrayGetSize(nodes->stack) - 1; + size_t sz = taosArrayGetSize(nodes->stack) - 1; FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz); fstBuilderNodeUnfinishedLastCompiled(un, addr); } @@ -99,181 +100,177 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output if (fstSliceIsEmpty(s)) { return; } - size_t sz = taosArrayGetSize(nodes->stack) - 1; + size_t sz = taosArrayGetSize(nodes->stack) - 1; FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz); assert(un->last == NULL); - //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); - //trn->inp = s->data[s->start]; - //trn->out = out; - int32_t len = 0; + // FstLastTransition *trn = malloc(sizeof(FstLastTransition)); + // trn->inp = s->data[s->start]; + // trn->out = out; + int32_t len = 0; uint8_t *data = fstSliceData(s, &len); - un->last = fstLastTransitionCreate(data[0], out); + un->last = fstLastTransitionCreate(data[0], out); for (uint64_t i = 1; i < len; i++) { FstBuilderNode *n = malloc(sizeof(FstBuilderNode)); - n->isFinal = false; + n->isFinal = false; n->finalOutput = 0; - n->trans = taosArrayInit(16, sizeof(FstTransition)); - - //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); - //trn->inp = s->data[i]; - //trn->out = out; + n->trans = taosArrayInit(16, sizeof(FstTransition)); + + // FstLastTransition *trn = malloc(sizeof(FstLastTransition)); + // trn->inp = s->data[i]; + // trn->out = out; FstLastTransition *trn = fstLastTransitionCreate(data[i], 0); - FstBuilderNodeUnfinished un = {.node = n, .last = trn}; - taosArrayPush(nodes->stack, &un); + FstBuilderNodeUnfinished un = {.node = n, .last = trn}; + taosArrayPush(nodes->stack, &un); } - fstUnFinishedNodesPushEmpty(nodes, true); + fstUnFinishedNodesPushEmpty(nodes, true); } - uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) { FstSlice *s = &bs; - size_t ssz = taosArrayGetSize(node->stack); // stack size + size_t ssz = taosArrayGetSize(node->stack); // stack size uint64_t count = 0; - int32_t lsz; // data len - uint8_t *data = fstSliceData(s, &lsz); + int32_t lsz; // data len + uint8_t *data = fstSliceData(s, &lsz); for (size_t i = 0; i < ssz && i < lsz; i++) { - FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); + FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); if (un->last->inp == data[i]) { count++; } else { break; - } + } } return count; } uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out) { FstSlice *s = &bs; - size_t lsz = (size_t)(s->end - s->start + 1); // data len - size_t ssz = taosArrayGetSize(node->stack); // stack size + size_t lsz = (size_t)(s->end - s->start + 1); // data len + size_t ssz = taosArrayGetSize(node->stack); // stack size *out = in; uint64_t i = 0; for (i = 0; i < lsz && i < ssz; i++) { FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); - FstLastTransition *t = un->last; - uint64_t addPrefix = 0; - uint8_t *data = fstSliceData(s, NULL); + FstLastTransition *t = un->last; + uint64_t addPrefix = 0; + uint8_t * data = fstSliceData(s, NULL); if (t && t->inp == data[i]) { - uint64_t commPrefix = MIN(t->out, *out); - uint64_t tAddPrefix = t->out - commPrefix; - (*out) = (*out) - commPrefix; + uint64_t commPrefix = MIN(t->out, *out); + uint64_t tAddPrefix = t->out - commPrefix; + (*out) = (*out) - commPrefix; t->out = commPrefix; - addPrefix = tAddPrefix; + addPrefix = tAddPrefix; } else { - break; + break; } if (addPrefix != 0) { if (i + 1 < ssz) { FstBuilderNodeUnfinished *unf = taosArrayGet(node->stack, i + 1); - fstBuilderNodeUnfinishedAddOutputPrefix(unf, addPrefix); + fstBuilderNodeUnfinishedAddOutputPrefix(unf, addPrefix); } } - } + } return i; -} - +} -FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) { +FstState fstStateCreateFrom(FstSlice *slice, CompiledAddr addr) { FstState fs = {.state = EmptyFinal, .val = 0}; if (addr == EMPTY_ADDRESS) { - return fs; + return fs; } - + uint8_t *data = fstSliceData(slice, NULL); - uint8_t v = data[addr]; - uint8_t t = (v & 0b11000000) >> 6; + uint8_t v = data[addr]; + uint8_t t = (v & 0b11000000) >> 6; if (t == 0b11) { fs.state = OneTransNext; } else if (t == 0b10) { - fs.state = OneTrans; + fs.state = OneTrans; } else { - fs.state = AnyTrans; + fs.state = AnyTrans; } fs.val = v; return fs; } -static FstState fstStateDict[] = { - {.state = OneTransNext, .val = 0b11000000}, - {.state = OneTrans, .val = 0b10000000}, - {.state = AnyTrans, .val = 0b00000000}, - {.state = EmptyFinal, .val = 0b00000000} -}; -// debug -static const char *fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; +static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000}, {.state = OneTrans, .val = 0b10000000}, + {.state = AnyTrans, .val = 0b00000000}, {.state = EmptyFinal, .val = 0b00000000}}; +// debug +static const char *fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; -FstState fstStateCreate(State state){ +FstState fstStateCreate(State state) { uint8_t idx = (uint8_t)state; return fstStateDict[idx]; } -//compile +// compile void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uint8_t inp) { - FstState s = fstStateCreate(OneTransNext); + FstState s = fstStateCreate(OneTransNext); fstStateSetCommInput(&s, inp); - bool null = false; + bool null = false; uint8_t v = fstStateCommInput(&s, &null); if (null) { // w->write_all(&[inp]) fstCountingWriterWrite(w, &inp, 1); - } + } fstCountingWriterWrite(w, &(s.val), 1); // w->write_all(&[s.val]) return; } -void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTransition* trn) { - Output out = trn->out; - uint8_t outPackSize = (out == 0 ? 0 : fstCountingWriterPackUint(w, out)); - uint8_t transPackSize = fstPackDetla(w, addr, trn->addr); +void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTransition *trn) { + Output out = trn->out; + uint8_t outPackSize = (out == 0 ? 0 : fstCountingWriterPackUint(w, out)); + uint8_t transPackSize = fstPackDetla(w, addr, trn->addr); PackSizes packSizes = 0; FST_SET_OUTPUT_PACK_SIZE(packSizes, outPackSize); FST_SET_TRANSITION_PACK_SIZE(packSizes, transPackSize); - fstCountingWriterWrite(w, (char *)&packSizes, sizeof(packSizes)); + fstCountingWriterWrite(w, (char *)&packSizes, sizeof(packSizes)); + + FstState st = fstStateCreate(OneTrans); - FstState st = fstStateCreate(OneTrans); - fstStateSetCommInput(&st, trn->inp); - bool null = false; - uint8_t inp = fstStateCommInput(&st, &null); + bool null = false; + uint8_t inp = fstStateCommInput(&st, &null); if (null == true) { fstCountingWriterWrite(w, (char *)&trn->inp, sizeof(trn->inp)); } fstCountingWriterWrite(w, (char *)(&(st.val)), sizeof(st.val)); - return ; - + return; } void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node) { - size_t sz = taosArrayGetSize(node->trans); + size_t sz = taosArrayGetSize(node->trans); assert(sz <= 256); uint8_t tSize = 0; - uint8_t oSize = packSize(node->finalOutput) ; - + uint8_t oSize = packSize(node->finalOutput); + // finalOutput.is_zero() - bool anyOuts = (node->finalOutput != 0) ; + bool anyOuts = (node->finalOutput != 0); for (size_t i = 0; i < sz; i++) { - FstTransition *t = taosArrayGet(node->trans, i); - tSize = MAX(tSize, packDeltaSize(addr, t->addr)); + FstTransition *t = taosArrayGet(node->trans, i); + tSize = MAX(tSize, packDeltaSize(addr, t->addr)); oSize = MAX(oSize, packSize(t->out)); - anyOuts = anyOuts || (t->out != 0); + anyOuts = anyOuts || (t->out != 0); } - PackSizes packSizes = 0; - if (anyOuts) { FST_SET_OUTPUT_PACK_SIZE(packSizes, oSize); } - else { FST_SET_OUTPUT_PACK_SIZE(packSizes, 0); } + PackSizes packSizes = 0; + if (anyOuts) { + FST_SET_OUTPUT_PACK_SIZE(packSizes, oSize); + } else { + FST_SET_OUTPUT_PACK_SIZE(packSizes, 0); + } FST_SET_TRANSITION_PACK_SIZE(packSizes, tSize); - + FstState st = fstStateCreate(AnyTrans); - fstStateSetFinalState(&st, node->isFinal); + fstStateSetFinalState(&st, node->isFinal); fstStateSetStateNtrans(&st, (uint8_t)sz); - + if (anyOuts) { if (FST_BUILDER_NODE_IS_FINAL(node)) { fstCountingWriterPackUintIn(w, node->finalOutput, oSize); @@ -282,123 +279,115 @@ void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuil FstTransition *t = taosArrayGet(node->trans, i); fstCountingWriterPackUintIn(w, t->out, oSize); } - } + } for (int32_t i = sz - 1; i >= 0; i--) { - FstTransition *t = taosArrayGet(node->trans, i); - fstPackDeltaIn(w, addr, t->addr, tSize); + FstTransition *t = taosArrayGet(node->trans, i); + fstPackDeltaIn(w, addr, t->addr, tSize); } for (int32_t i = sz - 1; i >= 0; i--) { - FstTransition *t = taosArrayGet(node->trans, i); - fstCountingWriterWrite(w, (char *)&t->inp, 1); - //fstPackDeltaIn(w, addr, t->addr, tSize); + FstTransition *t = taosArrayGet(node->trans, i); + fstCountingWriterWrite(w, (char *)&t->inp, 1); + // fstPackDeltaIn(w, addr, t->addr, tSize); } if (sz > TRANS_INDEX_THRESHOLD) { // A value of 255 indicates that no transition exists for the byte // at that index. (Except when there are 256 transitions.) Namely, // any value greater than or equal to the number of transitions in // this node indicates an absent transition. - uint8_t *index = (uint8_t *)malloc(sizeof(uint8_t) * 256); + uint8_t *index = (uint8_t *)malloc(sizeof(uint8_t) * 256); memset(index, 255, sizeof(uint8_t) * 256); - ///for (uint8_t i = 0; i < 256; i++) { + /// for (uint8_t i = 0; i < 256; i++) { // index[i] = 255; ///} for (size_t i = 0; i < sz; i++) { FstTransition *t = taosArrayGet(node->trans, i); index[t->inp] = i; - //fstPackDeltaIn(w, addr, t->addr, tSize); + // fstPackDeltaIn(w, addr, t->addr, tSize); } - fstCountingWriterWrite(w, (char *)index, 256); + fstCountingWriterWrite(w, (char *)index, 256); free(index); } fstCountingWriterWrite(w, (char *)&packSizes, 1); bool null = false; fstStateStateNtrans(&st, &null); if (null == true) { - // 256 can't be represented in a u8, so we abuse the fact that - // the # of transitions can never be 1 here, since 1 is always - // encoded in the state byte. + // 256 can't be represented in a u8, so we abuse the fact that + // the # of transitions can never be 1 here, since 1 is always + // encoded in the state byte. uint8_t v = 1; - if (sz == 256) { fstCountingWriterWrite(w, (char *)&v, 1); } - else { fstCountingWriterWrite(w, (char *)&sz, 1); } + if (sz == 256) { + fstCountingWriterWrite(w, (char *)&v, 1); + } else { + fstCountingWriterWrite(w, (char *)&sz, 1); + } } fstCountingWriterWrite(w, (char *)(&(st.val)), 1); return; } // set_comm_input -void fstStateSetCommInput(FstState* s, uint8_t inp) { +void fstStateSetCommInput(FstState *s, uint8_t inp) { assert(s->state == OneTransNext || s->state == OneTrans); uint8_t val; - COMMON_INDEX(inp, 0x111111, val); - s->val = (s->val & fstStateDict[s->state].val) | val; + COMMON_INDEX(inp, 0x111111, val); + s->val = (s->val & fstStateDict[s->state].val) | val; } // comm_input -uint8_t fstStateCommInput(FstState* s, bool *null) { +uint8_t fstStateCommInput(FstState *s, bool *null) { assert(s->state == OneTransNext || s->state == OneTrans); uint8_t v = s->val & 0b00111111; - if (v == 0) { - *null = true; + if (v == 0) { + *null = true; return v; - } - //v = 0 indicate that common_input is None - return v == 0 ? 0 : COMMON_INPUT(v); + } + // v = 0 indicate that common_input is None + return v == 0 ? 0 : COMMON_INPUT(v); } // input_len -uint64_t fstStateInputLen(FstState* s) { +uint64_t fstStateInputLen(FstState *s) { assert(s->state == OneTransNext || s->state == OneTrans); bool null = false; - fstStateCommInput(s, &null); - return null ? 1 : 0 ; -} - -// end_addr -uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice *data) { + fstStateCommInput(s, &null); + return null ? 1 : 0; +} + +// end_addr +uint64_t fstStateEndAddrForOneTransNext(FstState *s, FstSlice *data) { assert(s->state == OneTransNext); return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s); } uint64_t fstStateEndAddrForOneTrans(FstState *s, FstSlice *data, PackSizes sizes) { assert(s->state == OneTrans); - return FST_SLICE_LEN(data) - - 1 - - fstStateInputLen(s) - - 1 // pack size - - FST_GET_TRANSITION_PACK_SIZE(sizes) - - FST_GET_OUTPUT_PACK_SIZE(sizes); -} -uint64_t fstStateEndAddrForAnyTrans(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans) { - uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); - uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; - return FST_SLICE_LEN(date) - - 1 - - fstStateNtransLen(state) - - 1 //pack size - - fstStateTotalTransSize(state, version, sizes, nTrans) - - nTrans * oSizes // output values - - finalOsize; // final output -} -// input -uint8_t fstStateInput(FstState *s, FstNode *node) { + return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size + - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); +} +uint64_t fstStateEndAddrForAnyTrans( + FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans) { + uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); + uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; + return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size + - fstStateTotalTransSize(state, version, sizes, nTrans) - nTrans * oSizes // output values + - finalOsize; // final output +} +// input +uint8_t fstStateInput(FstState *s, FstNode *node) { assert(s->state == OneTransNext || s->state == OneTrans); FstSlice *slice = &node->data; - bool null = false; - uint8_t inp = fstStateCommInput(s, &null); - uint8_t *data = fstSliceData(slice, NULL); + bool null = false; + uint8_t inp = fstStateCommInput(s, &null); + uint8_t * data = fstSliceData(slice, NULL); return null == false ? inp : data[-1]; } -uint8_t fstStateInputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { +uint8_t fstStateInputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { assert(s->state == AnyTrans); - FstSlice *slice = &node->data; + FstSlice *slice = &node->data; - uint64_t at = node->start - - fstStateNtransLen(s) - - 1 // pack size - - fstStateTransIndexSize(s, node->version, node->nTrans) - - i - - 1; // the output size + uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size + - fstStateTransIndexSize(s, node->version, node->nTrans) - i - 1; // the output size uint8_t *data = fstSliceData(slice, NULL); return data[at]; @@ -409,84 +398,68 @@ CompiledAddr fstStateTransAddr(FstState *s, FstNode *node) { assert(s->state == OneTransNext || s->state == OneTrans); FstSlice *slice = &node->data; if (s->state == OneTransNext) { - return (CompiledAddr)(node->end) - 1; + return (CompiledAddr)(node->end) - 1; } else { - PackSizes sizes = node->sizes; - uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(sizes); - uint64_t i = node->start - - fstStateInputLen(s) - - 1 // PackSizes - - tSizes; - - // refactor error logic + PackSizes sizes = node->sizes; + uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(sizes); + uint64_t i = node->start - fstStateInputLen(s) - 1 // PackSizes + - tSizes; + + // refactor error logic uint8_t *data = fstSliceData(slice, NULL); - return unpackDelta(data +i, tSizes, node->end); - } + return unpackDelta(data + i, tSizes, node->end); + } } CompiledAddr fstStateTransAddrForAnyTrans(FstState *s, FstNode *node, uint64_t i) { assert(s->state == AnyTrans); FstSlice *slice = &node->data; - uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); - uint64_t at = node->start - - fstStateNtransLen(s) - - 1 - - fstStateTransIndexSize(s, node->version, node->nTrans) - - node->nTrans - - (i * tSizes) - - tSizes; + uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); + uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - + node->nTrans - (i * tSizes) - tSizes; uint8_t *data = fstSliceData(slice, NULL); - return unpackDelta(data + at, tSizes, node->end); + return unpackDelta(data + at, tSizes, node->end); } -// sizes +// sizes PackSizes fstStateSizes(FstState *s, FstSlice *slice) { - assert(s->state == OneTrans || s->state == AnyTrans) ; - uint64_t i; + assert(s->state == OneTrans || s->state == AnyTrans); + uint64_t i; if (s->state == OneTrans) { - i = FST_SLICE_LEN(slice) - 1 - fstStateInputLen(s) - 1; + i = FST_SLICE_LEN(slice) - 1 - fstStateInputLen(s) - 1; } else { i = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1; } uint8_t *data = fstSliceData(slice, NULL); - return (PackSizes)(*(data +i)); + return (PackSizes)(*(data + i)); } -// Output +// Output Output fstStateOutput(FstState *s, FstNode *node) { - assert(s->state == OneTrans); - + assert(s->state == OneTrans); + uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); if (oSizes == 0) { return 0; } FstSlice *slice = &node->data; - uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); - - uint64_t i = node->start - - fstStateInputLen(s) - - 1 - - tSizes - - oSizes; - uint8_t *data = fstSliceData(slice, NULL); + uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); + + uint64_t i = node->start - fstStateInputLen(s) - 1 - tSizes - oSizes; + uint8_t *data = fstSliceData(slice, NULL); return unpackUint64(data + i, oSizes); - } Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { assert(s->state == AnyTrans); - uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); + uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); if (oSizes == 0) { - return 0; - } + return 0; + } FstSlice *slice = &node->data; - uint8_t *data = fstSliceData(slice, NULL); - uint64_t at = node->start - - fstStateNtransLen(s) - - 1 // pack size - - fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - - (i * oSizes) - - oSizes; + uint8_t * data = fstSliceData(slice, NULL); + uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size + - fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - (i * oSizes) - oSizes; return unpackUint64(data + at, oSizes); } @@ -494,230 +467,226 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { // anyTrans specify function void fstStateSetFinalState(FstState *s, bool yes) { - assert(s->state == AnyTrans); - if (yes) { s->val |= 0b01000000; } + assert(s->state == AnyTrans); + if (yes) { + s->val |= 0b01000000; + } return; } bool fstStateIsFinalState(FstState *s) { - assert(s->state == AnyTrans); - return (s->val & 0b01000000) == 0b01000000; -} + assert(s->state == AnyTrans); + return (s->val & 0b01000000) == 0b01000000; +} void fstStateSetStateNtrans(FstState *s, uint8_t n) { - assert(s->state == AnyTrans); + assert(s->state == AnyTrans); if (n <= 0b00111111) { - s->val = (s->val & 0b11000000) | n; - } + s->val = (s->val & 0b11000000) | n; + } return; } // state_ntrans uint8_t fstStateStateNtrans(FstState *s, bool *null) { - assert(s->state == AnyTrans); + assert(s->state == AnyTrans); *null = false; - uint8_t n = s->val & 0b00111111; + uint8_t n = s->val & 0b00111111; if (n == 0) { - *null = true; // None - } + *null = true; // None + } return n; } uint64_t fstStateTotalTransSize(FstState *s, uint64_t version, PackSizes sizes, uint64_t nTrans) { - assert(s->state == AnyTrans); - uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans); + assert(s->state == AnyTrans); + uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans); return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize; } uint64_t fstStateTransIndexSize(FstState *s, uint64_t version, uint64_t nTrans) { - assert(s->state == AnyTrans); - return (version >= 2 &&nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0; + assert(s->state == AnyTrans); + return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0; } uint64_t fstStateNtransLen(FstState *s) { assert(s->state == AnyTrans); bool null = false; fstStateStateNtrans(s, &null); - return null == true ? 1 : 0; + return null == true ? 1 : 0; } uint64_t fstStateNtrans(FstState *s, FstSlice *slice) { - bool null = false; + bool null = false; uint8_t n = fstStateStateNtrans(s, &null); if (null != true) { - return n; - } - int32_t len; + return n; + } + int32_t len; uint8_t *data = fstSliceData(slice, &len); n = data[len - 2]; - //n = data[slice->end - 1]; // data[data.len() - 2] - return n == 1 ? 256: n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, then it is encoded in the state byte -} -Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, PackSizes sizes, uint64_t nTrans) { - uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); - if (oSizes == 0 || !fstStateIsFinalState(s)) { - return 0; - } - - uint64_t at = FST_SLICE_LEN(slice) - - 1 - - fstStateNtransLen(s) - - 1 // pack size - - fstStateTotalTransSize(s, version, sizes, nTrans) - - (nTrans * oSizes) - - oSizes; - uint8_t *data = fstSliceData(slice, NULL); - return unpackUint64(data + at, (uint8_t)oSizes); + // n = data[slice->end - 1]; // data[data.len() - 2] + return n == 1 ? 256 : n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, + // then it is encoded in the state byte +} +Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, PackSizes sizes, uint64_t nTrans) { + uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); + if (oSizes == 0 || !fstStateIsFinalState(s)) { + return 0; + } + uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size + - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; + uint8_t *data = fstSliceData(slice, NULL); + return unpackUint64(data + at, (uint8_t)oSizes); } uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { assert(s->state == AnyTrans); FstSlice *slice = &node->data; if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) { - uint64_t at = node->start - - fstStateNtransLen(s) - - 1 // pack size + uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size - fstStateTransIndexSize(s, node->version, node->nTrans); - int32_t dlen = 0; + int32_t dlen = 0; uint8_t *data = fstSliceData(slice, &dlen); uint64_t i = data[at + b]; - //uint64_t i = slice->data[slice->start + at + b]; + // uint64_t i = slice->data[slice->start + at + b]; if (i >= node->nTrans) { *null = true; - } + } return i; } else { - uint64_t start = node->start - - fstStateNtransLen(s) - - 1 // pack size - - node->nTrans; - uint64_t end = start + node->nTrans; + uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size + - node->nTrans; + uint64_t end = start + node->nTrans; FstSlice t = fstSliceCopy(slice, start, end - 1); - int32_t len = 0; + int32_t len = 0; uint8_t *data = fstSliceData(&t, &len); - int i = 0; - for(; i < len; i++) { - uint8_t v = data[i]; + int i = 0; + for (; i < len; i++) { + uint8_t v = data[i]; if (v == b) { fstSliceDestroy(&t); - return node->nTrans - i - 1; // bug + return node->nTrans - i - 1; // bug } - } - if (i == len) { *null = true; } + } + if (i == len) { + *null = true; + } fstSliceDestroy(&t); - } + } } - -// fst node function +// fst node function FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) { - FstNode *n = (FstNode *)malloc(sizeof(FstNode)); - if (n == NULL) { return NULL; } + FstNode *n = (FstNode *)malloc(sizeof(FstNode)); + if (n == NULL) { + return NULL; + } - FstState st = fstStateCreateFrom(slice, addr); + FstState st = fstStateCreateFrom(slice, addr); if (st.state == EmptyFinal) { - n->data = fstSliceCreate(NULL, 0); - n->version = version; - n->state = st; - n->start = EMPTY_ADDRESS; - n->end = EMPTY_ADDRESS; - n->isFinal = true; - n->nTrans = 0; - n->sizes = 0; - n->finalOutput = 0; + n->data = fstSliceCreate(NULL, 0); + n->version = version; + n->state = st; + n->start = EMPTY_ADDRESS; + n->end = EMPTY_ADDRESS; + n->isFinal = true; + n->nTrans = 0; + n->sizes = 0; + n->finalOutput = 0; } else if (st.state == OneTransNext) { - n->data = fstSliceCopy(slice, 0, addr); - n->version = version; - n->state = st; - n->start = addr; - n->end = fstStateEndAddrForOneTransNext(&st, &n->data); //? s.end_addr(data); - n->isFinal = false; - n->sizes = 0; - n->nTrans = 1; - n->finalOutput = 0; + n->data = fstSliceCopy(slice, 0, addr); + n->version = version; + n->state = st; + n->start = addr; + n->end = fstStateEndAddrForOneTransNext(&st, &n->data); //? s.end_addr(data); + n->isFinal = false; + n->sizes = 0; + n->nTrans = 1; + n->finalOutput = 0; } else if (st.state == OneTrans) { - FstSlice data = fstSliceCopy(slice, 0, addr); - PackSizes sz = fstStateSizes(&st, &data); - n->data = data; - n->version = version; - n->state = st; - n->start = addr; - n->end = fstStateEndAddrForOneTrans(&st, &data, sz); // s.end_addr(data, sz); - n->isFinal = false; - n->nTrans = 1; - n->sizes = sz; - n->finalOutput = 0; + FstSlice data = fstSliceCopy(slice, 0, addr); + PackSizes sz = fstStateSizes(&st, &data); + n->data = data; + n->version = version; + n->state = st; + n->start = addr; + n->end = fstStateEndAddrForOneTrans(&st, &data, sz); // s.end_addr(data, sz); + n->isFinal = false; + n->nTrans = 1; + n->sizes = sz; + n->finalOutput = 0; } else { - FstSlice data = fstSliceCopy(slice, 0, addr); - uint64_t sz = fstStateSizes(&st, &data); // s.sizes(data) - uint32_t nTrans = fstStateNtrans(&st, &data); // s.ntrans(data) - n->data = data; - n->version = version; - n->state = st; - n->start = addr; - n->end = fstStateEndAddrForAnyTrans(&st, version, &data, sz, nTrans); // s.end_addr(version, data, sz, ntrans); - n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); - n->nTrans = nTrans; - n->sizes = sz; - n->finalOutput = fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); - } - return n; + FstSlice data = fstSliceCopy(slice, 0, addr); + uint64_t sz = fstStateSizes(&st, &data); // s.sizes(data) + uint32_t nTrans = fstStateNtrans(&st, &data); // s.ntrans(data) + n->data = data; + n->version = version; + n->state = st; + n->start = addr; + n->end = fstStateEndAddrForAnyTrans(&st, version, &data, sz, nTrans); // s.end_addr(version, data, sz, ntrans); + n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); + n->nTrans = nTrans; + n->sizes = sz; + n->finalOutput = + fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); + } + return n; } // debug state transition static const char *fstNodeState(FstNode *node) { - FstState *st = &node->state; - return fstStateStr[st->state]; + FstState *st = &node->state; + return fstStateStr[st->state]; } - void fstNodeDestroy(FstNode *node) { - fstSliceDestroy(&node->data); + fstSliceDestroy(&node->data); free(node); } -FstTransitions* fstNodeTransitions(FstNode *node) { +FstTransitions *fstNodeTransitions(FstNode *node) { FstTransitions *t = malloc(sizeof(FstTransitions)); if (NULL == t) { - return NULL; + return NULL; } FstRange range = {.start = 0, .end = FST_NODE_LEN(node)}; t->range = range; - t->node = node; - return t; -} + t->node = node; + return t; +} -// Returns the transition at index `i`. +// Returns the transition at index `i`. bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *trn) { - bool s = true; + bool s = true; FstState *st = &node->state; if (st->state == OneTransNext) { - trn->inp = fstStateInput(st, node); - trn->out = 0; - trn->addr = fstStateTransAddr(st, node); + trn->inp = fstStateInput(st, node); + trn->out = 0; + trn->addr = fstStateTransAddr(st, node); } else if (st->state == OneTrans) { - trn->inp = fstStateInput(st, node); - trn->out = fstStateOutput(st, node); - trn->addr = fstStateTransAddr(st, node); + trn->inp = fstStateInput(st, node); + trn->out = fstStateOutput(st, node); + trn->addr = fstStateTransAddr(st, node); } else if (st->state == AnyTrans) { - trn->inp = fstStateInputForAnyTrans(st, node, i); - trn->out = fstStateOutputForAnyTrans(st, node, i); - trn->addr = fstStateTransAddrForAnyTrans(st, node, i); + trn->inp = fstStateInputForAnyTrans(st, node, i); + trn->out = fstStateOutputForAnyTrans(st, node, i); + trn->addr = fstStateTransAddrForAnyTrans(st, node, i); } else { s = false; } return s; -} +} // Returns the transition address of the `i`th transition bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res) { - bool s = true; + bool s = true; FstState *st = &node->state; if (st->state == OneTransNext) { assert(i == 0); fstStateTransAddr(st, node); } else if (st->state == OneTrans) { - assert(i == 0); + assert(i == 0); fstStateTransAddr(st, node); } else if (st->state == AnyTrans) { fstStateTransAddrForAnyTrans(st, node, i); - } else if (FST_STATE_EMPTY_FINAL(node)){ + } else if (FST_STATE_EMPTY_FINAL(node)) { s = false; } else { assert(0); @@ -726,129 +695,138 @@ bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res) { } // Finds the `i`th transition corresponding to the given input byte. -// If no transition for this byte exists, then `false` is returned. +// If no transition for this byte exists, then `false` is returned. bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res) { - bool s = true; + bool s = true; FstState *st = &node->state; if (st->state == OneTransNext) { - if (fstStateInput(st,node) == b) { *res = 0; } - else { s = false; } } - else if (st->state == OneTrans) { - if (fstStateInput(st, node) == b) { *res = 0 ;} - else { s = false; } + if (fstStateInput(st, node) == b) { + *res = 0; + } else { + s = false; + } + } else if (st->state == OneTrans) { + if (fstStateInput(st, node) == b) { + *res = 0; + } else { + s = false; + } } else if (st->state == AnyTrans) { - bool null = false; - uint64_t out = fstStateFindInput(st, node, b, &null); - if (null == false) { *res = out; } - else { s = false;} + bool null = false; + uint64_t out = fstStateFindInput(st, node, b, &null); + if (null == false) { + *res = out; + } else { + s = false; + } } return s; -} +} bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode) { - size_t sz = taosArrayGetSize(builderNode->trans); + size_t sz = taosArrayGetSize(builderNode->trans); assert(sz < 256); if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) { - return true; + return true; } else if (sz != 1 || builderNode->isFinal) { - fstStateCompileForAnyTrans(w, addr, builderNode); + fstStateCompileForAnyTrans(w, addr, builderNode); // AnyTrans->Compile(w, addr, node); } else { - FstTransition *tran = taosArrayGet(builderNode->trans, 0); + FstTransition *tran = taosArrayGet(builderNode->trans, 0); if (tran->addr == lastAddr && tran->out == 0) { - fstStateCompileForOneTransNext(w, addr, tran->inp); - //OneTransNext::compile(w, lastAddr, tran->inp); - return true; + fstStateCompileForOneTransNext(w, addr, tran->inp); + // OneTransNext::compile(w, lastAddr, tran->inp); + return true; } else { - fstStateCompileForOneTrans(w, addr, tran); - //OneTrans::Compile(w, lastAddr, *tran); - return true; - } - } - return true; -} + fstStateCompileForOneTrans(w, addr, tran); + // OneTrans::Compile(w, lastAddr, *tran); + return true; + } + } + return true; +} bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b); } +FstBuilder *fstBuilderCreate(void *w, FstType ty) { + FstBuilder *b = malloc(sizeof(FstBuilder)); + if (NULL == b) { + return b; + } + b->wrt = fstCountingWriterCreate(w); + b->unfinished = fstUnFinishedNodesCreate(); + b->registry = fstRegistryCreate(10000, 2); + b->last = fstSliceCreate(NULL, 0); + b->lastAddr = NONE_ADDRESS; + b->len = 0; -FstBuilder *fstBuilderCreate(void *w, FstType ty) { - FstBuilder *b = malloc(sizeof(FstBuilder)); - if (NULL == b) { return b; } - - - b->wrt = fstCountingWriterCreate(w, false); - b->unfinished = fstUnFinishedNodesCreate(); - b->registry = fstRegistryCreate(10000, 2) ; - b->last = fstSliceCreate(NULL, 0); - b->lastAddr = NONE_ADDRESS; - b->len = 0; - - char buf64[8] = {0}; + char buf64[8] = {0}; void *pBuf64 = buf64; - taosEncodeFixedU64(&pBuf64, VERSION); + taosEncodeFixedU64(&pBuf64, VERSION); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); - - memset(buf64, 0, sizeof(buf64)); + + memset(buf64, 0, sizeof(buf64)); pBuf64 = buf64; - taosEncodeFixedU64(&pBuf64, ty); + taosEncodeFixedU64(&pBuf64, ty); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); return b; } void fstBuilderDestroy(FstBuilder *b) { - if (b == NULL) { return; } + if (b == NULL) { + return; + } - fstCountingWriterDestroy(b->wrt); - fstUnFinishedNodesDestroy(b->unfinished); + fstCountingWriterDestroy(b->wrt); + fstUnFinishedNodesDestroy(b->unfinished); fstRegistryDestroy(b->registry); fstSliceDestroy(&b->last); free(b); } - bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in) { - OrderType t = fstBuilderCheckLastKey(b, bs, true); + OrderType t = fstBuilderCheckLastKey(b, bs, true); if (t == Ordered) { // add log info - fstBuilderInsertOutput(b, bs, in); - return true; - } + fstBuilderInsertOutput(b, bs, in); + return true; + } indexInfo("key must be ordered"); return false; } void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { - FstSlice *s = &bs; - if (fstSliceIsEmpty(s)) { - b->len = 1; - fstUnFinishedNodesSetRootOutput(b->unfinished, in); - return; - } - //if (in != 0) { //if let Some(in) = in - // prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); - //} else { - // prefixLen = fstUnFinishedNodesFindCommPrefix(b->unfinished, bs); - // out = 0; - //} - Output out; - uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); - - if (prefixLen == FST_SLICE_LEN(s)) { - assert(out == 0); - return; - } - - b->len += 1; - fstBuilderCompileFrom(b, prefixLen); - - FstSlice sub = fstSliceCopy(s, prefixLen, s->end); - fstUnFinishedNodesAddSuffix(b->unfinished, sub, out); - fstSliceDestroy(&sub); - return; - } + FstSlice *s = &bs; + if (fstSliceIsEmpty(s)) { + b->len = 1; + fstUnFinishedNodesSetRootOutput(b->unfinished, in); + return; + } + // if (in != 0) { //if let Some(in) = in + // prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); + //} else { + // prefixLen = fstUnFinishedNodesFindCommPrefix(b->unfinished, bs); + // out = 0; + //} + Output out; + uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); + + if (prefixLen == FST_SLICE_LEN(s)) { + assert(out == 0); + return; + } + + b->len += 1; + fstBuilderCompileFrom(b, prefixLen); + + FstSlice sub = fstSliceCopy(s, prefixLen, s->end); + fstUnFinishedNodesAddSuffix(b->unfinished, sub, out); + fstSliceDestroy(&sub); + return; +} OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { FstSlice *input = &bs; @@ -859,16 +837,16 @@ OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { } else { int comp = fstSliceCompare(&b->last, &bs); if (comp == 0 && ckDup) { - return DuplicateKey; + return DuplicateKey; } else if (comp == 1) { return OutOfOrdered; } // deep copy or not fstSliceDestroy(&b->last); - b->last = fstSliceDeepCopy(&bs, input->start, input->end); - } + b->last = fstSliceDeepCopy(&bs, input->start, input->end); + } return Ordered; -} +} void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { CompiledAddr addr = NONE_ADDRESS; while (istate + 1 < FST_UNFINISHED_NODES_LEN(b->unfinished)) { @@ -881,252 +859,240 @@ void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { addr = fstBuilderCompile(b, bn); fstBuilderNodeDestroy(bn); - assert(addr != NONE_ADDRESS); - //fstBuilderNodeDestroy(n); + assert(addr != NONE_ADDRESS); + // fstBuilderNodeDestroy(n); } fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); - return; + return; } CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn) { - if (FST_BUILDER_NODE_IS_FINAL(bn) - && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) - && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) { - return EMPTY_ADDRESS; + if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) { + return EMPTY_ADDRESS; } - FstRegistryEntry *entry = fstRegistryGetEntry(b->registry, bn); - if (entry->state == FOUND) { + FstRegistryEntry *entry = fstRegistryGetEntry(b->registry, bn); + if (entry->state == FOUND) { CompiledAddr ret = entry->addr; fstRegistryEntryDestroy(entry); return ret; - } + } CompiledAddr startAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt)); - fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); - b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); + fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); + b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); if (entry->state == NOTFOUND) { - FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); + FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); } fstRegistryEntryDestroy(entry); - - return b->lastAddr; + + return b->lastAddr; } -void* fstBuilderInsertInner(FstBuilder *b) { - fstBuilderCompileFrom(b, 0); - FstBuilderNode *rootNode = fstUnFinishedNodesPopRoot(b->unfinished); - CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); +void *fstBuilderInsertInner(FstBuilder *b) { + fstBuilderCompileFrom(b, 0); + FstBuilderNode *rootNode = fstUnFinishedNodesPopRoot(b->unfinished); + CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); fstBuilderNodeDestroy(rootNode); - char buf64[8] = {0}; + char buf64[8] = {0}; + + void *pBuf64 = buf64; + taosEncodeFixedU64(&pBuf64, b->len); + fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); - void *pBuf64 = buf64; - taosEncodeFixedU64(&pBuf64, b->len); - fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); - pBuf64 = buf64; - taosEncodeFixedU64(&pBuf64, rootAddr); - fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); + taosEncodeFixedU64(&pBuf64, rootAddr); + fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); - char buf32[4] = {0}; - void *pBuf32 = buf32; + char buf32[4] = {0}; + void * pBuf32 = buf32; uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt); - taosEncodeFixedU32(&pBuf32, sum); - fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32)); - + taosEncodeFixedU32(&pBuf32, sum); + fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32)); + fstCountingWriterFlush(b->wrt); - //fstCountingWriterDestroy(b->wrt); - //b->wrt = NULL; + // fstCountingWriterDestroy(b->wrt); + // b->wrt = NULL; return b->wrt; } -void fstBuilderFinish(FstBuilder *b) { - fstBuilderInsertInner(b); -} - - +void fstBuilderFinish(FstBuilder *b) { fstBuilderInsertInner(b); } FstSlice fstNodeAsSlice(FstNode *node) { - FstSlice *slice = &node->data; - FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1); - return s; + FstSlice *slice = &node->data; + FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1); + return s; } FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out) { FstLastTransition *trn = malloc(sizeof(FstLastTransition)); - if (trn == NULL) { return NULL; } + if (trn == NULL) { + return NULL; + } trn->inp = inp; trn->out = out; return trn; } -void fstLastTransitionDestroy(FstLastTransition *trn) { - free(trn); -} +void fstLastTransitionDestroy(FstLastTransition *trn) { free(trn); } void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *unNode, CompiledAddr addr) { - FstLastTransition *trn = unNode->last; - if (trn == NULL) { return; } - FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; - taosArrayPush(unNode->node->trans, &t); - fstLastTransitionDestroy(trn); + FstLastTransition *trn = unNode->last; + if (trn == NULL) { + return; + } + FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; + taosArrayPush(unNode->node->trans, &t); + fstLastTransitionDestroy(trn); unNode->last = NULL; return; } void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *unNode, Output out) { if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { - unNode->node->finalOutput += out; + unNode->node->finalOutput += out; } size_t sz = taosArrayGetSize(unNode->node->trans); for (size_t i = 0; i < sz; i++) { - FstTransition *trn = taosArrayGet(unNode->node->trans, i); + FstTransition *trn = taosArrayGet(unNode->node->trans, i); trn->out += out; } if (unNode->last) { - unNode->last->out += out; + unNode->last->out += out; } return; } -Fst* fstCreate(FstSlice *slice) { +Fst *fstCreate(FstSlice *slice) { int32_t slen; - char *buf = fstSliceData(slice, &slen); - if (slen < 36) { - return NULL; + char * buf = fstSliceData(slice, &slen); + if (slen < 36) { + return NULL; } uint64_t len = slen; - uint64_t skip = 0; + uint64_t skip = 0; - uint64_t version; - taosDecodeFixedU64(buf, &version); - skip += sizeof(version); + uint64_t version; + taosDecodeFixedU64(buf, &version); + skip += sizeof(version); if (version == 0 || version > VERSION) { - return NULL; - } + return NULL; + } uint64_t type; taosDecodeFixedU64(buf + skip, &type); - skip += sizeof(type); + skip += sizeof(type); uint32_t checkSum = 0; len -= sizeof(checkSum); - taosDecodeFixedU32(buf + len, &checkSum); + taosDecodeFixedU32(buf + len, &checkSum); CompiledAddr rootAddr; - len -= sizeof(rootAddr); - taosDecodeFixedU64(buf + len, &rootAddr); + len -= sizeof(rootAddr); + taosDecodeFixedU64(buf + len, &rootAddr); - uint64_t fstLen; - len -= sizeof(fstLen); + uint64_t fstLen; + len -= sizeof(fstLen); taosDecodeFixedU64(buf + len, &fstLen); - //TODO(validate root addr) - Fst *fst= (Fst *)calloc(1, sizeof(Fst)); - if (fst == NULL) { return NULL; } - + // TODO(validate root addr) + Fst *fst = (Fst *)calloc(1, sizeof(Fst)); + if (fst == NULL) { + return NULL; + } + fst->meta = (FstMeta *)malloc(sizeof(FstMeta)); - if (NULL == fst->meta) { - goto FST_CREAT_FAILED; + if (NULL == fst->meta) { + goto FST_CREAT_FAILED; } - fst->meta->version = version; - fst->meta->rootAddr = rootAddr; - fst->meta->ty = type; - fst->meta->len = fstLen; + fst->meta->version = version; + fst->meta->rootAddr = rootAddr; + fst->meta->ty = type; + fst->meta->len = fstLen; fst->meta->checkSum = checkSum; FstSlice *s = calloc(1, sizeof(FstSlice)); - *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice)); - fst->data = s; - + *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice)); + fst->data = s; + return fst; -FST_CREAT_FAILED: - free(fst->meta); +FST_CREAT_FAILED: + free(fst->meta); free(fst); - } void fstDestroy(Fst *fst) { - if (fst) { - free(fst->meta); + if (fst) { + free(fst->meta); fstSliceDestroy(fst->data); free(fst->data); - } - free(fst); + } + free(fst); } bool fstGet(Fst *fst, FstSlice *b, Output *out) { - FstNode *root = fstGetRoot(fst); - Output tOut = 0; - int32_t len; + FstNode *root = fstGetRoot(fst); + Output tOut = 0; + int32_t len; uint8_t *data = fstSliceData(b, &len); - SArray *nodes = (SArray *)taosArrayInit(len, sizeof(FstNode *)); + SArray *nodes = (SArray *)taosArrayInit(len, sizeof(FstNode *)); taosArrayPush(nodes, &root); for (uint32_t i = 0; i < len; i++) { uint8_t inp = data[i]; Output res = 0; if (false == fstNodeFindInput(root, inp, &res)) { - return false; - } + return false; + } - FstTransition trn; + FstTransition trn; fstNodeGetTransitionAt(root, res, &trn); - tOut += trn.out; + tOut += trn.out; root = fstGetNode(fst, trn.addr); taosArrayPush(nodes, &root); } if (!FST_NODE_IS_FINAL(root)) { return false; } else { - tOut = tOut + FST_NODE_FINAL_OUTPUT(root); + tOut = tOut + FST_NODE_FINAL_OUTPUT(root); } for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { - FstNode **node = (FstNode **)taosArrayGet(nodes, i); - fstNodeDestroy(*node); + FstNode **node = (FstNode **)taosArrayGet(nodes, i); + fstNodeDestroy(*node); } taosArrayDestroy(nodes); fst->root = NULL; *out = tOut; - - return true; -} -FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) { - return fstStreamBuilderCreate(fst, ctx); + + return true; } -StreamWithState* streamBuilderIntoStream(FstStreamBuilder *sb) { - if (sb == NULL) { return NULL; } +FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); } +StreamWithState * streamBuilderIntoStream(FstStreamBuilder *sb) { + if (sb == NULL) { + return NULL; + } return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); } -FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) { - return fstStreamBuilderCreate(fst, ctx); -} +FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) { return fstStreamBuilderCreate(fst, ctx); } FstNode *fstGetRoot(Fst *fst) { if (fst->root != NULL) { return fst->root; } - CompiledAddr rAddr = fstGetRootAddr(fst); - fst->root = fstGetNode(fst, rAddr); + CompiledAddr rAddr = fstGetRootAddr(fst); + fst->root = fstGetNode(fst, rAddr); return fst->root; } -FstNode* fstGetNode(Fst *fst, CompiledAddr addr) { - return fstNodeCreate(fst->meta->version, addr, fst->data); - -} -FstType fstGetType(Fst *fst) { - return fst->meta->ty; -} -CompiledAddr fstGetRootAddr(Fst *fst) { - return fst->meta->rootAddr; -} +FstNode * fstGetNode(Fst *fst, CompiledAddr addr) { return fstNodeCreate(fst->meta->version, addr, fst->data); } +FstType fstGetType(Fst *fst) { return fst->meta->ty; } +CompiledAddr fstGetRootAddr(Fst *fst) { return fst->meta->rootAddr; } Output fstEmptyFinalOutput(Fst *fst, bool *null) { - Output res = 0; + Output res = 0; FstNode *node = fstGetRoot(fst); if (FST_NODE_IS_FINAL(node)) { *null = false; - res = FST_NODE_FINAL_OUTPUT(node); + res = FST_NODE_FINAL_OUTPUT(node); } else { *null = true; } @@ -1135,9 +1101,9 @@ Output fstEmptyFinalOutput(Fst *fst, bool *null) { bool fstVerify(Fst *fst) { uint32_t checkSum = fst->meta->checkSum; - int32_t len; + int32_t len; uint8_t *data = fstSliceData(fst->data, &len); - TSCKSUM initSum = 0; + TSCKSUM initSum = 0; if (!taosCheckChecksumWhole(data, len)) { return false; } @@ -1145,9 +1111,11 @@ bool fstVerify(Fst *fst) { } // data bound function -FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data) { +FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data) { FstBoundWithData *b = calloc(1, sizeof(FstBoundWithData)); - if (b == NULL) { return NULL; } + if (b == NULL) { + return NULL; + } if (data != NULL) { b->data = fstSliceCopy(data, data->start, data->end); @@ -1156,10 +1124,9 @@ FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data) { } b->type = type; - return b; + return b; } - bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) { int comp = fstSliceCompare(slice, &bound->data); if (bound->type == Included) { @@ -1173,62 +1140,62 @@ bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) { bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) { if (bound->type == Unbounded) { return true; - } else { - return fstSliceIsEmpty(&bound->data); - } + } else { + return fstSliceIsEmpty(&bound->data); + } } +bool fstBoundWithDataIsIncluded(FstBoundWithData *bound) { return bound->type == Excluded ? false : true; } -bool fstBoundWithDataIsIncluded(FstBoundWithData *bound) { - return bound->type == Excluded? false : true; -} - -void fstBoundDestroy(FstBoundWithData *bound) { - free(bound); -} +void fstBoundDestroy(FstBoundWithData *bound) { free(bound); } -StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) { +StreamWithState *streamWithStateCreate( + Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) { StreamWithState *sws = calloc(1, sizeof(StreamWithState)); - if (sws == NULL) { return NULL; } + if (sws == NULL) { + return NULL; + } + + sws->fst = fst; + sws->aut = automation; + sws->inp = (SArray *)taosArrayInit(256, sizeof(uint8_t)); - sws->fst = fst; - sws->aut = automation; - sws->inp = (SArray *)taosArrayInit(256, sizeof(uint8_t)); - sws->emptyOutput.null = false; - sws->emptyOutput.out = 0; + sws->emptyOutput.out = 0; - sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); - sws->endAt = max; + sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + sws->endAt = max; streamWithStateSeekMin(sws, min); return sws; } void streamWithStateDestroy(StreamWithState *sws) { - if (sws == NULL) { return; } + if (sws == NULL) { + return; + } taosArrayDestroy(sws->inp); taosArrayDestroyEx(sws->stack, streamStateDestroy); - free(sws); + free(sws); } bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { - AutomationCtx *aut = sws->aut; if (fstBoundWithDataIsEmpty(min)) { if (fstBoundWithDataIsIncluded(min)) { - sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); - } - StreamState s = {.node = fstGetRoot(sws->fst), - .trans = 0, - .out = {.null = false, .out = 0}, - .autState = automFuncs[aut->type].start(aut)}; // auto.start callback + sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); + } + StreamState s = {.node = fstGetRoot(sws->fst), + .trans = 0, + .out = {.null = false, .out = 0}, + .autState = automFuncs[aut->type].start(aut)}; // auto.start callback taosArrayPush(sws->stack, &s); return true; - } + } FstSlice *key = NULL; - bool inclusize = false;; + bool inclusize = false; + ; if (min->type == Included) { key = &min->data; @@ -1239,86 +1206,77 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { return false; } - FstNode *node = fstGetRoot(sws->fst); - Output out = 0; - //void* autState = sws->aut->start(); - void* autState = automFuncs[aut->type].start(aut); + FstNode *node = fstGetRoot(sws->fst); + Output out = 0; + // void* autState = sws->aut->start(); + void *autState = automFuncs[aut->type].start(aut); - int32_t len; - uint8_t *data = fstSliceData(key, &len); + int32_t len; + uint8_t *data = fstSliceData(key, &len); for (uint32_t i = 0; i < len; i++) { - uint8_t b = data[i]; + uint8_t b = data[i]; uint64_t res = 0; - bool null = fstNodeFindInput(node, b, &res); + bool null = fstNodeFindInput(node, b, &res); if (null == false) { FstTransition trn; - fstNodeGetTransitionAt(node, res, &trn); + fstNodeGetTransitionAt(node, res, &trn); void *preState = autState; // autState = sws->aut->accept(preState, b); autState = automFuncs[aut->type].accept(aut, preState, b); taosArrayPush(sws->inp, &b); - StreamState s = {.node = node, - .trans = res + 1, - .out = {.null = false, .out = out}, - .autState = preState}; + StreamState s = {.node = node, .trans = res + 1, .out = {.null = false, .out = out}, .autState = preState}; taosArrayPush(sws->stack, &s); out += trn.out; - node = fstGetNode(sws->fst, trn.addr); + node = fstGetNode(sws->fst, trn.addr); fstNodeDestroy(node); } else { - // This is a little tricky. We're in this case if the // given bound is not a prefix of any key in the FST. // Since this is a minimum bound, we need to find the // first transition in this node that proceeds the current - // input byte. + // input byte. FstTransitions *trans = fstNodeTransitions(node); - uint64_t i = 0; + uint64_t i = 0; for (i = trans->range.start; i < trans->range.end; i++) { FstTransition trn; if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { - break; - } + break; + } } - - StreamState s = {.node = node, - .trans = i, - .out = {.null = false, .out = out}, - .autState = autState}; - taosArrayPush(sws->stack, &s); - return true; + + StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; + taosArrayPush(sws->stack, &s); + return true; } } - uint32_t sz = taosArrayGetSize(sws->stack); + uint32_t sz = taosArrayGetSize(sws->stack); if (sz != 0) { - StreamState *s = taosArrayGet(sws->stack, sz - 1); + StreamState *s = taosArrayGet(sws->stack, sz - 1); if (inclusize) { s->trans -= 1; taosArrayPop(sws->inp); } else { - FstNode *n = s->node; - uint64_t trans = s->trans; - FstTransition trn; + FstNode * n = s->node; + uint64_t trans = s->trans; + FstTransition trn; fstNodeGetTransitionAt(n, trans - 1, &trn); - StreamState s = {.node = fstGetNode(sws->fst, trn.addr), - .trans= 0, - .out = {.null = false, .out = out}, - .autState = autState}; + StreamState s = { + .node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; taosArrayPush(sws->stack, &s); - return true; - } - return false; - } -} + return true; + } + return false; + } +} StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback) { AutomationCtx *aut = sws->aut; - FstOutput output = sws->emptyOutput; + FstOutput output = sws->emptyOutput; if (output.null == false) { - FstSlice emptySlice = fstSliceCreate(NULL, 0); + FstSlice emptySlice = fstSliceCreate(NULL, 0); if (fstBoundWithDataExceededBy(sws->endAt, &emptySlice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); - sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); return NULL; } void *start = automFuncs[aut->type].start(aut); @@ -1327,117 +1285,125 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb return swsResultCreate(&s, output, callback(start)); } } - SArray *nodes = taosArrayInit(8, sizeof(FstNode *)); + SArray *nodes = taosArrayInit(8, sizeof(FstNode *)); while (taosArrayGetSize(sws->stack) > 0) { - StreamState *p = (StreamState *)taosArrayPop(sws->stack); + StreamState *p = (StreamState *)taosArrayPop(sws->stack); if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); } - streamStateDestroy(p); + streamStateDestroy(p); continue; } - FstTransition trn; + FstTransition trn; fstNodeGetTransitionAt(p->node, p->trans, &trn); - Output out = p->out.out + trn.out; - void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); - void* tState = callback(nextState); - bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); - FstNode *nextNode = fstGetNode(sws->fst, trn.addr); - taosArrayPush(nodes, &nextNode); - taosArrayPush(sws->inp, &(trn.inp)); + Output out = p->out.out + trn.out; + void * nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); + void * tState = callback(nextState); + bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); + FstNode *nextNode = fstGetNode(sws->fst, trn.addr); + taosArrayPush(nodes, &nextNode); + taosArrayPush(sws->inp, &(trn.inp)); if (FST_NODE_IS_FINAL(nextNode)) { - //void *eofState = sws->aut->acceptEof(nextState); + // void *eofState = sws->aut->acceptEof(nextState); void *eofState = automFuncs[aut->type].acceptEof(aut, nextState); if (eofState != NULL) { isMatch = automFuncs[aut->type].isMatch(aut, eofState); } - } - StreamState s1 = { .node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; + } + StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; taosArrayPush(sws->stack, &s1); StreamState s2 = {.node = nextNode, .trans = 0, .out = {.null = false, .out = out}, .autState = nextState}; taosArrayPush(sws->stack, &s2); - - size_t isz = taosArrayGetSize(sws->inp); - uint8_t *buf = (uint8_t *)malloc(isz * sizeof(uint8_t)); + size_t isz = taosArrayGetSize(sws->inp); + uint8_t *buf = (uint8_t *)malloc(isz * sizeof(uint8_t)); for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t *)taosArrayGet(sws->inp, i); } FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); - sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); free(buf); fstSliceDestroy(&slice); return NULL; } if (FST_NODE_IS_FINAL(nextNode) && isMatch) { - FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; - StreamWithStateResult *result = swsResultCreate(&slice, fOutput, tState); + FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; + StreamWithStateResult *result = swsResultCreate(&slice, fOutput, tState); free(buf); fstSliceDestroy(&slice); - return result; + return result; } free(buf); fstSliceDestroy(&slice); } for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { - FstNode** node = (FstNode **)taosArrayGet(nodes, i); + FstNode **node = (FstNode **)taosArrayGet(nodes, i); fstNodeDestroy(*node); } taosArrayDestroy(nodes); - return NULL; - + return NULL; } StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state) { - StreamWithStateResult *result = calloc(1, sizeof(StreamWithStateResult)); - if (result == NULL) { return NULL; } - - result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); - result->out = fOut; - result->state = state; + StreamWithStateResult *result = calloc(1, sizeof(StreamWithStateResult)); + if (result == NULL) { + return NULL; + } + + result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); + result->out = fOut; + result->state = state; return result; } void swsResultDestroy(StreamWithStateResult *result) { - if (NULL == result) { return; } - + if (NULL == result) { + return; + } + fstSliceDestroy(&result->data); - startWithStateValueDestroy(result->state); + startWithStateValueDestroy(result->state); free(result); } void streamStateDestroy(void *s) { - if (NULL == s) { return; } + if (NULL == s) { + return; + } StreamState *ss = (StreamState *)s; fstNodeDestroy(ss->node); - //free(s->autoState); + // free(s->autoState); } FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) { FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder)); - if (NULL == b) { return NULL; } + if (NULL == b) { + return NULL; + } b->fst = fst; b->aut = aut; b->min = fstBoundStateCreate(Unbounded, NULL); - b->max = fstBoundStateCreate(Unbounded, NULL); + b->max = fstBoundStateCreate(Unbounded, NULL); return b; } void fstStreamBuilderDestroy(FstStreamBuilder *b) { fstSliceDestroy(&b->min->data); - tfree(b->min); + tfree(b->min); fstSliceDestroy(&b->max->data); tfree(b->max); free(b); } FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) { - if (b == NULL) { return NULL; } + if (b == NULL) { + return NULL; + } if (type == GE) { b->min->type = Included; @@ -1458,9 +1424,3 @@ FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, Rang } return b; } - - - - - - diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 07ad45079b172cc94fe90e5b84826156f098e857..cf3165709cdb961190d32dd646a54468c6d6ae95 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -15,44 +15,49 @@ #include "index_fst_automation.h" - StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val) { StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { return NULL; } + if (nsv == NULL) { + return NULL; + } nsv->kind = kind; nsv->type = ty; if (ty == FST_INT) { nsv->val = *(int *)val; } else if (ty == FST_CHAR) { - size_t len = strlen((char *)val); - nsv->ptr = (char *)calloc(1, len + 1); + size_t len = strlen((char *)val); + nsv->ptr = (char *)calloc(1, len + 1); memcpy(nsv->ptr, val, len); } else if (ty == FST_ARRAY) { - //TODO, - //nsv->arr = taosArrayFromList() + // TODO, + // nsv->arr = taosArrayFromList() } return nsv; } void startWithStateValueDestroy(void *val) { StartWithStateValue *sv = (StartWithStateValue *)val; - if (sv == NULL) { return; } + if (sv == NULL) { + return; + } if (sv->type == FST_INT) { - // + // } else if (sv->type == FST_CHAR) { free(sv->ptr); } else if (sv->type == FST_ARRAY) { taosArrayDestroy(sv->arr); } - free(sv); + free(sv); } StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { return NULL; } + if (nsv == NULL) { + return NULL; + } nsv->kind = sv->kind; - nsv->type= sv->type; + nsv->type = sv->type; if (nsv->type == FST_INT) { nsv->val = sv->val; } else if (nsv->type == FST_CHAR) { @@ -64,93 +69,67 @@ StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { return nsv; } - // prefix query, impl later -static void* prefixStart(AutomationCtx *ctx) { +static void *prefixStart(AutomationCtx *ctx) { StartWithStateValue *data = (StartWithStateValue *)(ctx->stdata); - return startWithStateValueDump(data); + return startWithStateValueDump(data); }; static bool prefixIsMatch(AutomationCtx *ctx, void *sv) { - StartWithStateValue* ssv = (StartWithStateValue *)sv; - return ssv->val == strlen(ctx->data); -} -static bool prefixCanMatch(AutomationCtx *ctx, void *sv) { - StartWithStateValue* ssv = (StartWithStateValue *)sv; - return ssv->val >= 0; + StartWithStateValue *ssv = (StartWithStateValue *)sv; + return ssv->val == strlen(ctx->data); } -static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { - return true; +static bool prefixCanMatch(AutomationCtx *ctx, void *sv) { + StartWithStateValue *ssv = (StartWithStateValue *)sv; + return ssv->val >= 0; } -static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { - StartWithStateValue* ssv = (StartWithStateValue *)state; - if (ssv == NULL || ctx == NULL) {return NULL;} +static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; } +static void *prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { + StartWithStateValue *ssv = (StartWithStateValue *)state; + if (ssv == NULL || ctx == NULL) { + return NULL; + } char *data = ctx->data; if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); } if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { - int val = ssv->val + 1; + int val = ssv->val + 1; StartWithStateValue *nsv = startWithStateValueCreate(Running, FST_INT, &val); if (prefixIsMatch(ctx, nsv)) { nsv->kind = Done; } else { nsv->kind = Running; - } + } return nsv; - } - return NULL; -} -static void* prefixAcceptEof(AutomationCtx *ctx, void *state) { + } return NULL; } +static void *prefixAcceptEof(AutomationCtx *ctx, void *state) { return NULL; } // pattern query, impl later -static void* patternStart(AutomationCtx *ctx) { - return NULL; -} -static bool patternIsMatch(AutomationCtx *ctx, void *data) { - return true; -} -static bool patternCanMatch(AutomationCtx *ctx, void *data) { - return true; -} -static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) { - return true; -} +static void *patternStart(AutomationCtx *ctx) { return NULL; } +static bool patternIsMatch(AutomationCtx *ctx, void *data) { return true; } +static bool patternCanMatch(AutomationCtx *ctx, void *data) { return true; } +static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) { return true; } -static void* patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) { - return NULL; -} +static void *patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) { return NULL; } -static void* patternAcceptEof(AutomationCtx *ctx, void *state) { - return NULL; -} +static void *patternAcceptEof(AutomationCtx *ctx, void *state) { return NULL; } -AutomationFunc automFuncs[] = {{ - prefixStart, - prefixIsMatch, - prefixCanMatch, - prefixWillAlwaysMatch, - prefixAccept, - prefixAcceptEof - }, - { - patternStart, - patternIsMatch, - patternCanMatch, - patternWillAlwaysMatch, - patternAccept, - patternAcceptEof - } - // add more search type +AutomationFunc automFuncs[] = { + {prefixStart, prefixIsMatch, prefixCanMatch, prefixWillAlwaysMatch, prefixAccept, prefixAcceptEof}, + {patternStart, patternIsMatch, patternCanMatch, patternWillAlwaysMatch, patternAccept, patternAcceptEof} + // add more search type }; -AutomationCtx* automCtxCreate(void *data,AutomationType atype) { +AutomationCtx *automCtxCreate(void *data, AutomationType atype) { AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx)); - if (ctx == NULL) { return NULL; } + if (ctx == NULL) { + return NULL; + } StartWithStateValue *sv = NULL; if (atype == AUTOMATION_PREFIX) { @@ -158,22 +137,21 @@ AutomationCtx* automCtxCreate(void *data,AutomationType atype) { sv = startWithStateValueCreate(Running, FST_INT, &val); ctx->stdata = (void *)sv; } else if (atype == AUTMMATION_MATCH) { - } else { // add more search type } - char* src = (char *)data; + char * src = (char *)data; size_t len = strlen(src); - char* dst = (char *)malloc(len * sizeof(char) + 1); + char * dst = (char *)malloc(len * sizeof(char) + 1); memcpy(dst, src, len); dst[len] = 0; - - ctx->data = dst; - ctx->type = atype; - ctx->stdata = (void *)sv; - return ctx; -} + + ctx->data = dst; + ctx->type = atype; + ctx->stdata = (void *)sv; + return ctx; +} void automCtxDestroy(AutomationCtx *ctx) { startWithStateValueDestroy(ctx->stdata); free(ctx->data); diff --git a/source/libs/index/src/index_fst_common.c b/source/libs/index/src/index_fst_common.c index 97fb88d60eb44ea684bb949b8f304535059074e7..ee3b07713da13008c29c863e164e72f9b56a3cda 100644 --- a/source/libs/index/src/index_fst_common.c +++ b/source/libs/index/src/index_fst_common.c @@ -12,296 +12,522 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - -#include "tutil.h" +#include "index_fst_common.h" const uint8_t COMMON_INPUTS[] = { - 84, // '\x00' - 85, // '\x01' - 86, // '\x02' - 87, // '\x03' - 88, // '\x04' - 89, // '\x05' - 90, // '\x06' - 91, // '\x07' - 92, // '\x08' - 93, // '\t' - 94, // '\n' - 95, // '\x0b' - 96, // '\x0c' - 97, // '\r' - 98, // '\x0e' - 99, // '\x0f' - 100, // '\x10' - 101, // '\x11' - 102, // '\x12' - 103, // '\x13' - 104, // '\x14' - 105, // '\x15' - 106, // '\x16' - 107, // '\x17' - 108, // '\x18' - 109, // '\x19' - 110, // '\x1a' - 111, // '\x1b' - 112, // '\x1c' - 113, // '\x1d' - 114, // '\x1e' - 115, // '\x1f' - 116, // ' ' - 80, // '!' - 117, // '"' - 118, // '#' - 79, // '$' - 39, // '%' - 30, // '&' - 81, // "'" - 75, // '(' - 74, // ')' - 82, // '*' - 57, // '+' - 66, // ',' - 16, // '-' - 12, // '.' - 2, // '/' - 19, // '0' - 20, // '1' - 21, // '2' - 27, // '3' - 32, // '4' - 29, // '5' - 35, // '6' - 36, // '7' - 37, // '8' - 34, // '9' - 24, // ':' - 73, // ';' - 119, // '<' - 23, // '=' - 120, // '>' - 40, // '?' - 83, // '@' - 44, // 'A' - 48, // 'B' - 42, // 'C' - 43, // 'D' - 49, // 'E' - 46, // 'F' - 62, // 'G' - 61, // 'H' - 47, // 'I' - 69, // 'J' - 68, // 'K' - 58, // 'L' - 56, // 'M' - 55, // 'N' - 59, // 'O' - 51, // 'P' - 72, // 'Q' - 54, // 'R' - 45, // 'S' - 52, // 'T' - 64, // 'U' - 65, // 'V' - 63, // 'W' - 71, // 'X' - 67, // 'Y' - 70, // 'Z' - 77, // '[' - 121, // '\\' - 78, // ']' - 122, // '^' - 31, // '_' - 123, // '`' - 4, // 'a' - 25, // 'b' - 9, // 'c' - 17, // 'd' - 1, // 'e' - 26, // 'f' - 22, // 'g' - 13, // 'h' - 7, // 'i' - 50, // 'j' - 38, // 'k' - 14, // 'l' - 15, // 'm' - 10, // 'n' - 3, // 'o' - 8, // 'p' - 60, // 'q' - 6, // 'r' - 5, // 's' - 0, // 't' - 18, // 'u' - 33, // 'v' - 11, // 'w' - 41, // 'x' - 28, // 'y' - 53, // 'z' - 124, // '{' - 125, // '|' - 126, // '}' - 76, // '~' - 127, // '\x7f' - 128, // '\x80' - 129, // '\x81' - 130, // '\x82' - 131, // '\x83' - 132, // '\x84' - 133, // '\x85' - 134, // '\x86' - 135, // '\x87' - 136, // '\x88' - 137, // '\x89' - 138, // '\x8a' - 139, // '\x8b' - 140, // '\x8c' - 141, // '\x8d' - 142, // '\x8e' - 143, // '\x8f' - 144, // '\x90' - 145, // '\x91' - 146, // '\x92' - 147, // '\x93' - 148, // '\x94' - 149, // '\x95' - 150, // '\x96' - 151, // '\x97' - 152, // '\x98' - 153, // '\x99' - 154, // '\x9a' - 155, // '\x9b' - 156, // '\x9c' - 157, // '\x9d' - 158, // '\x9e' - 159, // '\x9f' - 160, // '\xa0' - 161, // '¡' - 162, // '¢' - 163, // '£' - 164, // '¤' - 165, // '¥' - 166, // '¦' - 167, // '§' - 168, // '¨' - 169, // '©' - 170, // 'ª' - 171, // '«' - 172, // '¬' - 173, // '\xad' - 174, // '®' - 175, // '¯' - 176, // '°' - 177, // '±' - 178, // '²' - 179, // '³' - 180, // '´' - 181, // 'µ' - 182, // '¶' - 183, // '·' - 184, // '¸' - 185, // '¹' - 186, // 'º' - 187, // '»' - 188, // '¼' - 189, // '½' - 190, // '¾' - 191, // '¿' - 192, // 'À' - 193, // 'Á' - 194, // 'Â' - 195, // 'Ã' - 196, // 'Ä' - 197, // 'Å' - 198, // 'Æ' - 199, // 'Ç' - 200, // 'È' - 201, // 'É' - 202, // 'Ê' - 203, // 'Ë' - 204, // 'Ì' - 205, // 'Í' - 206, // 'Î' - 207, // 'Ï' - 208, // 'Ð' - 209, // 'Ñ' - 210, // 'Ò' - 211, // 'Ó' - 212, // 'Ô' - 213, // 'Õ' - 214, // 'Ö' - 215, // '×' - 216, // 'Ø' - 217, // 'Ù' - 218, // 'Ú' - 219, // 'Û' - 220, // 'Ü' - 221, // 'Ý' - 222, // 'Þ' - 223, // 'ß' - 224, // 'à' - 225, // 'á' - 226, // 'â' - 227, // 'ã' - 228, // 'ä' - 229, // 'å' - 230, // 'æ' - 231, // 'ç' - 232, // 'è' - 233, // 'é' - 234, // 'ê' - 235, // 'ë' - 236, // 'ì' - 237, // 'í' - 238, // 'î' - 239, // 'ï' - 240, // 'ð' - 241, // 'ñ' - 242, // 'ò' - 243, // 'ó' - 244, // 'ô' - 245, // 'õ' - 246, // 'ö' - 247, // '÷' - 248, // 'ø' - 249, // 'ù' - 250, // 'ú' - 251, // 'û' - 252, // 'ü' - 253, // 'ý' - 254, // 'þ' - 255, // 'ÿ' + 84, // '\x00' + 85, // '\x01' + 86, // '\x02' + 87, // '\x03' + 88, // '\x04' + 89, // '\x05' + 90, // '\x06' + 91, // '\x07' + 92, // '\x08' + 93, // '\t' + 94, // '\n' + 95, // '\x0b' + 96, // '\x0c' + 97, // '\r' + 98, // '\x0e' + 99, // '\x0f' + 100, // '\x10' + 101, // '\x11' + 102, // '\x12' + 103, // '\x13' + 104, // '\x14' + 105, // '\x15' + 106, // '\x16' + 107, // '\x17' + 108, // '\x18' + 109, // '\x19' + 110, // '\x1a' + 111, // '\x1b' + 112, // '\x1c' + 113, // '\x1d' + 114, // '\x1e' + 115, // '\x1f' + 116, // ' ' + 80, // '!' + 117, // '"' + 118, // '#' + 79, // '$' + 39, // '%' + 30, // '&' + 81, // "'" + 75, // '(' + 74, // ')' + 82, // '*' + 57, // '+' + 66, // ',' + 16, // '-' + 12, // '.' + 2, // '/' + 19, // '0' + 20, // '1' + 21, // '2' + 27, // '3' + 32, // '4' + 29, // '5' + 35, // '6' + 36, // '7' + 37, // '8' + 34, // '9' + 24, // ':' + 73, // ';' + 119, // '<' + 23, // '=' + 120, // '>' + 40, // '?' + 83, // '@' + 44, // 'A' + 48, // 'B' + 42, // 'C' + 43, // 'D' + 49, // 'E' + 46, // 'F' + 62, // 'G' + 61, // 'H' + 47, // 'I' + 69, // 'J' + 68, // 'K' + 58, // 'L' + 56, // 'M' + 55, // 'N' + 59, // 'O' + 51, // 'P' + 72, // 'Q' + 54, // 'R' + 45, // 'S' + 52, // 'T' + 64, // 'U' + 65, // 'V' + 63, // 'W' + 71, // 'X' + 67, // 'Y' + 70, // 'Z' + 77, // '[' + 121, // '\\' + 78, // ']' + 122, // '^' + 31, // '_' + 123, // '`' + 4, // 'a' + 25, // 'b' + 9, // 'c' + 17, // 'd' + 1, // 'e' + 26, // 'f' + 22, // 'g' + 13, // 'h' + 7, // 'i' + 50, // 'j' + 38, // 'k' + 14, // 'l' + 15, // 'm' + 10, // 'n' + 3, // 'o' + 8, // 'p' + 60, // 'q' + 6, // 'r' + 5, // 's' + 0, // 't' + 18, // 'u' + 33, // 'v' + 11, // 'w' + 41, // 'x' + 28, // 'y' + 53, // 'z' + 124, // '{' + 125, // '|' + 126, // '}' + 76, // '~' + 127, // '\x7f' + 128, // '\x80' + 129, // '\x81' + 130, // '\x82' + 131, // '\x83' + 132, // '\x84' + 133, // '\x85' + 134, // '\x86' + 135, // '\x87' + 136, // '\x88' + 137, // '\x89' + 138, // '\x8a' + 139, // '\x8b' + 140, // '\x8c' + 141, // '\x8d' + 142, // '\x8e' + 143, // '\x8f' + 144, // '\x90' + 145, // '\x91' + 146, // '\x92' + 147, // '\x93' + 148, // '\x94' + 149, // '\x95' + 150, // '\x96' + 151, // '\x97' + 152, // '\x98' + 153, // '\x99' + 154, // '\x9a' + 155, // '\x9b' + 156, // '\x9c' + 157, // '\x9d' + 158, // '\x9e' + 159, // '\x9f' + 160, // '\xa0' + 161, // '¡' + 162, // '¢' + 163, // '£' + 164, // '¤' + 165, // '¥' + 166, // '¦' + 167, // '§' + 168, // '¨' + 169, // '©' + 170, // 'ª' + 171, // '«' + 172, // '¬' + 173, // '\xad' + 174, // '®' + 175, // '¯' + 176, // '°' + 177, // '±' + 178, // '²' + 179, // '³' + 180, // '´' + 181, // 'µ' + 182, // '¶' + 183, // '·' + 184, // '¸' + 185, // '¹' + 186, // 'º' + 187, // '»' + 188, // '¼' + 189, // '½' + 190, // '¾' + 191, // '¿' + 192, // 'À' + 193, // 'Á' + 194, // 'Â' + 195, // 'Ã' + 196, // 'Ä' + 197, // 'Å' + 198, // 'Æ' + 199, // 'Ç' + 200, // 'È' + 201, // 'É' + 202, // 'Ê' + 203, // 'Ë' + 204, // 'Ì' + 205, // 'Í' + 206, // 'Î' + 207, // 'Ï' + 208, // 'Ð' + 209, // 'Ñ' + 210, // 'Ò' + 211, // 'Ó' + 212, // 'Ô' + 213, // 'Õ' + 214, // 'Ö' + 215, // '×' + 216, // 'Ø' + 217, // 'Ù' + 218, // 'Ú' + 219, // 'Û' + 220, // 'Ü' + 221, // 'Ý' + 222, // 'Þ' + 223, // 'ß' + 224, // 'à' + 225, // 'á' + 226, // 'â' + 227, // 'ã' + 228, // 'ä' + 229, // 'å' + 230, // 'æ' + 231, // 'ç' + 232, // 'è' + 233, // 'é' + 234, // 'ê' + 235, // 'ë' + 236, // 'ì' + 237, // 'í' + 238, // 'î' + 239, // 'ï' + 240, // 'ð' + 241, // 'ñ' + 242, // 'ò' + 243, // 'ó' + 244, // 'ô' + 245, // 'õ' + 246, // 'ö' + 247, // '÷' + 248, // 'ø' + 249, // 'ù' + 250, // 'ú' + 251, // 'û' + 252, // 'ü' + 253, // 'ý' + 254, // 'þ' + 255, // 'ÿ' }; const char COMMON_INPUTS_INV[] = { - 't', 'e', '/', 'o', 'a', 's', 'r', 'i', 'p', 'c', 'n', 'w', - '.', 'h', 'l', 'm', '-', 'd', 'u', '0', '1', '2', 'g', '=', - ':', 'b', 'f', '3', 'y', '5', '&', '_', '4', 'v', '9', '6', - '7', '8', 'k', '%', '?', 'x', 'C', 'D', 'A', 'S', 'F', 'I', - 'B', 'E', 'j', 'P', 'T', 'z', 'R', 'N', 'M', '+', 'L', 'O', - 'q', 'H', 'G', 'W', 'U', 'V', ',', 'Y', 'K', 'J', 'Z', 'X', - 'Q', ';', ')', '(', '~', '[', ']', '$', '!', '\'', '*', '@', - '\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', - '\x08', '\t', '\n', '\x0b', '\x0c', '\r', '\x0e', '\x0f', '\x10', - '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', '\x18', - '\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f', ' ', '"', - '#', '<', '>', '\\', '^', '`', '{', '|', '}','\x7f','\x80', - '\x81', '\x82', '\x83', '\x84', '\x85', '\x86', '\x87', '\x88', - '\x89', '\x8a', '\x8b', '\x8c', '\x8d', '\x8e', '\x8f', '\x90', - '\x91', '\x92', '\x93', '\x94', '\x95', '\x96', '\x97', '\x98', - '\x99', '\x9a', '\x9b', '\x9c', '\x9d', '\x9e', '\x9f', '\xa0', - '\xa1', '\xa2', '\xa3', '\xa4', '\xa5', '\xa6', '\xa7', '\xa8', - '\xa9', '\xaa', '\xab', '\xac', '\xad', '\xae', '\xaf', '\xb0', - '\xb1', '\xb2', '\xb3', '\xb4', '\xb5', '\xb6', '\xb7', '\xb8', - '\xb9', '\xba', '\xbb', '\xbc', '\xbd', '\xbe', '\xbf', '\xc0', - '\xc1', '\xc2', '\xc3', '\xc4', '\xc5', '\xc6', '\xc7', '\xc8', - '\xc9', '\xca', '\xcb', '\xcc', '\xcd', '\xce', '\xcf', '\xd0', - '\xd1', '\xd2', '\xd3', '\xd4', '\xd5', '\xd6', '\xd7', '\xd8', - '\xd9', '\xda', '\xdb', '\xdc', '\xdd', '\xde', '\xdf', '\xe0', - '\xe1', '\xe2', '\xe3', '\xe4', '\xe5', '\xe6', '\xe7', '\xe8', - '\xe9', '\xea', '\xeb', '\xec', '\xed', '\xee', '\xef', '\xf0', - '\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', '\xf8', - '\xf9', '\xfa', '\xfb', '\xfc', '\xfd', '\xfe', '\xff', + 't', + 'e', + '/', + 'o', + 'a', + 's', + 'r', + 'i', + 'p', + 'c', + 'n', + 'w', + '.', + 'h', + 'l', + 'm', + '-', + 'd', + 'u', + '0', + '1', + '2', + 'g', + '=', + ':', + 'b', + 'f', + '3', + 'y', + '5', + '&', + '_', + '4', + 'v', + '9', + '6', + '7', + '8', + 'k', + '%', + '?', + 'x', + 'C', + 'D', + 'A', + 'S', + 'F', + 'I', + 'B', + 'E', + 'j', + 'P', + 'T', + 'z', + 'R', + 'N', + 'M', + '+', + 'L', + 'O', + 'q', + 'H', + 'G', + 'W', + 'U', + 'V', + ',', + 'Y', + 'K', + 'J', + 'Z', + 'X', + 'Q', + ';', + ')', + '(', + '~', + '[', + ']', + '$', + '!', + '\'', + '*', + '@', + '\x00', + '\x01', + '\x02', + '\x03', + '\x04', + '\x05', + '\x06', + '\x07', + '\x08', + '\t', + '\n', + '\x0b', + '\x0c', + '\r', + '\x0e', + '\x0f', + '\x10', + '\x11', + '\x12', + '\x13', + '\x14', + '\x15', + '\x16', + '\x17', + '\x18', + '\x19', + '\x1a', + '\x1b', + '\x1c', + '\x1d', + '\x1e', + '\x1f', + ' ', + '"', + '#', + '<', + '>', + '\\', + '^', + '`', + '{', + '|', + '}', + '\x7f', + '\x80', + '\x81', + '\x82', + '\x83', + '\x84', + '\x85', + '\x86', + '\x87', + '\x88', + '\x89', + '\x8a', + '\x8b', + '\x8c', + '\x8d', + '\x8e', + '\x8f', + '\x90', + '\x91', + '\x92', + '\x93', + '\x94', + '\x95', + '\x96', + '\x97', + '\x98', + '\x99', + '\x9a', + '\x9b', + '\x9c', + '\x9d', + '\x9e', + '\x9f', + '\xa0', + '\xa1', + '\xa2', + '\xa3', + '\xa4', + '\xa5', + '\xa6', + '\xa7', + '\xa8', + '\xa9', + '\xaa', + '\xab', + '\xac', + '\xad', + '\xae', + '\xaf', + '\xb0', + '\xb1', + '\xb2', + '\xb3', + '\xb4', + '\xb5', + '\xb6', + '\xb7', + '\xb8', + '\xb9', + '\xba', + '\xbb', + '\xbc', + '\xbd', + '\xbe', + '\xbf', + '\xc0', + '\xc1', + '\xc2', + '\xc3', + '\xc4', + '\xc5', + '\xc6', + '\xc7', + '\xc8', + '\xc9', + '\xca', + '\xcb', + '\xcc', + '\xcd', + '\xce', + '\xcf', + '\xd0', + '\xd1', + '\xd2', + '\xd3', + '\xd4', + '\xd5', + '\xd6', + '\xd7', + '\xd8', + '\xd9', + '\xda', + '\xdb', + '\xdc', + '\xdd', + '\xde', + '\xdf', + '\xe0', + '\xe1', + '\xe2', + '\xe3', + '\xe4', + '\xe5', + '\xe6', + '\xe7', + '\xe8', + '\xe9', + '\xea', + '\xeb', + '\xec', + '\xed', + '\xee', + '\xef', + '\xf0', + '\xf1', + '\xf2', + '\xf3', + '\xf4', + '\xf5', + '\xf6', + '\xf7', + '\xf8', + '\xf9', + '\xfa', + '\xfb', + '\xfc', + '\xfd', + '\xfe', + '\xff', }; - diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 9ec346cebc026cba7d03e1cf39890a6c262d1a9a..824021c9e379fa0bc3239e1bad5b2acee525ffae 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -12,10 +12,10 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "tutil.h" +#include "index_fst_counting_writer.h" #include "indexInt.h" #include "index_fst_util.h" -#include "index_fst_counting_writer.h" +#include "tutil.h" static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { if (ctx->offset + len > ctx->limit) { @@ -23,121 +23,133 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { } if (ctx->type == TFile) { - assert(len == tfWrite(ctx->fd, buf, len)); + assert(len == tfWrite(ctx->file.fd, buf, len)); } else { - memcpy(ctx->mem + ctx->offset, buf, len); - } + memcpy(ctx->mem.buf + ctx->offset, buf, len); + } ctx->offset += len; return len; } static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { - int nRead = 0; + int nRead = 0; if (ctx->type == TFile) { - nRead = tfRead(ctx->fd, buf, len); + nRead = tfRead(ctx->file.fd, buf, len); } else { - memcpy(buf, ctx->mem + ctx->offset, len); + memcpy(buf, ctx->mem.buf + ctx->offset, len); } ctx->offset += nRead; return nRead; -} +} static int writeCtxDoFlush(WriterCtx *ctx) { if (ctx->type == TFile) { - //tfFsync(ctx->fd); - //tfFlush(ctx->fd); + // tfFsync(ctx->fd); + // tfFlush(ctx->file.fd); } else { // do nothing } return 1; } -WriterCtx* writerCtxCreate(WriterType type, bool readOnly) { +WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity) { WriterCtx *ctx = calloc(1, sizeof(WriterCtx)); - if (ctx == NULL) { return NULL; } + if (ctx == NULL) { + return NULL; + } ctx->type = type; if (ctx->type == TFile) { - tfInit(); // ugly code, refactor later + ctx->file.readOnly = readOnly; if (readOnly == false) { - ctx->fd = tfOpenCreateWriteAppend(tmpFile); + ctx->file.fd = tfOpenCreateWriteAppend(tmpFile); } else { - ctx->fd = tfOpenReadWrite(tmpFile); - } - if (ctx->fd < 0) { - indexError("open file error %d", errno); + ctx->file.fd = tfOpenReadWrite(tmpFile); + } + if (ctx->file.fd < 0) { + goto END; + indexError("open file error %d", errno); } } else if (ctx->type == TMemory) { - ctx->mem = calloc(1, DefaultMem * sizeof(uint8_t)); - } + ctx->mem.buf = calloc(1, sizeof(char) * capacity); + ctx->mem.capa = capacity; + } ctx->write = writeCtxDoWrite; - ctx->read = writeCtxDoRead; + ctx->read = writeCtxDoRead; ctx->flush = writeCtxDoFlush; ctx->offset = 0; - ctx->limit = DefaultMem; + ctx->limit = capacity; return ctx; +END: + if (ctx->type == TMemory) { + free(ctx->mem.buf); + } + free(ctx); } void writerCtxDestroy(WriterCtx *ctx) { if (ctx->type == TMemory) { - free(ctx->mem); + free(ctx->mem.buf); } else { - tfClose(ctx->fd); - tfCleanup(); + tfClose(ctx->file.fd); } free(ctx); } +FstCountingWriter *fstCountingWriterCreate(void *wrt) { + FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); + if (cw == NULL) { + return NULL; + } -FstCountingWriter *fstCountingWriterCreate(void *wrt, bool readOnly) { - FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); - if (cw == NULL) { return NULL; } - - cw->wrt = (void *)(writerCtxCreate(TFile, readOnly)); - return cw; + cw->wrt = wrt; + //(void *)(writerCtxCreate(TFile, readOnly)); + return cw; } void fstCountingWriterDestroy(FstCountingWriter *cw) { - // free wrt object: close fd or free mem + // free wrt object: close fd or free mem fstCountingWriterFlush(cw); - writerCtxDestroy((WriterCtx *)(cw->wrt)); + // writerCtxDestroy((WriterCtx *)(cw->wrt)); free(cw); } int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len) { - if (write == NULL) { return 0; } - // update checksum + if (write == NULL) { + return 0; + } + // update checksum // write data to file/socket or mem WriterCtx *ctx = write->wrt; - int nWrite = ctx->write(ctx, buf, len); + int nWrite = ctx->write(ctx, buf, len); assert(nWrite == len); write->count += len; - return len; -} + return len; +} int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len) { - if (write == NULL) { return 0; } + if (write == NULL) { + return 0; + } WriterCtx *ctx = write->wrt; - int nRead = ctx->read(ctx, buf, len); - //assert(nRead == len); - return nRead; -} - -uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { - return 0; + int nRead = ctx->read(ctx, buf, len); + // assert(nRead == len); + return nRead; } -int fstCountingWriterFlush(FstCountingWriter *write) { + +uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { return 0; } +int fstCountingWriterFlush(FstCountingWriter *write) { WriterCtx *ctx = write->wrt; ctx->flush(ctx); - //write->wtr->flush + // write->wtr->flush return 1; } -void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes) { +void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes) { assert(1 <= nBytes && nBytes <= 8); - uint8_t *buf = calloc(8, sizeof(uint8_t)); + uint8_t *buf = calloc(8, sizeof(uint8_t)); for (uint8_t i = 0; i < nBytes; i++) { - buf[i] = (uint8_t)n; + buf[i] = (uint8_t)n; n = n >> 8; } fstCountingWriterWrite(writer, buf, nBytes); @@ -148,7 +160,5 @@ void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n) { uint8_t nBytes = packSize(n); fstCountingWriterPackUintIn(writer, n, nBytes); - return nBytes; -} - - + return nBytes; +} diff --git a/source/libs/index/src/index_fst_node.c b/source/libs/index/src/index_fst_node.c index 5abe8ad5a04af47f68d562705f3c2c569c3f6a0b..084f280bc3d3b8d224824005b0bb9b3a9d4ec894 100644 --- a/source/libs/index/src/index_fst_node.c +++ b/source/libs/index/src/index_fst_node.c @@ -16,30 +16,34 @@ FstBuilderNode *fstBuilderNodeDefault() { FstBuilderNode *bn = malloc(sizeof(FstBuilderNode)); - bn->isFinal = false; - bn->finalOutput = 0; - bn->trans = taosArrayInit(16, sizeof(FstTransition)); + bn->isFinal = false; + bn->finalOutput = 0; + bn->trans = taosArrayInit(16, sizeof(FstTransition)); return bn; } void fstBuilderNodeDestroy(FstBuilderNode *node) { - if (node == NULL) { return; } + if (node == NULL) { + return; + } taosArrayDestroy(node->trans); free(node); -} +} bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) { - if (n1 == n2) { return true; } - if (n1 == NULL || n2 == NULL ) { + if (n1 == n2) { + return true; + } + if (n1 == NULL || n2 == NULL) { return false; } if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) { return false; } - size_t s1 = n1->trans? taosArrayGetSize(n1->trans): 0; - size_t s2 = n2->trans? taosArrayGetSize(n2->trans): 0; - if (s1 != s2) { + size_t s1 = n1->trans ? taosArrayGetSize(n1->trans) : 0; + size_t s2 = n2->trans ? taosArrayGetSize(n2->trans) : 0; + if (s1 != s2) { return false; } for (size_t i = 0; i < s1; i++) { @@ -47,69 +51,70 @@ bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) { FstTransition *t2 = taosArrayGet(n2->trans, i); if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { return false; - } + } } - + return true; } FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) { - FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); - if (node == NULL) { return NULL; } + FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); + if (node == NULL) { + return NULL; + } - // - size_t sz = taosArrayGetSize(src->trans); + // + size_t sz = taosArrayGetSize(src->trans); SArray *trans = taosArrayInit(sz, sizeof(FstTransition)); for (size_t i = 0; i < sz; i++) { FstTransition *tran = taosArrayGet(src->trans, i); - taosArrayPush(trans, tran); + taosArrayPush(trans, tran); } - node->trans = trans; + node->trans = trans; node->isFinal = src->isFinal; node->finalOutput = src->finalOutput; return node; - } -// not destroy src, User's bussiness +// not destroy src, User's bussiness void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) { - if (dst == NULL || src == NULL) { return; } + if (dst == NULL || src == NULL) { + return; + } - dst->isFinal = src->isFinal; + dst->isFinal = src->isFinal; dst->finalOutput = src->finalOutput; - //release free avoid mem leak - taosArrayDestroy(dst->trans); + // release free avoid mem leak + taosArrayDestroy(dst->trans); size_t sz = taosArrayGetSize(src->trans); - dst->trans = taosArrayInit(sz, sizeof(FstTransition)); + dst->trans = taosArrayInit(sz, sizeof(FstTransition)); for (size_t i = 0; i < sz; i++) { - FstTransition *trn = taosArrayGet(src->trans, i); + FstTransition *trn = taosArrayGet(src->trans, i); taosArrayPush(dst->trans, trn); - } + } } +// bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr +// startAddr) { -//bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { - - //size_t sz = taosArrayGetSize(b->trans); - //assert(sz < 256); - //if (FST_BUILDER_NODE_IS_FINAL(b) - // && FST_BUILDER_NODE_TRANS_ISEMPTY(b) - // && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(b)) { - // return true; - //} else if (sz != 1 || b->isFinal) { - // // AnyTrans->Compile(w, addr, node); - //} else { - // FstTransition *tran = taosArrayGet(b->trans, 0); - // if (tran->addr == lastAddr && tran->out == 0) { - // //OneTransNext::compile(w, lastAddr, tran->inp); - // return true; - // } else { - // //OneTrans::Compile(w, lastAddr, *tran); - // return true; - // } - //} - //return true; -//} - - +// size_t sz = taosArrayGetSize(b->trans); +// assert(sz < 256); +// if (FST_BUILDER_NODE_IS_FINAL(b) +// && FST_BUILDER_NODE_TRANS_ISEMPTY(b) +// && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(b)) { +// return true; +//} else if (sz != 1 || b->isFinal) { +// // AnyTrans->Compile(w, addr, node); +//} else { +// FstTransition *tran = taosArrayGet(b->trans, 0); +// if (tran->addr == lastAddr && tran->out == 0) { +// //OneTransNext::compile(w, lastAddr, tran->inp); +// return true; +// } else { +// //OneTrans::Compile(w, lastAddr, *tran); +// return true; +// } +//} +// return true; +//} diff --git a/source/libs/index/src/index_fst_registry.c b/source/libs/index/src/index_fst_registry.c index 8fb0dbfcaa2f86d40b270b197e273802e95ddfe7..7bb2e722307a0b676fd6eb7485a3502508e9bbd9 100644 --- a/source/libs/index/src/index_fst_registry.c +++ b/source/libs/index/src/index_fst_registry.c @@ -15,33 +15,33 @@ #include "index_fst_registry.h" - uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) { - //TODO(yihaoDeng): refactor later + // TODO(yihaoDeng): refactor later const uint64_t FNV_PRIME = 1099511628211; - uint64_t h = 14695981039346656037u; + uint64_t h = 14695981039346656037u; - h = (h ^ (uint64_t)bNode->isFinal) * FNV_PRIME; + h = (h ^ (uint64_t)bNode->isFinal) * FNV_PRIME; h = (h ^ (bNode)->finalOutput) * FNV_PRIME; - uint32_t sz = (uint32_t)taosArrayGetSize(bNode->trans); + uint32_t sz = (uint32_t)taosArrayGetSize(bNode->trans); for (uint32_t i = 0; i < sz; i++) { FstTransition *trn = taosArrayGet(bNode->trans, i); - h = (h ^ (uint64_t)(trn->inp)) * FNV_PRIME; - h = (h ^ (uint64_t)(trn->out)) * FNV_PRIME; - h = (h ^ (uint64_t)(trn->addr))* FNV_PRIME; - } - return h %(registry->tableSize); - + h = (h ^ (uint64_t)(trn->inp)) * FNV_PRIME; + h = (h ^ (uint64_t)(trn->out)) * FNV_PRIME; + h = (h ^ (uint64_t)(trn->addr)) * FNV_PRIME; + } + return h % (registry->tableSize); } static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) { size_t sz = taosArrayGetSize(arr); - if (a >= sz || b >= sz) { return; } + if (a >= sz || b >= sz) { + return; + } - FstRegistryCell *cell1 = (FstRegistryCell *)taosArrayGet(arr, a); + FstRegistryCell *cell1 = (FstRegistryCell *)taosArrayGet(arr, a); FstRegistryCell *cell2 = (FstRegistryCell *)taosArrayGet(arr, b); - FstRegistryCell t = {.addr = cell1->addr, .node = cell1->node}; + FstRegistryCell t = {.addr = cell1->addr, .node = cell1->node}; cell1->addr = cell2->addr; cell1->node = cell2->node; @@ -52,49 +52,55 @@ static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) { } static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) { - size_t sz = taosArrayGetSize(arr); - if (start >= sz && end >= sz) {return; } - + size_t sz = taosArrayGetSize(arr); + if (start >= sz && end >= sz) { + return; + } + assert(start >= end); int32_t s = (int32_t)start; int32_t e = (int32_t)end; - while(s > e) { + while (s > e) { fstRegistryCellSwap(arr, s - 1, s); s -= 1; } } -FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { - FstRegistry *registry = malloc(sizeof(FstRegistry)); - if (registry == NULL) { return NULL ;} +FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { + FstRegistry *registry = malloc(sizeof(FstRegistry)); + if (registry == NULL) { + return NULL; + } - uint64_t nCells = tableSize * mruSize; - SArray* tb = (SArray *)taosArrayInit(nCells, sizeof(FstRegistryCell)); - if (NULL == tb) { - free(registry); - return NULL; + uint64_t nCells = tableSize * mruSize; + SArray * tb = (SArray *)taosArrayInit(nCells, sizeof(FstRegistryCell)); + if (NULL == tb) { + free(registry); + return NULL; } for (uint64_t i = 0; i < nCells; i++) { - FstRegistryCell cell = {.addr = NONE_ADDRESS, .node = fstBuilderNodeDefault()}; - taosArrayPush(tb, &cell); + FstRegistryCell cell = {.addr = NONE_ADDRESS, .node = fstBuilderNodeDefault()}; + taosArrayPush(tb, &cell); } - - registry->table = tb; - registry->tableSize = tableSize; - registry->mruSize = mruSize; - return registry; + + registry->table = tb; + registry->tableSize = tableSize; + registry->mruSize = mruSize; + return registry; } void fstRegistryDestroy(FstRegistry *registry) { - if (registry == NULL) { return; } + if (registry == NULL) { + return; + } SArray *tb = registry->table; - size_t sz = taosArrayGetSize(tb); + size_t sz = taosArrayGetSize(tb); for (size_t i = 0; i < sz; i++) { - FstRegistryCell *cell = taosArrayGet(tb, i); - fstBuilderNodeDestroy(cell->node); + FstRegistryCell *cell = taosArrayGet(tb, i); + fstBuilderNodeDestroy(cell->node); } taosArrayDestroy(tb); free(registry); @@ -102,74 +108,70 @@ void fstRegistryDestroy(FstRegistry *registry) { FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode) { if (taosArrayGetSize(registry->table) <= 0) { - return NULL; - } + return NULL; + } uint64_t bucket = fstRegistryHash(registry, bNode); - uint64_t start = registry->mruSize * bucket; - uint64_t end = start + registry->mruSize; - + uint64_t start = registry->mruSize * bucket; + uint64_t end = start + registry->mruSize; + FstRegistryEntry *entry = malloc(sizeof(FstRegistryEntry)); if (end - start == 1) { - FstRegistryCell *cell = taosArrayGet(registry->table, start); - //cell->isNode && + FstRegistryCell *cell = taosArrayGet(registry->table, start); + // cell->isNode && if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { - entry->state = FOUND; - entry->addr = cell->addr ; - return entry; + entry->state = FOUND; + entry->addr = cell->addr; + return entry; } else { - fstBuilderNodeCloneFrom(cell->node, bNode); - entry->state = NOTFOUND; - entry->cell = cell; // copy or not + fstBuilderNodeCloneFrom(cell->node, bNode); + entry->state = NOTFOUND; + entry->cell = cell; // copy or not } } else if (end - start == 2) { - FstRegistryCell *cell1 = taosArrayGet(registry->table, start); + FstRegistryCell *cell1 = taosArrayGet(registry->table, start); if (cell1->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell1->node, bNode)) { - entry->state = FOUND; - entry->addr = cell1->addr; + entry->state = FOUND; + entry->addr = cell1->addr; return entry; - } - FstRegistryCell *cell2 = taosArrayGet(registry->table, start + 1); + } + FstRegistryCell *cell2 = taosArrayGet(registry->table, start + 1); if (cell2->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell2->node, bNode)) { - entry->state = FOUND; - entry->addr = cell2->addr; + entry->state = FOUND; + entry->addr = cell2->addr; // must swap here - fstRegistryCellSwap(registry->table, start, start + 1); - return entry; + fstRegistryCellSwap(registry->table, start, start + 1); + return entry; } - //clone from bNode, refactor later + // clone from bNode, refactor later fstBuilderNodeCloneFrom(cell2->node, bNode); fstRegistryCellSwap(registry->table, start, start + 1); FstRegistryCell *cCell = taosArrayGet(registry->table, start); - entry->state = NOTFOUND; - entry->cell = cCell; + entry->state = NOTFOUND; + entry->cell = cCell; } else { - uint32_t i = start; + uint32_t i = start; for (; i < end; i++) { FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, i); if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { - entry->state = FOUND; - entry->addr = cell->addr; + entry->state = FOUND; + entry->addr = cell->addr; fstRegistryCellPromote(registry->table, i, start); break; } - } + } if (i >= end) { - uint64_t last = end - 1; - FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, last); - //clone from bNode, refactor later - fstBuilderNodeCloneFrom(cell->node, bNode); + uint64_t last = end - 1; + FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, last); + // clone from bNode, refactor later + fstBuilderNodeCloneFrom(cell->node, bNode); fstRegistryCellPromote(registry->table, last, start); FstRegistryCell *cCell = taosArrayGet(registry->table, start); - entry->state = NOTFOUND; - entry->cell = cCell; + entry->state = NOTFOUND; + entry->cell = cCell; } - } + } return entry; } -void fstRegistryEntryDestroy(FstRegistryEntry *entry) { - free(entry); -} - - +void fstRegistryEntryDestroy(FstRegistryEntry *entry) { free(entry); } diff --git a/source/libs/index/src/index_fst_util.c b/source/libs/index/src/index_fst_util.c index c933c6d23b2eed040bcf9fd276d783c13b7ad025..597f7cc61ade3bedd5e2814c166e86e1433c71f6 100644 --- a/source/libs/index/src/index_fst_util.c +++ b/source/libs/index/src/index_fst_util.c @@ -15,37 +15,32 @@ #include "index_fst_util.h" #include "index_fst_common.h" - - -//A sentinel value used to indicate an empty final state -const CompiledAddr EMPTY_ADDRESS = 0; +// A sentinel value used to indicate an empty final state +const CompiledAddr EMPTY_ADDRESS = 0; /// A sentinel value used to indicate an invalid state. -const CompiledAddr NONE_ADDRESS = 1; +const CompiledAddr NONE_ADDRESS = 1; // This version number is written to every finite state transducer created by // this crate. When a finite state transducer is read, its version number is // checked against this value. -const uint64_t VERSION = 3; -// The threshold (in number of transitions) at which an index is created for -// a node's transitions. This speeds up lookup time at the expense of FST size +const uint64_t VERSION = 3; +// The threshold (in number of transitions) at which an index is created for +// a node's transitions. This speeds up lookup time at the expense of FST size const uint64_t TRANS_INDEX_THRESHOLD = 32; - -//uint8_t commonInput(uint8_t idx) { +// uint8_t commonInput(uint8_t idx) { // if (idx == 0) { return -1; } // else { -// return COMMON_INPUTS_INV[idx - 1]; +// return COMMON_INPUTS_INV[idx - 1]; // } -//} +//} // -//uint8_t commonIdx(uint8_t v, uint8_t max) { +// uint8_t commonIdx(uint8_t v, uint8_t max) { // uint8_t v = ((uint16_t)tCOMMON_INPUTS[v] + 1)%256; // return v > max ? 0: v; //} - - uint8_t packSize(uint64_t n) { if (n < (1u << 8)) { return 1; @@ -71,17 +66,17 @@ uint64_t unpackUint64(uint8_t *ch, uint8_t sz) { for (uint8_t i = 0; i < sz; i++) { n = n | (ch[i] << (8 * i)); } - return n; + return n; } uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr) { if (transAddr == EMPTY_ADDRESS) { - return packSize(EMPTY_ADDRESS); + return packSize(EMPTY_ADDRESS); } else { return packSize(nodeAddr - transAddr); - } -} + } +} CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) { - uint64_t delta = unpackUint64(data, len); + uint64_t delta = unpackUint64(data, len); // delta_add = u64_to_usize if (delta == EMPTY_ADDRESS) { return EMPTY_ADDRESS; @@ -95,56 +90,53 @@ CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) { FstSlice fstSliceCreate(uint8_t *data, uint64_t len) { FstString *str = (FstString *)malloc(sizeof(FstString)); - str->ref = 1; - str->len = len; + str->ref = 1; + str->len = len; str->data = malloc(len * sizeof(uint8_t)); memcpy(str->data, data, len); - + FstSlice s = {.str = str, .start = 0, .end = len - 1}; return s; -} +} // just shallow copy FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) { FstString *str = s->str; str->ref++; - //uint8_t *buf = fstSliceData(s, &alen); - //start = buf + start - (buf - s->start); - //end = buf + end - (buf - s->start); + // uint8_t *buf = fstSliceData(s, &alen); + // start = buf + start - (buf - s->start); + // end = buf + end - (buf - s->start); FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start}; return t; } FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { - - int32_t tlen = end - start + 1; - int32_t slen; - uint8_t *data = fstSliceData(s, &slen); + int32_t tlen = end - start + 1; + int32_t slen; + uint8_t *data = fstSliceData(s, &slen); assert(tlen <= slen); - uint8_t *buf = malloc(sizeof(uint8_t) * tlen); + uint8_t *buf = malloc(sizeof(uint8_t) * tlen); memcpy(buf, data + start, tlen); - - FstString *str = malloc(sizeof(FstString)); + + FstString *str = malloc(sizeof(FstString)); str->data = buf; - str->len = tlen; - str->ref = 1; + str->len = tlen; + str->ref = 1; FstSlice ans; - ans.str = str; - ans.start = 0; - ans.end = tlen - 1; - return ans; -} -bool fstSliceIsEmpty(FstSlice *s) { - return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; + ans.str = str; + ans.start = 0; + ans.end = tlen - 1; + return ans; } +bool fstSliceIsEmpty(FstSlice *s) { return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; } uint8_t *fstSliceData(FstSlice *s, int32_t *size) { - FstString *str = s->str; - if (size != NULL) { + FstString *str = s->str; + if (size != NULL) { *size = s->end - s->start + 1; - } - return str->data + s->start; + } + return str->data + s->start; } void fstSliceDestroy(FstSlice *s) { FstString *str = s->str; @@ -152,40 +144,45 @@ void fstSliceDestroy(FstSlice *s) { if (str->ref <= 0) { free(str->data); free(str); - s->str = NULL; + s->str = NULL; } } int fstSliceCompare(FstSlice *a, FstSlice *b) { - int32_t alen, blen; - uint8_t *aBuf = fstSliceData(a, &alen); - uint8_t *bBuf = fstSliceData(b, &blen); + int32_t alen, blen; + uint8_t *aBuf = fstSliceData(a, &alen); + uint8_t *bBuf = fstSliceData(b, &blen); uint32_t i, j; for (i = 0, j = 0; i < alen && j < blen; i++, j++) { uint8_t x = aBuf[i]; uint8_t y = bBuf[j]; - if (x == y) { continue;} - else if (x < y) { return -1; } - else { return 1; }; + if (x == y) { + continue; + } else if (x < y) { + return -1; + } else { + return 1; + }; + } + if (i < alen) { + return 1; + } else if (j < blen) { + return -1; + } else { + return 0; } - if (i < alen) { return 1; } - else if (j < blen) { return -1; } - else { return 0; } -} +} -//FstStack* fstStackCreate(size_t elemSize, StackFreeElem freeFn) { +// FstStack* fstStackCreate(size_t elemSize, StackFreeElem freeFn) { // FstStack *s = calloc(1, sizeof(FstStack)); // if (s == NULL) { return NULL; } -// s-> -// s->freeFn -// +// s-> +// s->freeFn +// //} -//void *fstStackPush(FstStack *s, void *elem); -//void *fstStackTop(FstStack *s); -//size_t fstStackLen(FstStack *s); -//void *fstStackGetAt(FstStack *s, size_t i); -//void fstStackDestory(FstStack *); - - - +// void *fstStackPush(FstStack *s, void *elem); +// void *fstStackTop(FstStack *s); +// size_t fstStackLen(FstStack *s); +// void *fstStackGetAt(FstStack *s, size_t i); +// void fstStackDestory(FstStack *); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index a1bba56391ce445899fe1fc23b8e422c8cf3cd8f..19e8ff67506ba0bc963ab910eaffa2f5754f6b60 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -13,19 +13,195 @@ * along with this program. If not, see . */ +//#include +//#include #include "index_tfile.h" +#include "index.h" +#include "index_fst.h" +#include "index_fst_counting_writer.h" +#include "index_util.h" +#include "taosdef.h" -IndexTFile *indexTFileCreate() { - IndexTFile *tfile = calloc(1, sizeof(IndexTFile)); - return tfile; +static FORCE_INLINE int tfileReadLoadHeader(TFileReader *reader) { + // TODO simple tfile header later + char buf[TFILE_HADER_PRE_SIZE]; + char * p = buf; + TFileReadHeader *header = &reader->header; + int64_t nread = reader->ctx->read(reader->ctx, buf, TFILE_HADER_PRE_SIZE); + assert(nread == TFILE_HADER_PRE_SIZE); + + memcpy(&header->suid, p, sizeof(header->suid)); + p += sizeof(header->suid); + + memcpy(&header->version, p, sizeof(header->version)); + p += sizeof(header->version); + + int32_t colLen = 0; + memcpy(&colLen, p, sizeof(colLen)); + assert(colLen < sizeof(header->colName)); + nread = reader->ctx->read(reader->ctx, header->colName, colLen); + assert(nread == colLen); + + nread = reader->ctx->read(reader->ctx, &header->colType, sizeof(header->colType)); + return 0; +}; +static int tfileGetFileList(const char *path, SArray *result) { + DIR *dir = opendir(path); + if (NULL == dir) { + return -1; + } + + struct dirent *entry; + while ((entry = readdir(dir)) != NULL) { + size_t len = strlen(entry->d_name); + char * buf = calloc(1, len + 1); + memcpy(buf, entry->d_name, len); + taosArrayPush(result, &buf); + } + closedir(dir); + return 0; } -void IndexTFileDestroy(IndexTFile *tfile) { - free(tfile); +static void tfileDestroyFileName(void *elem) { + char *p = *(char **)elem; + free(p); } -int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) { - IndexTFile *ptfile = (IndexTFile *)tfile; - return 0; +static int tfileCompare(const void *a, const void *b) { + const char *aName = *(char **)a; + const char *bName = *(char **)b; + size_t aLen = strlen(aName); + size_t bLen = strlen(bName); + return strncmp(aName, bName, aLen > bLen ? aLen : bLen); +} +// tfile name suid-colId-version.tindex +static int tfileParseFileName(const char *filename, uint64_t *suid, int *colId, int *version) { + if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) { + // read suid & colid & version success + return 0; + } + return -1; +} +static void tfileSerialCacheKey(TFileCacheKey *key, char *buf) { + SERIALIZE_MEM_TO_BUF(buf, key, suid); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_MEM_TO_BUF(buf, key, colType); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_MEM_TO_BUF(buf, key, version); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); } +TFileCache *tfileCacheCreate(const char *path) { + TFileCache *tcache = calloc(1, sizeof(TFileCache)); + if (tcache == NULL) { + return NULL; + } + tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + tcache->capacity = 64; + SArray *files = taosArrayInit(4, sizeof(void *)); + tfileGetFileList(path, files); + taosArraySort(files, tfileCompare); + for (size_t i = 0; i < taosArrayGetSize(files); i++) { + char * file = taosArrayGetP(files, i); + uint64_t suid; + int colId, version; + if (0 != tfileParseFileName(file, &suid, &colId, &version)) { + goto End; + continue; + } + + WriterCtx *wc = writerCtxCreate(TFile, file, true, 1024 * 64); + if (wc == NULL) { + indexError("failed to open index: %s", file); + goto End; + } + TFileReader *reader = tfileReaderCreate(wc); + if (0 != tfileReadLoadHeader(reader)) { + TFileReaderDestroy(reader); + indexError("failed to load index header, index Id: %s", file); + goto End; + } + } + taosArrayDestroyEx(files, tfileDestroyFileName); + return tcache; +End: + tfileCacheDestroy(tcache); + taosArrayDestroyEx(files, tfileDestroyFileName); + return NULL; +} +void tfileCacheDestroy(TFileCache *tcache) { + if (tcache == NULL) { + return; + } + + // free table cache + TFileReader **reader = taosHashIterate(tcache->tableCache, NULL); + while (reader) { + TFileReader *p = *reader; + indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, + p->header.colType); + TFileReaderDestroy(p); + reader = taosHashIterate(tcache->tableCache, reader); + } + taosHashCleanup(tcache->tableCache); + free(tcache); +} + +TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key) { + char buf[128] = {0}; + tfileSerialCacheKey(key, buf); + TFileReader *reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); + return reader; +} +void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader) { + char buf[128] = {0}; + tfileSerialCacheKey(key, buf); + taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void *)); + return; +} + +TFileReader *tfileReaderCreate(WriterCtx *ctx) { + TFileReader *reader = calloc(1, sizeof(TFileReader)); + if (reader == NULL) { + return NULL; + } + reader->ctx = ctx; + // T_REF_INC(reader); + return reader; +} +void TFileReaderDestroy(TFileReader *reader) { + if (reader == NULL) { + return; + } + // T_REF_INC(reader); + writerCtxDestroy(reader->ctx); + free(reader); +} + +TFileWriter *tfileWriterCreate(const char *suid, const char *colName); +void tfileWriterDestroy(TFileWriter *tw); + +IndexTFile *indexTFileCreate(const char *path) { + IndexTFile *tfile = calloc(1, sizeof(IndexTFile)); + tfile->cache = tfileCacheCreate(path); + + return tfile; +} +void IndexTFileDestroy(IndexTFile *tfile) { free(tfile); } + +int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) { + IndexTFile *pTfile = (IndexTFile *)tfile; + + SIndexTerm * term = query->term; + TFileCacheKey key = { + .suid = term->suid, .colType = term->colType, .version = 0, .colName = term->colName, .nColName = term->nColName}; + TFileReader *reader = tfileCacheGet(pTfile->cache, &key); + return 0; +} +int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid) { + TFileWriterOpt wOpt = { + .suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = 1}; + + return 0; +} diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 9dff2e9ea0ac901390cde63feee01f50c3179fe6..85d76bb716a887bf7e7b07d54afde9ae67f431de 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -26,6 +26,7 @@ class FstWriter { public: FstWriter() { + _wc = writerCtxCreate(TFile, "/tmp/tindex", false, 0); _b = fstBuilderCreate(NULL, 0); } bool Put(const std::string &key, uint64_t val) { @@ -37,15 +38,19 @@ class FstWriter { ~FstWriter() { fstBuilderFinish(_b); fstBuilderDestroy(_b); + + writerCtxDestroy(_wc); } private: FstBuilder *_b; + WriterCtx *_wc; }; class FstReadMemory { public: FstReadMemory(size_t size) { - _w = fstCountingWriterCreate(NULL, true); + _wc = writerCtxCreate(TFile, "/tmp/tindex", true, 0); + _w = fstCountingWriterCreate(_wc); _size = size; memset((void *)&_s, 0, sizeof(_s)); } @@ -94,12 +99,14 @@ class FstReadMemory { fstCountingWriterDestroy(_w); fstDestroy(_fst); fstSliceDestroy(&_s); + writerCtxDestroy(_wc); } private: FstCountingWriter *_w; Fst *_fst; FstSlice _s; + WriterCtx *_wc; size_t _size; }; diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index de7cdd58b85227c3a4dbb1742a0789fe01413802..1771bdc0edbeeb649f2c02e583e6d525df20a894 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -4,7 +4,9 @@ #include "parserInt.h" #include "taosmsg.h" -SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen); +SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); +SCreateAcctMsg* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); +SDropUserMsg* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen); SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen); diff --git a/source/libs/parser/inc/queryInfoUtil.h b/source/libs/parser/inc/queryInfoUtil.h index 5d51293939381bac3274864457e2e413e3184445..798c9bc97fde7f93b8d0a38e2927d124dd7c0cd2 100644 --- a/source/libs/parser/inc/queryInfoUtil.h +++ b/source/libs/parser/inc/queryInfoUtil.h @@ -36,7 +36,6 @@ void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, i void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize); SExprInfo* getExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index); -int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy); void addExprInfoParam(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes); diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 1b46faececee5dabb589d8fdae2bccc0ddf0aca1..c41ba97e39f3a080190989ee80d5c03a9b50d2e0 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -1,7 +1,7 @@ #include "parserInt.h" #include "parserUtil.h" -SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen) { +SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen) { SCreateUserMsg* pMsg = (SCreateUserMsg*)calloc(1, sizeof(SCreateUserMsg)); if (pMsg == NULL) { // tscError("0x%" PRIx64 " failed to malloc for query msg", id); @@ -20,6 +20,68 @@ SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgB strncpy(pMsg->pass, pUser->passwd.z, pUser->passwd.n); } + *outputLen = sizeof(SUserInfo); + return pMsg; +} + +SCreateAcctMsg* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen) { + SCreateAcctMsg* pMsg = (SCreateAcctMsg*)calloc(1, sizeof(SCreateAcctMsg)); + if (pMsg == NULL) { + // tscError("0x%" PRIx64 " failed to malloc for query msg", id); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + SCreateAcctMsg *pCreateMsg = (SCreateAcctMsg *) calloc(1, sizeof(SCreateAcctMsg)); + + SToken *pName = &pInfo->pMiscInfo->user.user; + SToken *pPwd = &pInfo->pMiscInfo->user.passwd; + + strncpy(pCreateMsg->user, pName->z, pName->n); + strncpy(pCreateMsg->pass, pPwd->z, pPwd->n); + + SCreateAcctInfo *pAcctOpt = &pInfo->pMiscInfo->acctOpt; + + pCreateMsg->maxUsers = htonl(pAcctOpt->maxUsers); + pCreateMsg->maxDbs = htonl(pAcctOpt->maxDbs); + pCreateMsg->maxTimeSeries = htonl(pAcctOpt->maxTimeSeries); + pCreateMsg->maxStreams = htonl(pAcctOpt->maxStreams); +// pCreateMsg->maxPointsPerSecond = htonl(pAcctOpt->maxPointsPerSecond); + pCreateMsg->maxStorage = htobe64(pAcctOpt->maxStorage); +// pCreateMsg->maxQueryTime = htobe64(pAcctOpt->maxQueryTime); +// pCreateMsg->maxConnections = htonl(pAcctOpt->maxConnections); + + if (pAcctOpt->stat.n == 0) { + pCreateMsg->accessState = -1; + } else { + if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { + pCreateMsg->accessState = TSDB_VN_READ_ACCCESS; + } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { + pCreateMsg->accessState = TSDB_VN_WRITE_ACCCESS; + } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { + pCreateMsg->accessState = TSDB_VN_ALL_ACCCESS; + } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { + pCreateMsg->accessState = 0; + } + } + + *outputLen = sizeof(SCreateAcctMsg); + return pMsg; +} +SDropUserMsg* buildDropUserMsg(SSqlInfo* pInfo, int32_t *msgLen, int64_t id, char* msgBuf, int32_t msgBufLen) { + SToken* pName = taosArrayGet(pInfo->pMiscInfo->a, 0); + if (pName->n >= TSDB_USER_LEN) { + return NULL; + } + + + SDropUserMsg* pMsg = calloc(1, sizeof(SDropUserMsg)); + if (pMsg == NULL) { + return NULL; + } + + strncpy(pMsg->user, pName->z, pName->n); + *msgLen = sizeof(SDropUserMsg); return pMsg; } @@ -89,7 +151,7 @@ static int32_t setTimePrecision(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreate pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default - SToken* pToken = &pCreateDbInfo->precision; + SToken* pToken = (SToken*) &pCreateDbInfo->precision; if (pToken->n > 0) { pToken->n = strdequote(pToken->z); @@ -116,8 +178,8 @@ static void doSetDbOptions(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb) { pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); pMsg->commitTime = htonl((int32_t)pCreateDb->commitTime); - pMsg->minRowsPerFileBlock = htonl(pCreateDb->minRowsPerBlock); - pMsg->maxRowsPerFileBlock = htonl(pCreateDb->maxRowsPerBlock); + pMsg->minRows = htonl(pCreateDb->minRowsPerBlock); + pMsg->maxRows = htonl(pCreateDb->maxRowsPerBlock); pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod); pMsg->compression = pCreateDb->compressionLevel; pMsg->walLevel = (char)pCreateDb->walLevel; @@ -141,7 +203,6 @@ int32_t setDbOptions(SCreateDbMsg* pCreateDbMsg, const SCreateDbInfo* pCreateDbS // todo configurable pCreateDbMsg->numOfVgroups = htonl(2); - return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 135774cd3b6921130c183efabf1dcacc12c51120..accc786bf6787ba52b72721a5da143c4a3d46e86 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4224,7 +4224,49 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, in } } - *output = buildUserManipulationMsg(pInfo, id, msgBuf, msgBufLen); + *output = buildUserManipulationMsg(pInfo, outputLen, id, msgBuf, msgBufLen); + break; + } + + case TSDB_SQL_CREATE_ACCT: + case TSDB_SQL_ALTER_ACCT: { + const char* msg1 = "invalid state option, available options[no, r, w, all]"; + const char* msg2 = "invalid user/account name"; + const char* msg3 = "name too long"; + + SToken* pName = &pInfo->pMiscInfo->user.user; + SToken* pPwd = &pInfo->pMiscInfo->user.passwd; + + if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + if (pName->n >= TSDB_USER_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + if (parserValidateNameToken(pName) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; + if (pAcctOpt->stat.n > 0) { + if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { + } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { + } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { + } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { + } else { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + } + + *output = buildAcctManipulationMsg(pInfo, outputLen, id, msgBuf, msgBufLen); + break; + } + + case TSDB_SQL_DROP_ACCT: + case TSDB_SQL_DROP_USER: { + *output = buildDropUserMsg(pInfo, outputLen, id, msgBuf, msgBufLen); break; } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index c548f1556a64daeedb9c99033245404f594297ce..502946675b40738c8ec7d3e35287669ce04dab71 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -1,10 +1,7 @@ +#include "os.h" #include "queryInfoUtil.h" -#include -#include "astGenerator.h" #include "function.h" -#include "os.h" #include "parser.h" -#include "parserInt.h" #include "parserUtil.h" static struct SSchema _s = { diff --git a/source/libs/parser/test/insertTest.cpp b/source/libs/parser/test/insertTest.cpp index 5877adf41cc303556f9ce5a311493bbffdd0cc6d..85db46d7bf1cfa29576926a0dafc7ef52310a2a8 100644 --- a/source/libs/parser/test/insertTest.cpp +++ b/source/libs/parser/test/insertTest.cpp @@ -57,17 +57,20 @@ void *__wrap_malloc(size_t c) { // [...]; class InsertTest : public Test { protected: - void setDatabase(const string& db) { + void setDatabase(const string& acctId, const string& db) { + acctId_ = acctId; db_ = db; } void bind(const char* sql) { reset(); - cxt_.sqlLen = strlen(sql); + cxt_.pAcctId = acctId_.c_str(); + cxt_.pDbname = db_.c_str(); strcpy(sqlBuf_, sql); + cxt_.sqlLen = strlen(sql); sqlBuf_[cxt_.sqlLen] = '\0'; cxt_.pSql = sqlBuf_; - cxt_.pDbname = db_.c_str(); + } int32_t run() { @@ -95,6 +98,7 @@ private: res_ = nullptr; } + string acctId_; string db_; char errMagBuf_[max_err_len]; char sqlBuf_[max_sql_len]; @@ -105,7 +109,7 @@ private: // INSERT INTO tb_name VALUES (field1_value, ...) TEST_F(InsertTest, simpleTest) { - setDatabase("test"); + setDatabase("root", "test"); bind("insert into t1 values (now, 1, \"wxy\")"); ASSERT_EQ(run(), TSDB_CODE_SUCCESS); @@ -116,7 +120,7 @@ TEST_F(InsertTest, simpleTest) { } TEST_F(InsertTest, toleranceTest) { - setDatabase("test"); + setDatabase("root", "test"); bind("insert into"); ASSERT_NE(run(), TSDB_CODE_SUCCESS); diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index f2b34971efcb1487929fa6274bc6b34001f7053c..dc341f6af7eb65a5197dedac76437668984d1d83 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -17,37 +17,68 @@ #include +#include "stub.h" +#include "addr_any.h" + namespace { void generateTestT1(MockCatalogService* mcs) { - ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 3) + ITableBuilder& builder = mcs->createTableBuilder("root.test", "t1", TSDB_NORMAL_TABLE, 3) .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); builder.done(); } void generateTestST1(MockCatalogService* mcs) { - ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 2) + ITableBuilder& builder = mcs->createTableBuilder("root.test", "st1", TSDB_SUPER_TABLE, 3, 2) .setPrecision(TSDB_TIME_PRECISION_MILLI).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) .addTag("tag1", TSDB_DATA_TYPE_INT).addTag("tag2", TSDB_DATA_TYPE_BINARY, 10) .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); builder.done(); - mcs->createSubTable("test", "st1", "st1s1", 1); - mcs->createSubTable("test", "st1", "st1s2", 2); + mcs->createSubTable("root.test", "st1", "st1s1", 1); + mcs->createSubTable("root.test", "st1", "st1s2", 2); +} + } +int32_t __catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { + return mockCatalogService->catalogGetHandle(clusterId, catalogHandle); } -void generateMetaData(MockCatalogService* mcs) { - generateTestT1(mcs); - generateTestST1(mcs); - mcs->showTables(); +int32_t __catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { + return mockCatalogService->catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, pTableMeta); +} + +void initMetaDataEnv() { + mockCatalogService.reset(new MockCatalogService()); + + static Stub stub; + stub.set(catalogGetHandle, __catalogGetHandle); + stub.set(catalogGetTableMeta, __catalogGetTableMeta); + { + AddrAny any("libcatalog.so"); + std::map result; + any.get_global_func_addr_dynsym("^catalogGetHandle$", result); + for (const auto& f : result) { + stub.set(f.second, __catalogGetHandle); + } + } + { + AddrAny any("libcatalog.so"); + std::map result; + any.get_global_func_addr_dynsym("^catalogGetTableMeta$", result); + for (const auto& f : result) { + stub.set(f.second, __catalogGetTableMeta); + } + } } -struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) { - return mockCatalogService->getCatalogHandle(pMgmtEps); +void generateMetaData() { + generateTestT1(mockCatalogService.get()); + generateTestST1(mockCatalogService.get()); + mockCatalogService->showTables(); } -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) { - return mockCatalogService->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); +void destroyMetaDataEnv() { + mockCatalogService.reset(); } diff --git a/source/libs/parser/test/mockCatalog.h b/source/libs/parser/test/mockCatalog.h index e60f7727e6804cc6fb6a8959ecb2440745671163..35eb1ddcd9ddfdd96c749db35b1271bcf6c531ff 100644 --- a/source/libs/parser/test/mockCatalog.h +++ b/source/libs/parser/test/mockCatalog.h @@ -18,10 +18,12 @@ #include "mockCatalogService.h" -void generateMetaData(MockCatalogService* mcs); +void initMetaDataEnv(); +void generateMetaData(); +void destroyMetaDataEnv(); // mock -struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); -int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData); +// int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); +// int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); #endif // MOCK_CATALOG_H diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 457f8d88bdb6d54156e1cc8be29e569978350be9..e7271c3cb2a2901f339500369030d3cb95ccf9e4 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -27,27 +27,28 @@ std::unique_ptr mockCatalogService; class TableBuilder : public ITableBuilder { public: virtual TableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) { - assert(colIndex_ < meta_->tableInfo.numOfTags + meta_->tableInfo.numOfColumns); - SSchema* col = meta_->schema + colIndex_; + assert(colId_ < schema()->tableInfo.numOfTags + schema()->tableInfo.numOfColumns); + SSchema* col = schema()->schema + colId_; col->type = type; - col->colId = colIndex_++; + col->colId = colId_++; col->bytes = bytes; strcpy(col->name, name.c_str()); return *this; } virtual TableBuilder& setVgid(int16_t vgid) { - meta_->vgId = vgid; + schema()->vgId = vgid; + meta_->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); return *this; } virtual TableBuilder& setPrecision(uint8_t precision) { - meta_->tableInfo.precision = precision; + schema()->tableInfo.precision = precision; return *this; } virtual void done() { - meta_->tableInfo.rowSize = rowsize_; + schema()->tableInfo.rowSize = rowsize_; } private: @@ -64,61 +65,62 @@ private: return std::unique_ptr(new TableBuilder(meta)); } - TableBuilder(STableMeta* meta) : colIndex_(0), rowsize_(0), meta_(meta) { + TableBuilder(STableMeta* schemaMeta) : colId_(0), rowsize_(0), meta_(new MockTableMeta()) { + meta_->schema.reset(schemaMeta); } - STableMeta* table() { + std::shared_ptr schema() { + return meta_->schema; + } + + std::shared_ptr table() { return meta_; } - int32_t colIndex_; + int32_t colId_; int32_t rowsize_; - STableMeta* meta_; + std::shared_ptr meta_; }; class MockCatalogServiceImpl { public: static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]); - MockCatalogServiceImpl() { + MockCatalogServiceImpl() : id_(1) { } - struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) const { - return (struct SCatalog*)0x01; + int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const { + return 0; } - int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const { - assert(nullptr != pMetaReq && 1 == taosArrayGetSize(pMetaReq->pTableName)); - SName* fullName = (SName*)taosArrayGet(pMetaReq->pTableName, 0); + int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { std::unique_ptr table; - int32_t code = copyTableMeta(fullName->dbname, fullName->tname, &table); + int32_t code = copyTableSchemaMeta(pDBName, pTableName, &table); if (TSDB_CODE_SUCCESS != code) { return code; } - std::unique_ptr tables((SArray*)taosArrayInit(1, sizeof(STableMeta*))); - if (!tables) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - STableMeta* elem = table.release(); - taosArrayPush(tables.get(), &elem); - pMetaData->pTableMeta = tables.release(); + *pTableMeta = table.release(); return TSDB_CODE_SUCCESS; } TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags); - meta_[db][tbname].reset(builder_->table()); - meta_[db][tbname]->uid = id_++; + meta_[db][tbname] = builder_->table(); + meta_[db][tbname]->schema->uid = id_++; return *(builder_.get()); } void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid) { std::unique_ptr table; - if (TSDB_CODE_SUCCESS != copyTableMeta(db, stbname, &table)) { - throw std::runtime_error("copyTableMeta failed"); + if (TSDB_CODE_SUCCESS != copyTableSchemaMeta(db, stbname, &table)) { + throw std::runtime_error("copyTableSchemaMeta failed"); } - meta_[db][tbname].reset(table.release()); - meta_[db][tbname]->uid = id_++; + meta_[db][tbname].reset(new MockTableMeta()); + meta_[db][tbname]->schema.reset(table.release()); + meta_[db][tbname]->schema->uid = id_++; + meta_[db][tbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + // super table + meta_[db][stbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); } void showTables() const { @@ -148,29 +150,43 @@ public: std::cout << SH("Table") << SH("Type") << SH("Precision") << IH("Vgid") << std::endl; std::cout << SL(3, 1) << std::endl; for (const auto& table : db.second) { - std::cout << SF(table.first) << SF(ttToString(table.second->tableType)) << SF(pToString(table.second->tableInfo.precision)) << IF(table.second->vgId) << std::endl; + const auto& schema = table.second->schema; + std::cout << SF(table.first) << SF(ttToString(schema->tableType)) << SF(pToString(schema->tableInfo.precision)) << IF(schema->vgId) << std::endl; } std::cout << std::endl; } for (const auto& db : meta_) { for (const auto& table : db.second) { + const auto& schema = table.second->schema; std::cout << "Table:" << table.first << std::endl; std::cout << SH("Field") << SH("Type") << SH("DataType") << IH("Bytes") << std::endl; std::cout << SL(3, 1) << std::endl; - int16_t numOfTags = table.second->tableInfo.numOfTags; - int16_t numOfFields = numOfTags + table.second->tableInfo.numOfColumns; + int16_t numOfTags = schema->tableInfo.numOfTags; + int16_t numOfFields = numOfTags + schema->tableInfo.numOfColumns; for (int16_t i = 0; i < numOfFields; ++i) { - const SSchema* schema = table.second->schema + i; - std::cout << SF(std::string(schema->name)) << SH(ftToString(i, numOfTags)) << SH(dtToString(schema->type)) << IF(schema->bytes) << std::endl; + const SSchema* col = schema->schema + i; + std::cout << SF(std::string(col->name)) << SH(ftToString(i, numOfTags)) << SH(dtToString(col->type)) << IF(col->bytes) << std::endl; } std::cout << std::endl; } } } + std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const { + DbMetaCache::const_iterator it = meta_.find(db); + if (meta_.end() == it) { + return std::shared_ptr(); + } + TableMetaCache::const_iterator tit = it->second.find(tbname); + if (it->second.end() == tit) { + return std::shared_ptr(); + } + return tit->second; + } + private: - typedef std::map > TableMetaCache; + typedef std::map> TableMetaCache; typedef std::map DbMetaCache; std::string ttToString(int8_t tableType) const { @@ -207,20 +223,13 @@ private: return (0 == colid ? "column" : (colid <= numOfTags ? "tag" : "column")); } - std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const { - DbMetaCache::const_iterator it = meta_.find(db); - if (meta_.end() == it) { - return std::shared_ptr(); - } - TableMetaCache::const_iterator tit = it->second.find(tbname); - if (it->second.end() == tit) { - return std::shared_ptr(); - } - return tit->second; + std::shared_ptr getTableSchemaMeta(const std::string& db, const std::string& tbname) const { + std::shared_ptr table = getTableMeta(db, tbname); + return table ? table->schema : std::shared_ptr(); } - int32_t copyTableMeta(const std::string& db, const std::string& tbname, std::unique_ptr* dst) const { - std::shared_ptr src = getTableMeta(db, tbname); + int32_t copyTableSchemaMeta(const std::string& db, const std::string& tbname, std::unique_ptr* dst) const { + std::shared_ptr src = getTableSchemaMeta(db, tbname); if (!src) { return TSDB_CODE_TSC_INVALID_TABLE_NAME; } @@ -244,14 +253,6 @@ MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) { MockCatalogService::~MockCatalogService() { } -struct SCatalog* MockCatalogService::getCatalogHandle(const SEpSet* pMgmtEps) const { - return impl_->getCatalogHandle(pMgmtEps); -} - -int32_t MockCatalogService::catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const { - return impl_->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); -} - ITableBuilder& MockCatalogService::createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { return impl_->createTableBuilder(db, tbname, tableType, numOfColumns, numOfTags); } @@ -262,4 +263,16 @@ void MockCatalogService::createSubTable(const std::string& db, const std::string void MockCatalogService::showTables() const { impl_->showTables(); -} \ No newline at end of file +} + +std::shared_ptr MockCatalogService::getTableMeta(const std::string& db, const std::string& tbname) const { + return impl_->getTableMeta(db, tbname); +} + +int32_t MockCatalogService::catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const { + return impl_->catalogGetHandle(clusterId, catalogHandle); +} + +int32_t MockCatalogService::catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { + return impl_->catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, pTableMeta); +} diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index 66b439b3e9f46a6cf1d1350d1777e9ee3eee6852..cd01db09cce7ff4c5949671ff44af041a47d80cb 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -18,6 +18,7 @@ #include #include +#include #include "catalog.h" @@ -41,19 +42,24 @@ public: virtual void done() = 0; }; -class MockCatalogServiceImpl; +struct MockTableMeta { + std::shared_ptr schema; + std::vector vgs; +}; +class MockCatalogServiceImpl; class MockCatalogService { public: - static const int32_t numOfDataTypes = sizeof(tDataTypes) / sizeof(tDataTypes[0]); - MockCatalogService(); ~MockCatalogService(); - struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) const; - int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SCatalogReq* pMetaReq, SMetaData* pMetaData) const; ITableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags = 0); void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid); void showTables() const; + std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const; + + // mock interface + int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const; + int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const; private: std::unique_ptr impl_; diff --git a/source/libs/parser/test/parserMain.cpp b/source/libs/parser/test/parserTestMain.cpp similarity index 88% rename from source/libs/parser/test/parserMain.cpp rename to source/libs/parser/test/parserTestMain.cpp index 5ec304a006291f280ed6f57459ba80a17e74f6ba..7de2cb66c202324d1bef2bf3cd47d00eae8feefa 100644 --- a/source/libs/parser/test/parserMain.cpp +++ b/source/libs/parser/test/parserTestMain.cpp @@ -22,12 +22,12 @@ class ParserEnv : public testing::Environment { public: virtual void SetUp() { - mockCatalogService.reset(new MockCatalogService()); - generateMetaData(mockCatalogService.get()); + initMetaDataEnv(); + generateMetaData(); } virtual void TearDown() { - mockCatalogService.reset(); + destroyMetaDataEnv(); } ParserEnv() {} diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index c9b7b6d235aadd18173c323ac1bacd55516e82ae..b1c4643f154f56be96a096d09161e1f0aa74540a 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -117,6 +117,7 @@ void destroyQueryPlan(struct SQueryPlanNode* pQueryNode); */ void* destroyQueryPhyPlan(struct SPhyNode* pQueryPhyNode); +const char* opTypeToOpName(int32_t type); int32_t opNameToOpType(const char* name); #ifdef __cplusplus diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 72b30012c8b253822501757d6818490abc01680d..1d29e48e308e1b60ee60e35ea6e6c3dcf489e643 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -33,6 +33,10 @@ static const char* gOpName[] = { #undef INCLUDE_AS_NAME }; +const char* opTypeToOpName(int32_t type) { + return gOpName[type]; +} + int32_t opNameToOpType(const char* name) { for (int32_t i = 1; i < sizeof(gOpName) / sizeof(gOpName[0]); ++i) { if (strcmp(name, gOpName[i])) { @@ -42,17 +46,43 @@ int32_t opNameToOpType(const char* name) { return OP_Unknown; } -static void toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { - SWAP(dataBlockSchema->pSchema, pPlanNode->pSchema, SSchema*); +static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { dataBlockSchema->numOfCols = pPlanNode->numOfCols; + dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfCols); + if (NULL == dataBlockSchema->pSchema) { + return false; + } + memcpy(dataBlockSchema->pSchema, pPlanNode->pSchema, sizeof(SSlotSchema) * pPlanNode->numOfCols); + return true; +} + +static bool cloneExprArray(SArray** dst, SArray* src) { + if (NULL == src) { + return true; + } + size_t size = taosArrayGetSize(src); + if (0 == size) { + return true; + } + *dst = taosArrayInit(size, POINTER_BYTES); + if (NULL == *dst) { + return false; + } + return (TSDB_CODE_SUCCESS == copyAllExprInfo(*dst, src, true) ? true : false); } static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) { SPhyNode* node = (SPhyNode*)calloc(1, size); + if (NULL == node) { + return NULL; + } node->info.type = type; - node->info.name = gOpName[type]; - SWAP(node->pTargets, pPlanNode->pExpr, SArray*); - toDataBlockSchema(pPlanNode, &(node->targetSchema)); + node->info.name = opTypeToOpName(type); + if (!cloneExprArray(&node->pTargets, pPlanNode->pExpr) || !toDataBlockSchema(pPlanNode, &(node->targetSchema))) { + free(node); + return NULL; + } + return node; } static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) { @@ -203,9 +233,6 @@ static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE); ++(pCxt->nextId.templateId); subplan->pNode = createPhyNode(pCxt, pRoot); - SArray* l0 = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); - taosArrayPush(l0, &subplan); - taosArrayPush(pCxt->pDag->pSubplans, &l0); // todo deal subquery } diff --git a/source/libs/planner/test/CMakeLists.txt b/source/libs/planner/test/CMakeLists.txt index 58743567846df2f4c80272cf7433cf3af6b9663a..7fbfcfe7efc31ecae16ffd3d4ad72db2cff58090 100644 --- a/source/libs/planner/test/CMakeLists.txt +++ b/source/libs/planner/test/CMakeLists.txt @@ -5,7 +5,12 @@ MESSAGE(STATUS "build planner unit test") SET(CMAKE_CXX_STANDARD 11) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) -ADD_EXECUTABLE(plannerTest ${SOURCE_LIST}) +ADD_EXECUTABLE(plannerTest + ${SOURCE_LIST} + "${SOURCE_LIST}/../../../parser/test/mockCatalog.cpp" + "${SOURCE_LIST}/../../../parser/test/mockCatalogService.cpp" +) + TARGET_LINK_LIBRARIES( plannerTest PUBLIC os util common planner parser catalog transport gtest function qcom @@ -15,4 +20,5 @@ TARGET_INCLUDE_DIRECTORIES( plannerTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/planner/" PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/planner/inc" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/parser/test" ) diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3be3337304534efa3d02a068585ec5f1a916ae3c --- /dev/null +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include "plannerInt.h" +#include "mockCatalogService.h" + +using namespace std; +using namespace testing; + +void* myCalloc(size_t nmemb, size_t size) { + if (void* p = calloc(nmemb, size)) { + return p; + } + throw bad_alloc(); +} + +class PhyPlanTest : public Test { +protected: + void pushScan(const string& db, const string& table, int32_t scanOp) { + shared_ptr meta = mockCatalogService->getTableMeta(db, table); + EXPECT_TRUE(meta); +// typedef struct SQueryPlanNode { +// SArray *pExpr; // the query functions or sql aggregations +// int32_t numOfExpr; // number of result columns, which is also the number of pExprs +// } SQueryPlanNode; + unique_ptr scan((SQueryPlanNode*)calloc(1, sizeof(SQueryPlanNode))); + scan->info.type = scanOp; + scan->numOfCols = meta->schema->tableInfo.numOfColumns; + scan->pSchema = (SSchema*)myCalloc(1, sizeof(SSchema) * scan->numOfCols); + memcpy(scan->pSchema, meta->schema->schema, sizeof(SSchema) * scan->numOfCols); + //todo 'pExpr' 'numOfExpr' + scan->pExtInfo = createScanExtInfo(meta); + pushNode(scan.release()); + } + + int32_t run() { + SQueryDag* dag = nullptr; + int32_t code = createDag(logicPlan_.get(), nullptr, &dag); + dag_.reset(dag); + return code; + } + + void explain() { + size_t level = taosArrayGetSize(dag_->pSubplans); + for (size_t i = 0; i < level; ++i) { + std::cout << "level " << i << ":" << std::endl; + const SArray* subplans = (const SArray*)taosArrayGetP(dag_->pSubplans, i); + size_t num = taosArrayGetSize(subplans); + for (size_t j = 0; j < num; ++j) { + std::cout << "no " << j << ":" << std::endl; + char* str = nullptr; + ASSERT_EQ (TSDB_CODE_SUCCESS, qSubPlanToString((const SSubplan*)taosArrayGetP(subplans, j), &str)); + std::cout << str << std::endl; + free(str); + } + } + } + + SQueryDag* reslut() { + return dag_.get(); + } + +private: + void pushNode(SQueryPlanNode* node) { + if (logicPlan_) { + // todo + } else { + logicPlan_.reset(node); + } + } + + void copySchemaMeta(STableMeta** dst, const STableMeta* src) { + int32_t size = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); + *dst = (STableMeta*)myCalloc(1, size); + memcpy(*dst, src, size); + } + + void copyStorageMeta(SVgroupsInfo** dst, const std::vector& src) { + *dst = (SVgroupsInfo*)myCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * src.size()); + (*dst)->numOfVgroups = src.size(); + for (int32_t i = 0; i < src.size(); ++i) { + (*dst)->vgroups[i].vgId = src[i].vgId; + (*dst)->vgroups[i].numOfEps = src[i].numOfEps; + memcpy((*dst)->vgroups[i].epAddr, src[i].epAddr, src[i].numOfEps); + } + } + + SQueryTableInfo* createScanExtInfo(shared_ptr& meta) { + SQueryTableInfo* info = (SQueryTableInfo*)myCalloc(1, sizeof(SQueryTableInfo)); + info->pMeta = (STableMetaInfo*)myCalloc(1, sizeof(STableMetaInfo)); + copySchemaMeta(&info->pMeta->pTableMeta, meta->schema.get()); + copyStorageMeta(&info->pMeta->vgroupList, meta->vgs); + return info; + } + + shared_ptr meta_; + unique_ptr logicPlan_; + unique_ptr dag_; +}; + +// select * from table +TEST_F(PhyPlanTest, tableScanTest) { + pushScan("root.test", "t1", QNODE_TABLESCAN); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + explain(); + SQueryDag* dag = reslut(); + // todo check +} + +// select * from supertable +TEST_F(PhyPlanTest, superTableScanTest) { + pushScan("root.test", "st1", QNODE_TABLESCAN); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + explain(); + SQueryDag* dag = reslut(); + // todo check +} diff --git a/source/libs/planner/test/plannerTests.cpp b/source/libs/planner/test/plannerTests.cpp index 5ede8dd15578b353953488a3104460bfd63d6cd8..11a31d15eb330981549992c1dc40272b76c28b32 100644 --- a/source/libs/planner/test/plannerTests.cpp +++ b/source/libs/planner/test/plannerTests.cpp @@ -20,6 +20,7 @@ #include "taos.h" #include "parser.h" +#include "mockCatalog.h" #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -27,9 +28,25 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); +class PlannerEnv : public testing::Environment { +public: + virtual void SetUp() { + initMetaDataEnv(); + generateMetaData(); + } + + virtual void TearDown() { + destroyMetaDataEnv(); + } + + PlannerEnv() {} + virtual ~PlannerEnv() {} +}; + +int main(int argc, char* argv[]) { + testing::AddGlobalTestEnvironment(new PlannerEnv()); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); } TEST(testCase, planner_test) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 9ca8b6789b328dc00f8d23e1c35264b70ece8413..728c47af1e71e4c8b15d1f2a921c34149f3c098f 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -294,7 +294,7 @@ void msgInit() { tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; - tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAG_VAL] = tscBuildUpdateTagMsg; tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index cdb4e08205b66abaa91edade1c82928b3e7ce32f..6baaab1ef4b5ce436306ebc98057b2c1e28f599d 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -11,3 +11,5 @@ target_link_libraries( scheduler PRIVATE os util planner qcom common catalog transport ) + +ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 0234472c706461be1c57ea277a351142abb6a558..3fab91edacd77096fbacceee5df3f762452ad16d 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -65,9 +65,8 @@ typedef struct SQueryJob { SQueryProfileSummary summary; SEpSet dataSrcEps; SEpAddr resEp; - struct SCatalog *catalog; - void *rpc; - SEpSet *mgmtEpSet; + void *transport; + SArray *qnodeList; tsem_t rspSem; int32_t userFetch; int32_t remoteFetch; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index efbb565578f208cb256960498b44fa1dcdd31d01..99a9b06fe441832d42fbc5bc86ad24bcc3ebc263 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -58,8 +58,8 @@ int32_t schBuildTaskRalation(SQueryJob *job, SHashObj *planToTask) { for (int32_t m = 0; m < level->taskNum; ++m) { SQueryTask *task = taosArrayGet(level->subTasks, m); SSubplan *plan = task->plan; - int32_t childNum = (int32_t)taosArrayGetSize(plan->pChildern); - int32_t parentNum = (int32_t)taosArrayGetSize(plan->pParents); + int32_t childNum = plan->pChildern ? (int32_t)taosArrayGetSize(plan->pChildern) : 0; + int32_t parentNum = plan->pParents ? (int32_t)taosArrayGetSize(plan->pParents) : 0; if (childNum > 0) { task->children = taosArrayInit(childNum, POINTER_BYTES); @@ -187,13 +187,19 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { for (int32_t n = 0; n < levelPlanNum; ++n) { SSubplan *plan = taosArrayGet(levelPlans, n); - SQueryTask *task = taosArrayGet(level.subTasks, n); + SQueryTask task = {0}; - task->taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); - task->plan = plan; - task->status = JOB_TASK_STATUS_NOT_START; + task.taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); + task.plan = plan; + task.status = JOB_TASK_STATUS_NOT_START; - if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &task, POINTER_BYTES)) { + void *p = taosArrayPush(level.subTasks, &task); + if (NULL == p) { + qError("taosArrayPush failed"); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &p, POINTER_BYTES)) { qError("taosHashPut failed"); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -225,20 +231,27 @@ _return: SCH_RET(code); } -int32_t schAvailableEpSet(SQueryJob *job, SEpSet *epSet) { +int32_t schSetTaskExecEpSet(SQueryJob *job, SEpSet *epSet) { if (epSet->numOfEps >= SCH_MAX_CONDIDATE_EP_NUM) { return TSDB_CODE_SUCCESS; } - if (SCH_HAS_QNODE_IN_CLUSTER(schMgmt.cfg.clusterType)) { - SCH_ERR_RET(catalogGetQnodeList(job->catalog, job->rpc, job->mgmtEpSet, epSet)); - } else { - for (int32_t i = 0; i < job->dataSrcEps.numOfEps; ++i) { - strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); - epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; - - ++epSet->numOfEps; - } + int32_t qnodeNum = taosArrayGetSize(job->qnodeList); + + for (int32_t i = 0; i < qnodeNum && epSet->numOfEps < tListLen(epSet->port); ++i) { + SEpAddr *addr = taosArrayGet(job->qnodeList, i); + + strncpy(epSet->fqdn[epSet->numOfEps], addr->fqdn, sizeof(addr->fqdn)); + epSet->port[epSet->numOfEps] = addr->port; + + ++epSet->numOfEps; + } + + for (int32_t i = 0; i < job->dataSrcEps.numOfEps && epSet->numOfEps < tListLen(epSet->port); ++i) { + strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); + epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; + + ++epSet->numOfEps; } return TSDB_CODE_SUCCESS; @@ -511,7 +524,12 @@ int32_t schLaunchTask(SQueryJob *job, SQueryTask *task) { SCH_ERR_RET(qSubPlanToString(plan, &task->msg)); if (plan->execEpSet.numOfEps <= 0) { - SCH_ERR_RET(schAvailableEpSet(job, &plan->execEpSet)); + SCH_ERR_RET(schSetTaskExecEpSet(job, &plan->execEpSet)); + } + + if (plan->execEpSet.numOfEps <= 0) { + SCH_TASK_ERR_LOG("invalid execEpSet num:%d", plan->execEpSet.numOfEps); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } SCH_ERR_RET(schAsyncSendMsg(job, task, TSDB_MSG_TYPE_QUERY)); @@ -554,20 +572,23 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { } -int32_t scheduleQueryJob(struct SCatalog *pCatalog, void *pRpc, const SEpSet* pMgmtEps, SQueryDag* pDag, void** pJob) { - if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { +int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob) { + if (NULL == transport || NULL == transport ||NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + if (taosArrayGetSize(qnodeList) <= 0) { + qInfo("qnodeList is empty"); + } + int32_t code = 0; SQueryJob *job = calloc(1, sizeof(SQueryJob)); if (NULL == job) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - job->catalog = pCatalog; - job->rpc = pRpc; - job->mgmtEpSet = (SEpSet *)pMgmtEps; + job->transport = transport; + job->qnodeList = qnodeList; SCH_ERR_JRET(schValidateAndBuildJob(pDag, job)); diff --git a/source/libs/scheduler/test/CMakeLists.txt b/source/libs/scheduler/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..00a6d08e5d7be527005a64f0e4111d9a3e35974d --- /dev/null +++ b/source/libs/scheduler/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build scheduler unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(schedulerTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + schedulerTest + PUBLIC os util common catalog transport gtest qcom taos planner scheduler +) + +TARGET_INCLUDE_DIRECTORIES( + schedulerTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scheduler/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scheduler/inc" +) diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..9e94553058e923086808cdb462d27ffa95e9a96e 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "catalog.h" +#include "scheduler.h" +#include "tep.h" +#include "trpc.h" + +namespace { +void mockBuildDag(SQueryDag *dag) { + uint64_t qId = 0x111111111111; + + dag->queryId = qId; + dag->numOfSubplans = 2; + dag->pSubplans = taosArrayInit(dag->numOfSubplans, POINTER_BYTES); + SArray *scan = taosArrayInit(1, sizeof(SSubplan)); + SArray *merge = taosArrayInit(1, sizeof(SSubplan)); + + SSubplan scanPlan = {0}; + SSubplan mergePlan = {0}; + + scanPlan.id.queryId = qId; + scanPlan.id.templateId = 0x2222222222; + scanPlan.id.subplanId = 0x3333333333; + scanPlan.type = QUERY_TYPE_SCAN; + scanPlan.level = 1; + scanPlan.execEpSet.numOfEps = 1; + scanPlan.pChildern = NULL; + scanPlan.pParents = taosArrayInit(1, POINTER_BYTES); + + mergePlan.id.queryId = qId; + mergePlan.id.templateId = 0x4444444444; + mergePlan.id.subplanId = 0x5555555555; + mergePlan.type = QUERY_TYPE_MERGE; + mergePlan.level = 0; + mergePlan.execEpSet.numOfEps = 1; + mergePlan.pChildern = taosArrayInit(1, POINTER_BYTES); + mergePlan.pParents = NULL; + + SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan); + SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan); + + taosArrayPush(mergePointer->pChildern, &scanPointer); + taosArrayPush(scanPointer->pParents, &mergePointer); + + taosArrayPush(dag->pSubplans, &merge); + taosArrayPush(dag->pSubplans, &scan); +} + +} + +TEST(testCase, normalCase) { + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; + SVgroupInfo vgInfo = {0}; + void *pJob = NULL; + SQueryDag dag = {0}; + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); + + int32_t code = schedulerInit(NULL); + ASSERT_EQ(code, 0); + + mockBuildDag(&dag); + + code = scheduleExecJob(mockPointer, qnodeList, &dag, &pJob); + ASSERT_EQ(code, 0); +} + + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + + + diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ffc59c5d6975833c25d5d53f1b6a391be0a07090..43ac7606432b6380fbac9b683ef8737e1903c8f3 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -240,6 +240,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment" TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") +// mnode-trans +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") + // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_DND_EXITING, "Dnode is exiting") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 23a923a315765778690c5ba72400148d99527792..5bac59f913462efd0118852d6f39fb047b3713e2 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -14,21 +14,21 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" #include "tlog.h" +#include "os.h" #include "tnote.h" #include "tutil.h" +#include "ulog.h" #include "zlib.h" - -#define MAX_LOGLINE_SIZE (1000) -#define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) -#define MAX_LOGLINE_CONTENT_SIZE (MAX_LOGLINE_SIZE - 100) -#define MAX_LOGLINE_DUMP_SIZE (65 * 1024) -#define MAX_LOGLINE_DUMP_BUFFER_SIZE (MAX_LOGLINE_DUMP_SIZE + 10) + +#define MAX_LOGLINE_SIZE (1000) +#define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) +#define MAX_LOGLINE_CONTENT_SIZE (MAX_LOGLINE_SIZE - 100) +#define MAX_LOGLINE_DUMP_SIZE (65 * 1024) +#define MAX_LOGLINE_DUMP_BUFFER_SIZE (MAX_LOGLINE_DUMP_SIZE + 10) #define MAX_LOGLINE_DUMP_CONTENT_SIZE (MAX_LOGLINE_DUMP_SIZE - 100) -#define LOG_FILE_NAME_LEN 300 +#define LOG_FILE_NAME_LEN 300 #define TSDB_DEFAULT_LOG_BUF_SIZE (20 * 1024 * 1024) // 20MB #define DEFAULT_LOG_INTERVAL 25 @@ -38,13 +38,13 @@ #define LOG_MAX_WAIT_MSEC 1000 #define LOG_BUF_BUFFER(x) ((x)->buffer) -#define LOG_BUF_START(x) ((x)->buffStart) -#define LOG_BUF_END(x) ((x)->buffEnd) -#define LOG_BUF_SIZE(x) ((x)->buffSize) -#define LOG_BUF_MUTEX(x) ((x)->buffMutex) +#define LOG_BUF_START(x) ((x)->buffStart) +#define LOG_BUF_END(x) ((x)->buffEnd) +#define LOG_BUF_SIZE(x) ((x)->buffSize) +#define LOG_BUF_MUTEX(x) ((x)->buffMutex) typedef struct { - char * buffer; + char *buffer; int32_t buffStart; int32_t buffEnd; int32_t buffSize; @@ -57,18 +57,18 @@ typedef struct { } SLogBuff; typedef struct { - int32_t fileNum; - int32_t maxLines; - int32_t lines; - int32_t flag; - int32_t openInProgress; - pid_t pid; - char logName[LOG_FILE_NAME_LEN]; - SLogBuff * logHandle; + int32_t fileNum; + int32_t maxLines; + int32_t lines; + int32_t flag; + int32_t openInProgress; + pid_t pid; + char logName[LOG_FILE_NAME_LEN]; + SLogBuff *logHandle; pthread_mutex_t logMutex; } SLogObj; -int8_t tscEmbeddedInUtil = 0; +int8_t tscEmbeddedInUtil = 0; int32_t tsLogKeepDays = 0; int8_t tsAsyncLog = 1; @@ -93,19 +93,19 @@ int32_t debugFlag = 0; int32_t sDebugFlag = 135; int32_t wDebugFlag = 135; int32_t tsdbDebugFlag = 131; +int32_t tqDebugFlag = 131; int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; int32_t ctgDebugFlag = 131; - int64_t dbgEmptyW = 0; int64_t dbgWN = 0; int64_t dbgSmallWN = 0; int64_t dbgBigWN = 0; int64_t dbgWSize = 0; -static SLogObj tsLogObj = { .fileNum = 1 }; -static void * taosAsyncOutputLog(void *param); +static SLogObj tsLogObj = {.fileNum = 1}; +static void *taosAsyncOutputLog(void *param); static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen); static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(int32_t oldFd); @@ -139,8 +139,8 @@ static void taosStopLog() { void taosCloseLog() { taosStopLog(); - //tsem_post(&(tsLogObj.logHandle->buffNotEmpty)); - taosMsleep(MAX_LOG_INTERVAL/1000); + // tsem_post(&(tsLogObj.logHandle->buffNotEmpty)); + taosMsleep(MAX_LOG_INTERVAL / 1000); if (taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { pthread_join(tsLogObj.logHandle->asyncThread, NULL); } @@ -217,7 +217,7 @@ static void *taosThreadToOpenNewFile(void *param) { tsLogObj.lines = 0; tsLogObj.openInProgress = 0; taosCloseLogByFd(oldFd); - + uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); taosPrintCfg(); @@ -308,9 +308,9 @@ static void taosGetLogFileName(char *fn) { static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { #ifdef WINDOWS /* - * always set maxFileNum to 1 - * means client log filename is unique in windows - */ + * always set maxFileNum to 1 + * means client log filename is unique in windows + */ maxFileNum = 1; #endif @@ -381,13 +381,14 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); + printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, + tsTotalLogDirGB); fflush(stdout); return; } va_list argpointer; - char buffer[MAX_LOGLINE_BUFFER_SIZE] = { 0 }; + char buffer[MAX_LOGLINE_BUFFER_SIZE] = {0}; int32_t len; struct tm Tm, *ptm; struct timeval timeSecs; @@ -434,20 +435,20 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } } - if (dflag & DEBUG_SCREEN) - taosWriteFile(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); if (dflag == 255) nInfo(buffer, len); } void taosDumpData(unsigned char *msg, int32_t len) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); + printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, + tsTotalLogDirGB); fflush(stdout); return; } - char temp[256]; - int32_t i, pos = 0, c = 0; + char temp[256]; + int32_t i, pos = 0, c = 0; for (i = 0; i < len; ++i) { sprintf(temp + pos, "%02x ", msg[i]); @@ -468,7 +469,8 @@ void taosDumpData(unsigned char *msg, int32_t len) { void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); + printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, + tsTotalLogDirGB); fflush(stdout); return; } @@ -503,7 +505,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . } else { taosWriteFile(tsLogObj.logHandle->fd, buffer, len); } - + if (tsLogObj.maxLines > 0) { atomic_add_fetch_32(&tsLogObj.lines, 1); @@ -542,7 +544,7 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { tLogBuff->stop = 0; if (pthread_mutex_init(&LOG_BUF_MUTEX(tLogBuff), NULL) < 0) goto _err; - //tsem_init(&(tLogBuff->buffNotEmpty), 0, 0); + // tsem_init(&(tLogBuff->buffNotEmpty), 0, 0); return tLogBuff; @@ -576,12 +578,12 @@ static void taosCopyLogBuffer(SLogBuff *tLogBuff, int32_t start, int32_t end, ch } static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) { - int32_t start = 0; - int32_t end = 0; - int32_t remainSize = 0; + int32_t start = 0; + int32_t end = 0; + int32_t remainSize = 0; static int64_t lostLine = 0; - char tmpBuf[40] = {0}; - int32_t tmpBufLen = 0; + char tmpBuf[40] = {0}; + int32_t tmpBufLen = 0; if (tLogBuff == NULL || tLogBuff->stop) return -1; @@ -592,7 +594,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) remainSize = (start > end) ? (start - end - 1) : (start + LOG_BUF_SIZE(tLogBuff) - end - 1); if (lostLine > 0) { - sprintf(tmpBuf, "...Lost %"PRId64" lines here...\n", lostLine); + sprintf(tmpBuf, "...Lost %" PRId64 " lines here...\n", lostLine); tmpBufLen = (int32_t)strlen(tmpBuf); } @@ -610,7 +612,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) taosCopyLogBuffer(tLogBuff, LOG_BUF_START(tLogBuff), LOG_BUF_END(tLogBuff), msg, msgLen); - //int32_t w = atomic_sub_fetch_32(&waitLock, 1); + // int32_t w = atomic_sub_fetch_32(&waitLock, 1); /* if (w <= 0 || ((remainSize - msgLen - tmpBufLen) < (LOG_BUF_SIZE(tLogBuff) * 4 /5))) { tsem_post(&(tLogBuff->buffNotEmpty)); @@ -622,7 +624,6 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); - return 0; } @@ -634,9 +635,9 @@ static int32_t taosGetLogRemainSize(SLogBuff *tLogBuff, int32_t start, int32_t e static void taosWriteLog(SLogBuff *tLogBuff) { static int32_t lastDuration = 0; - int32_t remainChecked = 0; - int32_t start, end, pollSize; - + int32_t remainChecked = 0; + int32_t start, end, pollSize; + do { if (remainChecked == 0) { start = LOG_BUF_START(tLogBuff); @@ -662,24 +663,24 @@ static void taosWriteLog(SLogBuff *tLogBuff) { if (start < end) { taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); } else { - int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); + int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; + taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); + taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); } dbgWN++; - dbgWSize+=pollSize; - + dbgWSize += pollSize; + if (pollSize < tLogBuff->minBuffSize) { dbgSmallWN++; if (writeInterval < MAX_LOG_INTERVAL) { writeInterval += LOG_INTERVAL_STEP; } - } else if (pollSize > LOG_BUF_SIZE(tLogBuff)/3) { + } else if (pollSize > LOG_BUF_SIZE(tLogBuff) / 3) { dbgBigWN++; writeInterval = MIN_LOG_INTERVAL; - } else if (pollSize > LOG_BUF_SIZE(tLogBuff)/4) { + } else if (pollSize > LOG_BUF_SIZE(tLogBuff) / 4) { if (writeInterval > MIN_LOG_INTERVAL) { writeInterval -= LOG_INTERVAL_STEP; } @@ -698,13 +699,13 @@ static void taosWriteLog(SLogBuff *tLogBuff) { writeInterval = MIN_LOG_INTERVAL; remainChecked = 1; - }while (1); + } while (1); } static void *taosAsyncOutputLog(void *param) { SLogBuff *tLogBuff = (SLogBuff *)param; setThreadName("log"); - + while (1) { taosMsleep(writeInterval); @@ -721,8 +722,8 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { int32_t compressSize = 163840; int32_t ret = 0; int32_t len = 0; - char * data = malloc(compressSize); - FILE * srcFp = NULL; + char *data = malloc(compressSize); + FILE *srcFp = NULL; gzFile dstFp = NULL; srcFp = fopen(srcFileName, "r"); diff --git a/source/util/src/tthread.c b/source/util/src/tthread.c index 5ed7fb5aa00d1a5c5c3fffae2de46dbea57e011f..44fce1c882c4f0d7d66f36d378db1594e59f19ea 100644 --- a/source/util/src/tthread.c +++ b/source/util/src/tthread.c @@ -13,16 +13,16 @@ * along with this program. If not, see . */ -#include "os.h" #include "tthread.h" +#include "os.h" +#include "taoserror.h" #include "tdef.h" #include "tutil.h" #include "ulog.h" -#include "taoserror.h" // create new thread -pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param) { - pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t)); +pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param) { + pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t)); pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -36,26 +36,24 @@ pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param) { return pthread; } -// destory thread +// destory thread bool taosDestoryThread(pthread_t* pthread) { - if(pthread == NULL) return false; - if(taosThreadRunning(pthread)) { + if (pthread == NULL) return false; + if (taosThreadRunning(pthread)) { pthread_cancel(*pthread); pthread_join(*pthread, NULL); } - + free(pthread); return true; } // thread running return true bool taosThreadRunning(pthread_t* pthread) { - if(pthread == NULL) return false; + if (pthread == NULL) return false; int ret = pthread_kill(*pthread, 0); - if(ret == ESRCH) - return false; - if(ret == EINVAL) - return false; + if (ret == ESRCH) return false; + if (ret == EINVAL) return false; // alive return true; } diff --git a/src/tsdb/CMakeLists.txt b/src/tsdb/CMakeLists.txt deleted file mode 100644 index c5b77df5a25f9f0b1e9294228520f171b9befddd..0000000000000000000000000000000000000000 --- a/src/tsdb/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(inc) -AUX_SOURCE_DIRECTORY(src SRC) -ADD_LIBRARY(tsdb ${SRC}) -TARGET_LINK_LIBRARIES(tsdb tfs common tutil) - -IF (TD_TSDB_PLUGINS) - TARGET_LINK_LIBRARIES(tsdb tsdbPlugins) -ENDIF () - -IF (TD_LINUX) - # Someone has no gtest directory, so comment it - # ADD_SUBDIRECTORY(tests) -ENDIF () diff --git a/src/tsdb/inc/tsdbBuffer.h b/src/tsdb/inc/tsdbBuffer.h deleted file mode 100644 index 4b650d3993a54f6a98caf00a3605feb37e972ebd..0000000000000000000000000000000000000000 --- a/src/tsdb/inc/tsdbBuffer.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_BUFFER_H_ -#define _TD_TSDB_BUFFER_H_ - -typedef struct { - int64_t blockId; - int offset; - int remain; - char data[]; -} STsdbBufBlock; - -typedef struct { - pthread_cond_t poolNotEmpty; - int bufBlockSize; - int tBufBlocks; - int nBufBlocks; - int nRecycleBlocks; - int nElasticBlocks; - int64_t index; - SList* bufBlockList; -} STsdbBufPool; - -#define TSDB_BUFFER_RESERVE 1024 // Reseve 1K as commit threshold - -STsdbBufPool* tsdbNewBufPool(); -void tsdbFreeBufPool(STsdbBufPool* pBufPool); -int tsdbOpenBufPool(STsdbRepo* pRepo); -void tsdbCloseBufPool(STsdbRepo* pRepo); -SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); -int tsdbExpandPool(STsdbRepo* pRepo, int32_t oldTotalBlocks); -void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode, bool bELastic); - -// health cite -STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize); -void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock); - -#endif /* _TD_TSDB_BUFFER_H_ */ diff --git a/src/tsdb/inc/tsdbLog.h b/src/tsdb/inc/tsdbLog.h deleted file mode 100644 index fdd04e968a2ce5345030aa4cef2bb87becd8f3f9..0000000000000000000000000000000000000000 --- a/src/tsdb/inc/tsdbLog.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_LOG_H_ -#define _TD_TSDB_LOG_H_ - -extern int32_t tsdbDebugFlag; - -#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) -#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) -#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) -#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) -#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) -#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) - -#endif /* _TD_TSDB_LOG_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbMeta.h b/src/tsdb/inc/tsdbMeta.h deleted file mode 100644 index 8ce5e7ade80b2006ac8c39fec178994073c5a26d..0000000000000000000000000000000000000000 --- a/src/tsdb/inc/tsdbMeta.h +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_META_H_ -#define _TD_TSDB_META_H_ - -#define TSDB_MAX_TABLE_SCHEMAS 16 - -typedef struct STable { - STableId tableId; - ETableType type; - tstr* name; // NOTE: there a flexible string here - uint64_t suid; - struct STable* pSuper; // super table pointer - SArray* schema; - STSchema* tagSchema; - SKVRow tagVal; - SSkipList* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index - void* eventHandler; // TODO - void* streamHandler; // TODO - TSKEY lastKey; - SMemRow lastRow; - char* sql; - void* cqhandle; - SRWLatch latch; // TODO: implementa latch functions - - SDataCol *lastCols; - int16_t maxColNum; - int16_t restoreColumnNum; - bool hasRestoreLastColumn; - int lastColSVersion; - T_REF_DECLARE() -} STable; - -typedef struct { - pthread_rwlock_t rwLock; - - int32_t nTables; - int32_t maxTables; - STable** tables; - SList* superList; - SHashObj* uidMap; - int maxRowBytes; - int maxCols; -} STsdbMeta; - -#define TSDB_INIT_NTABLES 1024 -#define TABLE_TYPE(t) (t)->type -#define TABLE_NAME(t) (t)->name -#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data -#define TABLE_UID(t) (t)->tableId.uid -#define TABLE_TID(t) (t)->tableId.tid -#define TABLE_SUID(t) (t)->suid -// #define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore) -#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch)) -#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch)) -#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch)) -#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch)) - -STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg); -void tsdbFreeMeta(STsdbMeta* pMeta); -int tsdbOpenMeta(STsdbRepo* pRepo); -int tsdbCloseMeta(STsdbRepo* pRepo); -STable* tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid); -STSchema* tsdbGetTableSchemaByVersion(STable* pTable, int16_t _version); -int tsdbWLockRepoMeta(STsdbRepo* pRepo); -int tsdbRLockRepoMeta(STsdbRepo* pRepo); -int tsdbUnlockRepoMeta(STsdbRepo* pRepo); -void tsdbRefTable(STable* pTable); -void tsdbUnRefTable(STable* pTable); -void tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct); -int tsdbRestoreTable(STsdbRepo* pRepo, void* cont, int contLen); -void tsdbOrgMeta(STsdbRepo* pRepo); -int tsdbInitColIdCacheWithSchema(STable* pTable, STSchema* pSchema); -int16_t tsdbGetLastColumnsIndexByColId(STable* pTable, int16_t colId); -int tsdbUpdateLastColSchema(STable *pTable, STSchema *pNewSchema); -STSchema* tsdbGetTableLatestSchema(STable *pTable); -void tsdbFreeLastColumns(STable* pTable); - -static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) { - if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) { - return -1; - } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) { - return 1; - } else { - return 0; - } -} - -static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t _version) { - STable* pDTable = (pTable->pSuper != NULL) ? pTable->pSuper : pTable; // for performance purpose - STSchema* pSchema = NULL; - STSchema* pTSchema = NULL; - - if (lock) TSDB_RLOCK_TABLE(pDTable); - if (_version < 0) { // get the latest version of schema - pTSchema = *(STSchema **)taosArrayGetLast(pDTable->schema); - } else { // get the schema with version - void* ptr = taosArraySearch(pDTable->schema, &_version, tsdbCompareSchemaVersion, TD_EQ); - if (ptr == NULL) { - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - goto _exit; - } - pTSchema = *(STSchema**)ptr; - } - - ASSERT(pTSchema != NULL); - - if (copy) { - if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - } else { - pSchema = pTSchema; - } - -_exit: - if (lock) TSDB_RUNLOCK_TABLE(pDTable); - return pSchema; -} - -static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) { - return tsdbGetTableSchemaImpl(pTable, false, false, -1); -} - -static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) { - if (pTable->type == TSDB_CHILD_TABLE) { // check child table first - STable *pSuper = pTable->pSuper; - if (pSuper == NULL) return NULL; - return pSuper->tagSchema; - } else if (pTable->type == TSDB_SUPER_TABLE) { - return pTable->tagSchema; - } else { - return NULL; - } -} - -static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) { - ASSERT((pTable->lastRow == NULL) || (pTable->lastKey == memRowKey(pTable->lastRow))); - return pTable->lastKey; -} - -#endif /* _TD_TSDB_META_H_ */ \ No newline at end of file diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c deleted file mode 100644 index 03110487807076bf8ac2ac7026ffdb828ea4c7c6..0000000000000000000000000000000000000000 --- a/src/tsdb/src/tsdbCommit.c +++ /dev/null @@ -1,1666 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "tsdbint.h" - -extern int32_t tsTsdbMetaCompactRatio; - -#define TSDB_MAX_SUBBLOCKS 8 -static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { - if (key < 0) { - return (int)((key + 1) / tsTickPerDay[precision] / days - 1); - } else { - return (int)((key / tsTickPerDay[precision] / days)); - } -} - -typedef struct { - SRtn rtn; // retention snapshot - SFSIter fsIter; // tsdb file iterator - int niters; // memory iterators - SCommitIter *iters; - bool isRFileSet; // read and commit FSET - SReadH readh; - SDFileSet wSet; - bool isDFileSame; - bool isLFileSame; - TSKEY minKey; - TSKEY maxKey; - SArray * aBlkIdx; // SBlockIdx array - STable * pTable; - SArray * aSupBlk; // Table super-block array - SArray * aSubBlk; // table sub-block array - SDataCols * pDataCols; -} SCommitH; - -#define TSDB_COMMIT_REPO(ch) TSDB_READ_REPO(&(ch->readh)) -#define TSDB_COMMIT_REPO_ID(ch) REPO_ID(TSDB_READ_REPO(&(ch->readh))) -#define TSDB_COMMIT_WRITE_FSET(ch) (&((ch)->wSet)) -#define TSDB_COMMIT_TABLE(ch) ((ch)->pTable) -#define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD) -#define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA) -#define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST) -#define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh)) -#define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh)) -#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock) -#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) - -static int tsdbCommitMeta(STsdbRepo *pRepo); -static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact); -static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); -static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile); -static int tsdbCommitTSData(STsdbRepo *pRepo); -static void tsdbStartCommit(STsdbRepo *pRepo); -static void tsdbEndCommit(STsdbRepo *pRepo, int eno); -static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); -static int tsdbCreateCommitIters(SCommitH *pCommith); -static void tsdbDestroyCommitIters(SCommitH *pCommith); -static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key); -static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo); -static void tsdbDestroyCommitH(SCommitH *pCommith); -static int tsdbGetFidLevel(int fid, SRtn *pRtn); -static int tsdbNextCommitFid(SCommitH *pCommith); -static int tsdbCommitToTable(SCommitH *pCommith, int tid); -static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable); -static int tsdbComparKeyBlock(const void *arg1, const void *arg2); -static int tsdbWriteBlockInfo(SCommitH *pCommih); -static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData); -static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx); -static int tsdbMoveBlock(SCommitH *pCommith, int bidx); -static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks); -static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, - bool isLastOneBlock); -static void tsdbResetCommitFile(SCommitH *pCommith); -static void tsdbResetCommitTable(SCommitH *pCommith); -static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); -static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError); -static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo); -static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows, int8_t update); - -void *tsdbCommitData(STsdbRepo *pRepo) { - if (pRepo->imem == NULL) { - return NULL; - } - tsdbStartCommit(pRepo); - - // Commit to update meta file - if (tsdbCommitMeta(pRepo) < 0) { - tsdbError("vgId:%d error occurs while committing META data since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - // Create the iterator to read from cache - if (tsdbCommitTSData(pRepo) < 0) { - tsdbError("vgId:%d error occurs while committing TS data since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS); - return NULL; - -_err: - ASSERT(terrno != TSDB_CODE_SUCCESS); - pRepo->code = terrno; - - tsdbEndCommit(pRepo, terrno); - return NULL; -} - -int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) { - SDiskID did; - SDFileSet nSet; - STsdbFS * pfs = REPO_FS(pRepo); - int level; - - ASSERT(pSet->fid >= pRtn->minFid); - - level = tsdbGetFidLevel(pSet->fid, pRtn); - - tfsAllocDisk(level, &(did.level), &(did.id)); - if (did.level == TFS_UNDECIDED_LEVEL) { - terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; - return -1; - } - - if (did.level > TSDB_FSET_LEVEL(pSet)) { - // Need to move the FSET to higher level - tsdbInitDFileSet(&nSet, did, REPO_ID(pRepo), pSet->fid, FS_TXN_VERSION(pfs)); - - if (tsdbCopyDFileSet(pSet, &nSet) < 0) { - tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno)); - return -1; - } - - if (tsdbUpdateDFileSet(pfs, &nSet) < 0) { - return -1; - } - - tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id); - } else { - // On a correct level - if (tsdbUpdateDFileSet(pfs, pSet) < 0) { - return -1; - } - } - - return 0; -} - -int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, - SBlockIdx *pIdx) { - size_t nSupBlocks; - size_t nSubBlocks; - uint32_t tlen; - SBlockInfo *pBlkInfo; - int64_t offset; - SBlock * pBlock; - - memset(pIdx, 0, sizeof(*pIdx)); - - nSupBlocks = taosArrayGetSize(pSupA); - nSubBlocks = (pSubA == NULL) ? 0 : taosArrayGetSize(pSubA); - - if (nSupBlocks <= 0) { - // No data (data all deleted) - return 0; - } - - tlen = (uint32_t)(sizeof(SBlockInfo) + sizeof(SBlock) * (nSupBlocks + nSubBlocks) + sizeof(TSCKSUM)); - if (tsdbMakeRoom(ppBuf, tlen) < 0) return -1; - pBlkInfo = *ppBuf; - - pBlkInfo->delimiter = TSDB_FILE_DELIMITER; - pBlkInfo->tid = TABLE_TID(pTable); - pBlkInfo->uid = TABLE_UID(pTable); - - memcpy((void *)(pBlkInfo->blocks), taosArrayGet(pSupA, 0), nSupBlocks * sizeof(SBlock)); - if (nSubBlocks > 0) { - memcpy((void *)(pBlkInfo->blocks + nSupBlocks), taosArrayGet(pSubA, 0), nSubBlocks * sizeof(SBlock)); - - for (int i = 0; i < nSupBlocks; i++) { - pBlock = pBlkInfo->blocks + i; - - if (pBlock->numOfSubBlocks > 1) { - pBlock->offset += (sizeof(SBlockInfo) + sizeof(SBlock) * nSupBlocks); - } - } - } - - taosCalcChecksumAppend(0, (uint8_t *)pBlkInfo, tlen); - - if (tsdbAppendDFile(pHeadf, (void *)pBlkInfo, tlen, &offset) < 0) { - return -1; - } - - tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(pBlkInfo, tlen - sizeof(TSCKSUM))); - - // Set pIdx - pBlock = taosArrayGetLast(pSupA); - - pIdx->tid = TABLE_TID(pTable); - pIdx->uid = TABLE_UID(pTable); - pIdx->hasLast = pBlock->last ? 1 : 0; - pIdx->maxKey = pBlock->keyLast; - pIdx->numOfBlocks = (uint32_t)nSupBlocks; - pIdx->len = tlen; - pIdx->offset = (uint32_t)offset; - - return 0; -} - -int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { - SBlockIdx *pBlkIdx; - size_t nidx = taosArrayGetSize(pIdxA); - int tlen = 0, size; - int64_t offset; - - if (nidx <= 0) { - // All data are deleted - pHeadf->info.offset = 0; - pHeadf->info.len = 0; - return 0; - } - - for (size_t i = 0; i < nidx; i++) { - pBlkIdx = (SBlockIdx *)taosArrayGet(pIdxA, i); - - size = tsdbEncodeSBlockIdx(NULL, pBlkIdx); - if (tsdbMakeRoom(ppBuf, tlen + size) < 0) return -1; - - void *ptr = POINTER_SHIFT(*ppBuf, tlen); - tsdbEncodeSBlockIdx(&ptr, pBlkIdx); - - tlen += size; - } - - tlen += sizeof(TSCKSUM); - if (tsdbMakeRoom(ppBuf, tlen) < 0) return -1; - taosCalcChecksumAppend(0, (uint8_t *)(*ppBuf), tlen); - - if (tsdbAppendDFile(pHeadf, *ppBuf, tlen, &offset) < tlen) { - return -1; - } - - tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(*ppBuf, tlen - sizeof(TSCKSUM))); - pHeadf->info.offset = (uint32_t)offset; - pHeadf->info.len = tlen; - - return 0; -} - - -// =================== Commit Meta Data -static int tsdbInitCommitMetaFile(STsdbRepo *pRepo, SMFile* pMf, bool open) { - STsdbFS * pfs = REPO_FS(pRepo); - SMFile * pOMFile = pfs->cstatus->pmf; - SDiskID did; - - // Create/Open a meta file or open the existing file - if (pOMFile == NULL) { - // Create a new meta file - did.level = TFS_PRIMARY_LEVEL; - did.id = TFS_PRIMARY_ID; - tsdbInitMFile(pMf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); - - if (open && tsdbCreateMFile(pMf, true) < 0) { - tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMf)); - } else { - tsdbInitMFileEx(pMf, pOMFile); - if (open && tsdbOpenMFile(pMf, O_WRONLY) < 0) { - tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - } - - return 0; -} - -static int tsdbCommitMeta(STsdbRepo *pRepo) { - STsdbFS * pfs = REPO_FS(pRepo); - SMemTable *pMem = pRepo->imem; - SMFile * pOMFile = pfs->cstatus->pmf; - SMFile mf; - SActObj * pAct = NULL; - SActCont * pCont = NULL; - SListNode *pNode = NULL; - - ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0); - - if (listNEles(pMem->actList) <= 0) { - // no meta data to commit, just keep the old meta file - tsdbUpdateMFile(pfs, pOMFile); - if (tsTsdbMetaCompactRatio > 0) { - if (tsdbInitCommitMetaFile(pRepo, &mf, false) < 0) { - return -1; - } - int ret = tsdbCompactMetaFile(pRepo, pfs, &mf); - if (ret < 0) tsdbError("compact meta file error"); - - return ret; - } - return 0; - } else { - if (tsdbInitCommitMetaFile(pRepo, &mf, true) < 0) { - return -1; - } - } - - // Loop to write - while ((pNode = tdListPopHead(pMem->actList)) != NULL) { - pAct = (SActObj *)pNode->data; - if (pAct->act == TSDB_UPDATE_META) { - pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); - if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len, false) < 0) { - tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, - tstrerror(terrno)); - tsdbCloseMFile(&mf); - (void)tsdbApplyMFileChange(&mf, pOMFile); - // TODO: need to reload metaCache - return -1; - } - } else if (pAct->act == TSDB_DROP_META) { - if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) { - tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, - tstrerror(terrno)); - tsdbCloseMFile(&mf); - tsdbApplyMFileChange(&mf, pOMFile); - // TODO: need to reload metaCache - return -1; - } - } else { - ASSERT(false); - } - } - - if (tsdbUpdateMFileHeader(&mf) < 0) { - tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno)); - tsdbApplyMFileChange(&mf, pOMFile); - // TODO: need to reload metaCache - return -1; - } - - TSDB_FILE_FSYNC(&mf); - tsdbCloseMFile(&mf); - tsdbUpdateMFile(pfs, &mf); - - if (tsTsdbMetaCompactRatio > 0 && tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) { - tsdbError("compact meta file error"); - } - - return 0; -} - -int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord) { - int tlen = 0; - tlen += taosEncodeFixedU64(buf, pRecord->uid); - tlen += taosEncodeFixedI64(buf, pRecord->offset); - tlen += taosEncodeFixedI64(buf, pRecord->size); - - return tlen; -} - -void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord) { - buf = taosDecodeFixedU64(buf, &(pRecord->uid)); - buf = taosDecodeFixedI64(buf, &(pRecord->offset)); - buf = taosDecodeFixedI64(buf, &(pRecord->size)); - - return buf; -} - -void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) { - STsdbCfg *pCfg = REPO_CFG(pRepo); - TSKEY minKey, midKey, maxKey, now; - - now = taosGetTimestamp(pCfg->precision); - minKey = now - pCfg->keep * tsTickPerDay[pCfg->precision]; - midKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision]; - maxKey = now - pCfg->keep1 * tsTickPerDay[pCfg->precision]; - - pRtn->minKey = minKey; - pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision)); - pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision)); - pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision)); - tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey, - pRtn->minFid, pRtn->midFid, pRtn->maxFid); -} - -static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact) { - char buf[64] = "\0"; - void * pBuf = buf; - SKVRecord rInfo; - int64_t offset; - - // Seek to end of meta file - offset = tsdbSeekMFile(pMFile, 0, SEEK_END); - if (offset < 0) { - return -1; - } - - rInfo.offset = offset; - rInfo.uid = uid; - rInfo.size = contLen; - - int tlen = tsdbEncodeKVRecord((void **)(&pBuf), &rInfo); - if (tsdbAppendMFile(pMFile, buf, tlen, NULL) < tlen) { - return -1; - } - - if (tsdbAppendMFile(pMFile, cont, contLen, NULL) < contLen) { - return -1; - } - - tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM))); - - SHashObj* cache = compact ? pfs->metaCacheComp : pfs->metaCache; - - pMFile->info.nRecords++; - - SKVRecord *pRecord = taosHashGet(cache, (void *)&uid, sizeof(uid)); - if (pRecord != NULL) { - pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord)); - } else { - pMFile->info.nRecords++; - } - taosHashPut(cache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); - - return 0; -} - -static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { - SKVRecord rInfo = {0}; - char buf[128] = "\0"; - - SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid)); - if (pRecord == NULL) { - tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid); - return -1; - } - - rInfo.offset = -pRecord->offset; - rInfo.uid = pRecord->uid; - rInfo.size = pRecord->size; - - void *pBuf = buf; - tsdbEncodeKVRecord(&pBuf, &rInfo); - - if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) { - return -1; - } - - pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord)); - pMFile->info.nDels++; - pMFile->info.nRecords--; - pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); - - taosHashRemove(pfs->metaCache, (void *)(&uid), sizeof(uid)); - return 0; -} - -static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { - float delPercent = (float)(pMFile->info.nDels) / (float)(pMFile->info.nRecords); - float tombPercent = (float)(pMFile->info.tombSize) / (float)(pMFile->info.size); - float compactRatio = (float)(tsTsdbMetaCompactRatio)/100; - - if (delPercent < compactRatio && tombPercent < compactRatio) { - return 0; - } - - if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) { - tsdbError("open meta file %s compact fail", pMFile->f.rname); - return -1; - } - - tsdbInfo("begin compact tsdb meta file, ratio:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 ",size:%" PRId64, - tsTsdbMetaCompactRatio, pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size); - - SMFile mf; - SDiskID did; - - // first create tmp meta file - did.level = TFS_PRIMARY_LEVEL; - did.id = TFS_PRIMARY_ID; - tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)) + 1); - - if (tsdbCreateMFile(&mf, true) < 0) { - tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - tsdbInfo("vgId:%d meta file %s is created to compact meta data", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); - - // second iterator metaCache - int code = -1; - int64_t maxBufSize = 1024; - SKVRecord *pRecord; - void *pBuf = NULL; - - pBuf = malloc((size_t)maxBufSize); - if (pBuf == NULL) { - goto _err; - } - - // init Comp - assert(pfs->metaCacheComp == NULL); - pfs->metaCacheComp = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (pfs->metaCacheComp == NULL) { - goto _err; - } - - pRecord = taosHashIterate(pfs->metaCache, NULL); - while (pRecord) { - if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { - tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), - tstrerror(terrno)); - goto _err; - } - if (pRecord->size > maxBufSize) { - maxBufSize = pRecord->size; - void* tmp = realloc(pBuf, (size_t)maxBufSize); - if (tmp == NULL) { - goto _err; - } - pBuf = tmp; - } - int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size); - if (nread < 0) { - tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), - tstrerror(terrno)); - goto _err; - } - - if (nread < pRecord->size) { - tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", - REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); - goto _err; - } - - if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, true) < 0) { - tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid, - tstrerror(terrno)); - goto _err; - } - - pRecord = taosHashIterate(pfs->metaCache, pRecord); - } - code = 0; - -_err: - if (code == 0) TSDB_FILE_FSYNC(&mf); - tsdbCloseMFile(&mf); - tsdbCloseMFile(pMFile); - - if (code == 0) { - // rename meta.tmp -> meta - tsdbInfo("vgId:%d meta file rename %s -> %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf), TSDB_FILE_FULL_NAME(pMFile)); - taosRename(mf.f.aname,pMFile->f.aname); - tstrncpy(mf.f.aname, pMFile->f.aname, TSDB_FILENAME_LEN); - tstrncpy(mf.f.rname, pMFile->f.rname, TSDB_FILENAME_LEN); - // update current meta file info - pfs->nstatus->pmf = NULL; - tsdbUpdateMFile(pfs, &mf); - - taosHashCleanup(pfs->metaCache); - pfs->metaCache = pfs->metaCacheComp; - pfs->metaCacheComp = NULL; - } else { - // remove meta.tmp file - remove(mf.f.aname); - taosHashCleanup(pfs->metaCacheComp); - pfs->metaCacheComp = NULL; - } - - tfree(pBuf); - - ASSERT(mf.info.nDels == 0); - ASSERT(mf.info.tombSize == 0); - - tsdbInfo("end compact tsdb meta file,code:%d,nRecords:%" PRId64 ",size:%" PRId64, - code,mf.info.nRecords,mf.info.size); - return code; -} - -// =================== Commit Time-Series Data -static int tsdbCommitTSData(STsdbRepo *pRepo) { - SMemTable *pMem = pRepo->imem; - SCommitH commith; - SDFileSet *pSet = NULL; - int fid; - - memset(&commith, 0, sizeof(commith)); - - if (pMem->numOfRows <= 0) { - // No memory data, just apply retention on each file on disk - if (tsdbApplyRtn(pRepo) < 0) { - return -1; - } - return 0; - } - - // Resource initialization - if (tsdbInitCommitH(&commith, pRepo) < 0) { - return -1; - } - - // Skip expired memory data and expired FSET - tsdbSeekCommitIter(&commith, commith.rtn.minKey); - while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { - if (pSet->fid < commith.rtn.minFid) { - tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); - } else { - break; - } - } - - // Loop to commit to each file - fid = tsdbNextCommitFid(&(commith)); - while (true) { - // Loop over both on disk and memory - if (pSet == NULL && fid == TSDB_IVLD_FID) break; - - if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) { - // Only has existing FSET but no memory data to commit in this - // existing FSET, only check if file in correct retention - if (tsdbApplyRtnOnFSet(pRepo, pSet, &(commith.rtn)) < 0) { - tsdbDestroyCommitH(&commith); - return -1; - } - - pSet = tsdbFSIterNext(&(commith.fsIter)); - } else { - // Has memory data to commit - SDFileSet *pCSet; - int cfid; - - if (pSet == NULL || pSet->fid > fid) { - // Commit to a new FSET with fid: fid - pCSet = NULL; - cfid = fid; - } else { - // Commit to an existing FSET - pCSet = pSet; - cfid = pSet->fid; - pSet = tsdbFSIterNext(&(commith.fsIter)); - } - - if (tsdbCommitToFile(&commith, pCSet, cfid) < 0) { - tsdbDestroyCommitH(&commith); - return -1; - } - - fid = tsdbNextCommitFid(&commith); - } - } - - tsdbDestroyCommitH(&commith); - return 0; -} - -static void tsdbStartCommit(STsdbRepo *pRepo) { - SMemTable *pMem = pRepo->imem; - - ASSERT(pMem->numOfRows > 0 || listNEles(pMem->actList) > 0); - - tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64 " meta rows: %d", - REPO_ID(pRepo), pMem->keyFirst, pMem->keyLast, pMem->numOfRows, listNEles(pMem->actList)); - - tsdbStartFSTxn(pRepo, pMem->pointsAdd, pMem->storageAdd); - - pRepo->code = TSDB_CODE_SUCCESS; -} - -static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { - if (eno != TSDB_CODE_SUCCESS) { - tsdbEndFSTxnWithError(REPO_FS(pRepo)); - } else { - tsdbEndFSTxn(pRepo); - } - - tsdbInfo("vgId:%d commit over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); - - if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno); - - SMemTable *pIMem = pRepo->imem; - (void)tsdbLockRepo(pRepo); - pRepo->imem = NULL; - (void)tsdbUnlockRepo(pRepo); - tsdbUnRefMemTable(pRepo, pIMem); - tsem_post(&(pRepo->readyToCommit)); -} - -#if 0 -static bool tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { - for (int i = 0; i < nIters; i++) { - TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter); - if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return true; - } - return false; -} -#endif - -static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); - - ASSERT(pSet == NULL || pSet->fid == fid); - - tsdbResetCommitFile(pCommith); - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &(pCommith->minKey), &(pCommith->maxKey)); - - // Set and open files - if (tsdbSetAndOpenCommitFile(pCommith, pSet, fid) < 0) { - return -1; - } - - // Loop to commit each table data - for (int tid = 1; tid < pCommith->niters; tid++) { - SCommitIter *pIter = pCommith->iters + tid; - - if (pIter->pTable == NULL) continue; - - if (tsdbCommitToTable(pCommith, tid) < 0) { - tsdbCloseCommitFile(pCommith, true); - // revert the file change - tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); - return -1; - } - } - - if (tsdbWriteBlockIdx(TSDB_COMMIT_HEAD_FILE(pCommith), pCommith->aBlkIdx, (void **)(&(TSDB_COMMIT_BUF(pCommith)))) < - 0) { - tsdbError("vgId:%d failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); - tsdbCloseCommitFile(pCommith, true); - // revert the file change - tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); - return -1; - } - - if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) { - tsdbError("vgId:%d failed to update FSET %d header since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); - tsdbCloseCommitFile(pCommith, true); - // revert the file change - tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); - return -1; - } - - // Close commit file - tsdbCloseCommitFile(pCommith, false); - - if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) { - return -1; - } - - return 0; -} - -static int tsdbCreateCommitIters(SCommitH *pCommith) { - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - SMemTable *pMem = pRepo->imem; - STsdbMeta *pMeta = pRepo->tsdbMeta; - - pCommith->niters = pMem->maxTables; - pCommith->iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter)); - if (pCommith->iters == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (tsdbRLockRepoMeta(pRepo) < 0) return -1; - - // reference all tables - for (int i = 0; i < pMem->maxTables; i++) { - if (pMeta->tables[i] != NULL) { - tsdbRefTable(pMeta->tables[i]); - pCommith->iters[i].pTable = pMeta->tables[i]; - } - } - - if (tsdbUnlockRepoMeta(pRepo) < 0) return -1; - - for (int i = 0; i < pMem->maxTables; i++) { - if ((pCommith->iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && - (TABLE_UID(pCommith->iters[i].pTable) == pMem->tData[i]->uid)) { - if ((pCommith->iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - tSkipListIterNext(pCommith->iters[i].pIter); - } - } - - return 0; -} - -static void tsdbDestroyCommitIters(SCommitH *pCommith) { - if (pCommith->iters == NULL) return; - - for (int i = 1; i < pCommith->niters; i++) { - if (pCommith->iters[i].pTable != NULL) { - tsdbUnRefTable(pCommith->iters[i].pTable); - tSkipListDestroyIter(pCommith->iters[i].pIter); - } - } - - free(pCommith->iters); - pCommith->iters = NULL; - pCommith->niters = 0; -} - -// Skip all keys until key (not included) -static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key) { - for (int i = 0; i < pCommith->niters; i++) { - SCommitIter *pIter = pCommith->iters + i; - if (pIter->pTable == NULL || pIter->pIter == NULL) continue; - - tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, key - 1, INT32_MAX, NULL, NULL, 0, true, NULL); - } -} - -static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo) { - STsdbCfg *pCfg = REPO_CFG(pRepo); - - memset(pCommith, 0, sizeof(*pCommith)); - tsdbGetRtnSnap(pRepo, &(pCommith->rtn)); - - TSDB_FSET_SET_CLOSED(TSDB_COMMIT_WRITE_FSET(pCommith)); - - // Init read handle - if (tsdbInitReadH(&(pCommith->readh), pRepo) < 0) { - return -1; - } - - // Init file iterator - tsdbFSIterInit(&(pCommith->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD); - - if (tsdbCreateCommitIters(pCommith) < 0) { - tsdbDestroyCommitH(pCommith); - return -1; - } - - pCommith->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx)); - if (pCommith->aBlkIdx == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCommitH(pCommith); - return -1; - } - - pCommith->aSupBlk = taosArrayInit(1024, sizeof(SBlock)); - if (pCommith->aSupBlk == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCommitH(pCommith); - return -1; - } - - pCommith->aSubBlk = taosArrayInit(1024, sizeof(SBlock)); - if (pCommith->aSubBlk == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCommitH(pCommith); - return -1; - } - - pCommith->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); - if (pCommith->pDataCols == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCommitH(pCommith); - return -1; - } - - return 0; -} - -static void tsdbDestroyCommitH(SCommitH *pCommith) { - pCommith->pDataCols = tdFreeDataCols(pCommith->pDataCols); - pCommith->aSubBlk = taosArrayDestroy(pCommith->aSubBlk); - pCommith->aSupBlk = taosArrayDestroy(pCommith->aSupBlk); - pCommith->aBlkIdx = taosArrayDestroy(pCommith->aBlkIdx); - tsdbDestroyCommitIters(pCommith); - tsdbDestroyReadH(&(pCommith->readh)); - tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); -} - -static int tsdbNextCommitFid(SCommitH *pCommith) { - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); - int fid = TSDB_IVLD_FID; - - for (int i = 0; i < pCommith->niters; i++) { - SCommitIter *pIter = pCommith->iters + i; - if (pIter->pTable == NULL || pIter->pIter == NULL) continue; - - TSKEY nextKey = tsdbNextIterKey(pIter->pIter); - if (nextKey == TSDB_DATA_TIMESTAMP_NULL) { - continue; - } else { - int tfid = (int)(TSDB_KEY_FID(nextKey, pCfg->daysPerFile, pCfg->precision)); - if (fid == TSDB_IVLD_FID || fid > tfid) { - fid = tfid; - } - } - } - - return fid; -} - -static int tsdbCommitToTable(SCommitH *pCommith, int tid) { - SCommitIter *pIter = pCommith->iters + tid; - TSKEY nextKey = tsdbNextIterKey(pIter->pIter); - - tsdbResetCommitTable(pCommith); - - TSDB_RLOCK_TABLE(pIter->pTable); - - // Set commit table - if (tsdbSetCommitTable(pCommith, pIter->pTable) < 0) { - TSDB_RUNLOCK_TABLE(pIter->pTable); - return -1; - } - - // No disk data and no memory data, just return - if (pCommith->readh.pBlkIdx == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) { - TSDB_RUNLOCK_TABLE(pIter->pTable); - return 0; - } - - // Must has disk data or has memory data - int nBlocks; - int bidx = 0; - SBlock *pBlock; - - if (pCommith->readh.pBlkIdx) { - if (tsdbLoadBlockInfo(&(pCommith->readh), NULL) < 0) { - TSDB_RUNLOCK_TABLE(pIter->pTable); - return -1; - } - - nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; - } else { - nBlocks = 0; - } - - if (bidx < nBlocks) { - pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - } else { - pBlock = NULL; - } - - while (true) { - if (pBlock == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) break; - - if ((nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey) || - (pBlock && (!pBlock->last) && tsdbComparKeyBlock((void *)(&nextKey), pBlock) > 0)) { - if (tsdbMoveBlock(pCommith, bidx) < 0) { - TSDB_RUNLOCK_TABLE(pIter->pTable); - return -1; - } - - bidx++; - if (bidx < nBlocks) { - pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - } else { - pBlock = NULL; - } - } else if (pBlock && (pBlock->last || tsdbComparKeyBlock((void *)(&nextKey), pBlock) == 0)) { - // merge pBlock data and memory data - if (tsdbMergeMemData(pCommith, pIter, bidx) < 0) { - TSDB_RUNLOCK_TABLE(pIter->pTable); - return -1; - } - - bidx++; - if (bidx < nBlocks) { - pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - } else { - pBlock = NULL; - } - nextKey = tsdbNextIterKey(pIter->pIter); - } else { - // Only commit memory data - if (pBlock == NULL) { - if (tsdbCommitMemData(pCommith, pIter, pCommith->maxKey, false) < 0) { - TSDB_RUNLOCK_TABLE(pIter->pTable); - return -1; - } - } else { - if (tsdbCommitMemData(pCommith, pIter, pBlock->keyFirst - 1, true) < 0) { - TSDB_RUNLOCK_TABLE(pIter->pTable); - return -1; - } - } - nextKey = tsdbNextIterKey(pIter->pIter); - } - } - - TSDB_RUNLOCK_TABLE(pIter->pTable); - - if (tsdbWriteBlockInfo(pCommith) < 0) { - tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), - TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); - return -1; - } - - return 0; -} - -static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) { - STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - - pCommith->pTable = pTable; - - if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (pCommith->isRFileSet) { - if (tsdbSetReadTable(&(pCommith->readh), pTable) < 0) { - return -1; - } - } else { - pCommith->readh.pBlkIdx = NULL; - } - return 0; -} - -static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { - TSKEY key = *(TSKEY *)arg1; - SBlock *pBlock = (SBlock *)arg2; - - if (key < pBlock->keyFirst) { - return -1; - } else if (key > pBlock->keyLast) { - return 1; - } else { - return 0; - } -} - -int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, - bool isLast, bool isSuper, void **ppBuf, void **ppCBuf) { - STsdbCfg * pCfg = REPO_CFG(pRepo); - SBlockData *pBlockData; - int64_t offset = 0; - int rowsToWrite = pDataCols->numOfRows; - - ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock); - ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock); - - // Make buffer space - if (tsdbMakeRoom(ppBuf, TSDB_BLOCK_STATIS_SIZE(pDataCols->numOfCols)) < 0) { - return -1; - } - pBlockData = (SBlockData *)(*ppBuf); - - // Get # of cols not all NULL(not including key column) - int nColsNotAllNull = 0; - for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column - SDataCol * pDataCol = pDataCols->cols + ncol; - SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; - - if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it - continue; - } - - memset(pBlockCol, 0, sizeof(*pBlockCol)); - - pBlockCol->colId = pDataCol->colId; - pBlockCol->type = pDataCol->type; - if (tDataTypes[pDataCol->type].statisFunc) { - (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), - &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), - &(pBlockCol->numOfNull)); - } - nColsNotAllNull++; - } - - ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols); - - // Compress the data if neccessary - int tcol = 0; // counter of not all NULL and written columns - uint32_t toffset = 0; - int32_t tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull); - int32_t lsize = tsize; - int32_t keyLen = 0; - for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { - // All not NULL columns finish - if (ncol != 0 && tcol >= nColsNotAllNull) break; - - SDataCol * pDataCol = pDataCols->cols + ncol; - SBlockCol *pBlockCol = pBlockData->cols + tcol; - - if (ncol != 0 && (pDataCol->colId != pBlockCol->colId)) continue; - - int32_t flen; // final length - int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite); - void * tptr; - - // Make room - if (tsdbMakeRoom(ppBuf, lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { - return -1; - } - pBlockData = (SBlockData *)(*ppBuf); - pBlockCol = pBlockData->cols + tcol; - tptr = POINTER_SHIFT(pBlockData, lsize); - - if (pCfg->compression == TWO_STAGE_COMP && - tsdbMakeRoom(ppCBuf, tlen + COMP_OVERFLOW_BYTES) < 0) { - return -1; - } - - // Compress or just copy - if (pCfg->compression) { - flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, - tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, - tlen + COMP_OVERFLOW_BYTES); - } else { - flen = tlen; - memcpy(tptr, pDataCol->pData, flen); - } - - // Add checksum - ASSERT(flen > 0); - flen += sizeof(TSCKSUM); - taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); - tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM))); - - if (ncol != 0) { - tsdbSetBlockColOffset(pBlockCol, toffset); - pBlockCol->len = flen; - tcol++; - } else { - keyLen = flen; - } - - toffset += flen; - lsize += flen; - } - - pBlockData->delimiter = TSDB_FILE_DELIMITER; - pBlockData->uid = TABLE_UID(pTable); - pBlockData->numOfCols = nColsNotAllNull; - - taosCalcChecksumAppend(0, (uint8_t *)pBlockData, tsize); - tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(pBlockData, tsize - sizeof(TSCKSUM))); - - // Write the whole block to file - if (tsdbAppendDFile(pDFile, (void *)pBlockData, lsize, &offset) < lsize) { - return -1; - } - - // Update pBlock membership vairables - pBlock->last = isLast; - pBlock->offset = offset; - pBlock->algorithm = pCfg->compression; - pBlock->numOfRows = rowsToWrite; - pBlock->len = lsize; - pBlock->keyLen = keyLen; - pBlock->numOfSubBlocks = isSuper ? 1 : 0; - pBlock->numOfCols = nColsNotAllNull; - pBlock->keyFirst = dataColsKeyFirst(pDataCols); - pBlock->keyLast = dataColsKeyLast(pDataCols); - - tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 - " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, - REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, - pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); - - return 0; -} - -static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, - bool isSuper) { - return tsdbWriteBlockImpl(TSDB_COMMIT_REPO(pCommith), TSDB_COMMIT_TABLE(pCommith), pDFile, pDataCols, pBlock, isLast, - isSuper, (void **)(&(TSDB_COMMIT_BUF(pCommith))), - (void **)(&(TSDB_COMMIT_COMP_BUF(pCommith)))); -} - - -static int tsdbWriteBlockInfo(SCommitH *pCommih) { - SDFile * pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); - SBlockIdx blkIdx; - STable * pTable = TSDB_COMMIT_TABLE(pCommih); - - if (tsdbWriteBlockInfoImpl(pHeadf, pTable, pCommih->aSupBlk, pCommih->aSubBlk, (void **)(&(TSDB_COMMIT_BUF(pCommih))), - &blkIdx) < 0) { - return -1; - } - - if (blkIdx.numOfBlocks == 0) { - return 0; - } - - if (taosArrayPush(pCommih->aBlkIdx, (void *)(&blkIdx)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData) { - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); - SMergeInfo mInfo; - int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); - SDFile * pDFile; - bool isLast; - SBlock block; - - while (true) { - tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, defaultRows, pCommith->pDataCols, NULL, 0, - pCfg->update, &mInfo); - - if (pCommith->pDataCols->numOfRows <= 0) break; - - if (toData || pCommith->pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - isLast = false; - } else { - pDFile = TSDB_COMMIT_LAST_FILE(pCommith); - isLast = true; - } - - if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; - - if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) { - return -1; - } - } - - return 0; -} - -static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); - int nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; - SBlock * pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - TSKEY keyLimit; - int16_t colId = 0; - SMergeInfo mInfo; - SBlock subBlocks[TSDB_MAX_SUBBLOCKS]; - SBlock block, supBlock; - SDFile * pDFile; - - if (bidx == nBlocks - 1) { - keyLimit = pCommith->maxKey; - } else { - keyLimit = pBlock[1].keyFirst - 1; - } - - SSkipListIterator titer = *(pIter->pIter); - if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1) < 0) return -1; - - tsdbLoadDataFromCache(pIter->pTable, &titer, keyLimit, INT32_MAX, NULL, pCommith->readh.pDCols[0]->cols[0].pData, - pCommith->readh.pDCols[0]->numOfRows, pCfg->update, &mInfo); - - if (mInfo.nOperations == 0) { - // no new data to insert (all updates denied) - if (tsdbMoveBlock(pCommith, bidx) < 0) { - return -1; - } - *(pIter->pIter) = titer; - } else if (pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed == 0) { - // Ignore the block - ASSERT(0); - *(pIter->pIter) = titer; - } else if (tsdbCanAddSubBlock(pCommith, pBlock, &mInfo)) { - // Add a sub-block - tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, INT32_MAX, pCommith->pDataCols, - pCommith->readh.pDCols[0]->cols[0].pData, pCommith->readh.pDCols[0]->numOfRows, pCfg->update, - &mInfo); - if (pBlock->last) { - pDFile = TSDB_COMMIT_LAST_FILE(pCommith); - } else { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - } - - if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, pBlock->last, false) < 0) return -1; - - if (pBlock->numOfSubBlocks == 1) { - subBlocks[0] = *pBlock; - subBlocks[0].numOfSubBlocks = 0; - } else { - memcpy(subBlocks, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), - sizeof(SBlock) * pBlock->numOfSubBlocks); - } - subBlocks[pBlock->numOfSubBlocks] = block; - supBlock = *pBlock; - supBlock.keyFirst = mInfo.keyFirst; - supBlock.keyLast = mInfo.keyLast; - supBlock.numOfSubBlocks++; - supBlock.numOfRows = pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed; - supBlock.offset = taosArrayGetSize(pCommith->aSubBlk) * sizeof(SBlock); - - if (tsdbCommitAddBlock(pCommith, &supBlock, subBlocks, supBlock.numOfSubBlocks) < 0) return -1; - } else { - if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; - if (tsdbMergeBlockData(pCommith, pIter, pCommith->readh.pDCols[0], keyLimit, bidx == (nBlocks - 1)) < 0) return -1; - } - - return 0; -} - -static int tsdbMoveBlock(SCommitH *pCommith, int bidx) { - SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - SDFile *pDFile; - SBlock block; - bool isSameFile; - - ASSERT(pBlock->numOfSubBlocks > 0); - - if (pBlock->last) { - pDFile = TSDB_COMMIT_LAST_FILE(pCommith); - isSameFile = pCommith->isLFileSame; - } else { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - isSameFile = pCommith->isDFileSame; - } - - if (isSameFile) { - if (pBlock->numOfSubBlocks == 1) { - if (tsdbCommitAddBlock(pCommith, pBlock, NULL, 0) < 0) { - return -1; - } - } else { - block = *pBlock; - block.offset = sizeof(SBlock) * taosArrayGetSize(pCommith->aSubBlk); - - if (tsdbCommitAddBlock(pCommith, &block, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), - pBlock->numOfSubBlocks) < 0) { - return -1; - } - } - } else { - if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; - if (tsdbWriteBlock(pCommith, pDFile, pCommith->readh.pDCols[0], &block, pBlock->last, true) < 0) return -1; - if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; - } - - return 0; -} - -static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks) { - if (taosArrayPush(pCommith->aSupBlk, pSupBlock) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (pSubBlocks && taosArrayAddBatch(pCommith->aSubBlk, pSubBlocks, nSubBlocks) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, bool isLastOneBlock) { - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); - SBlock block; - SDFile * pDFile; - bool isLast; - int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); - - int biter = 0; - while (true) { - tsdbLoadAndMergeFromCache(pCommith->readh.pDCols[0], &biter, pIter, pCommith->pDataCols, keyLimit, defaultRows, - pCfg->update); - - if (pCommith->pDataCols->numOfRows == 0) break; - - if (isLastOneBlock) { - if (pCommith->pDataCols->numOfRows < pCfg->minRowsPerFileBlock) { - pDFile = TSDB_COMMIT_LAST_FILE(pCommith); - isLast = true; - } else { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - isLast = false; - } - } else { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - isLast = false; - } - - if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; - if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; - } - - return 0; -} - -static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows, int8_t update) { - TSKEY key1 = INT64_MAX; - TSKEY key2 = INT64_MAX; - STSchema *pSchema = NULL; - - ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); - tdResetDataCols(pTarget); - - while (true) { - key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); - SMemRow row = tsdbNextIterRow(pCommitIter->pIter); - if (row == NULL || memRowKey(row) > maxKey) { - key2 = INT64_MAX; - } else { - key2 = memRowKey(row); - } - - if (key1 == INT64_MAX && key2 == INT64_MAX) break; - - if (key1 < key2) { - for (int i = 0; i < pDataCols->numOfCols; i++) { - //TODO: dataColAppendVal may fail - dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, - pTarget->maxPoints); - } - - pTarget->numOfRows++; - (*iter)++; - } else if (key1 > key2) { - if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row)); - ASSERT(pSchema != NULL); - } - - tdAppendMemRowToDataCol(row, pSchema, pTarget, true); - - tSkipListIterNext(pCommitIter->pIter); - } else { - if (update != TD_ROW_OVERWRITE_UPDATE) { - //copy disk data - for (int i = 0; i < pDataCols->numOfCols; i++) { - //TODO: dataColAppendVal may fail - dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, - pTarget->maxPoints); - } - - if(update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++; - } - if (update != TD_ROW_DISCARD_UPDATE) { - //copy mem data - if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row)); - ASSERT(pSchema != NULL); - } - - tdAppendMemRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE); - } - (*iter)++; - tSkipListIterNext(pCommitIter->pIter); - } - - if (pTarget->numOfRows >= maxRows) break; - } -} - -static void tsdbResetCommitFile(SCommitH *pCommith) { - pCommith->isRFileSet = false; - pCommith->isDFileSame = false; - pCommith->isLFileSame = false; - taosArrayClear(pCommith->aBlkIdx); -} - -static void tsdbResetCommitTable(SCommitH *pCommith) { - taosArrayClear(pCommith->aSubBlk); - taosArrayClear(pCommith->aSupBlk); - pCommith->pTable = NULL; -} - -static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { - SDiskID did; - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith); - - tfsAllocDisk(tsdbGetFidLevel(fid, &(pCommith->rtn)), &(did.level), &(did.id)); - if (did.level == TFS_UNDECIDED_LEVEL) { - terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; - return -1; - } - - // Open read FSET - if (pSet) { - if (tsdbSetAndOpenReadFSet(&(pCommith->readh), pSet) < 0) { - return -1; - } - - pCommith->isRFileSet = true; - - if (tsdbLoadBlockIdx(&(pCommith->readh)) < 0) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - - tsdbDebug("vgId:%d FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet), - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); - } else { - pCommith->isRFileSet = false; - } - - // Set and open commit FSET - if (pSet == NULL || did.level > TSDB_FSET_LEVEL(pSet)) { - // Create a new FSET to write data - tsdbInitDFileSet(pWSet, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo))); - - if (tsdbCreateDFileSet(pWSet, true) < 0) { - tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), - TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno)); - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - } - return -1; - } - - pCommith->isDFileSame = false; - pCommith->isLFileSame = false; - - tsdbDebug("vgId:%d FSET %d at level %d disk id %d is created to commit", REPO_ID(pRepo), TSDB_FSET_FID(pWSet), - TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet)); - } else { - did.level = TSDB_FSET_LEVEL(pSet); - did.id = TSDB_FSET_ID(pSet); - - pCommith->wSet.fid = fid; - pCommith->wSet.state = 0; - - // TSDB_FILE_HEAD - SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); - tsdbInitDFile(pWHeadf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); - if (tsdbCreateDFile(pWHeadf, true) < 0) { - tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), - tstrerror(terrno)); - - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - } - - // TSDB_FILE_DATA - SDFile *pRDataf = TSDB_READ_DATA_FILE(&(pCommith->readh)); - SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith); - tsdbInitDFileEx(pWDataf, pRDataf); - if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { - tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), - tstrerror(terrno)); - - tsdbCloseDFileSet(pWSet); - tsdbRemoveDFile(pWHeadf); - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - } - pCommith->isDFileSame = true; - - // TSDB_FILE_LAST - SDFile *pRLastf = TSDB_READ_LAST_FILE(&(pCommith->readh)); - SDFile *pWLastf = TSDB_COMMIT_LAST_FILE(pCommith); - if (pRLastf->info.size < 32 * 1024) { - tsdbInitDFileEx(pWLastf, pRLastf); - pCommith->isLFileSame = true; - - if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { - tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), - tstrerror(terrno)); - - tsdbCloseDFileSet(pWSet); - tsdbRemoveDFile(pWHeadf); - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - } - } else { - tsdbInitDFile(pWLastf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST); - pCommith->isLFileSame = false; - - if (tsdbCreateDFile(pWLastf, true) < 0) { - tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), - tstrerror(terrno)); - - tsdbCloseDFileSet(pWSet); - (void)tsdbRemoveDFile(pWHeadf); - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - } - } - } - - return 0; -} - -static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError) { - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - } - - if (!hasError) { - TSDB_FSET_FSYNC(TSDB_COMMIT_WRITE_FSET(pCommith)); - } - tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); -} - -static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo) { - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); - int mergeRows = pBlock->numOfRows + pInfo->rowsInserted - pInfo->rowsDeleteSucceed; - - ASSERT(mergeRows > 0); - - if (pBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pInfo->nOperations <= pCfg->maxRowsPerFileBlock) { - if (pBlock->last) { - if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true; - } else { - if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true; - } - } - - return false; -} - -int tsdbApplyRtn(STsdbRepo *pRepo) { - SRtn rtn; - SFSIter fsiter; - STsdbFS * pfs = REPO_FS(pRepo); - SDFileSet *pSet; - - // Get retention snapshot - tsdbGetRtnSnap(pRepo, &rtn); - - tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); - while ((pSet = tsdbFSIterNext(&fsiter))) { - if (pSet->fid < rtn.minFid) { - tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); - continue; - } - - if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) { - return -1; - } - } - - return 0; -} diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c deleted file mode 100644 index c2021963e0d0c8be4ed42588549153dcd20be63c..0000000000000000000000000000000000000000 --- a/src/tsdb/src/tsdbMain.c +++ /dev/null @@ -1,1008 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// no test file errors here -#include "taosdef.h" -#include "tsdbint.h" -#include "ttimer.h" -#include "tthread.h" - -#define IS_VALID_PRECISION(precision) \ - (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO)) -#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP -#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) - -static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); -static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH); -static void tsdbFreeRepo(STsdbRepo *pRepo); -static void tsdbStartStream(STsdbRepo *pRepo); -static void tsdbStopStream(STsdbRepo *pRepo); -static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh); -static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, SBlockIdx *pIdx); - -// Function declaration -int32_t tsdbCreateRepo(int repoid) { - char tsdbDir[TSDB_FILENAME_LEN] = "\0"; - char dataDir[TSDB_FILENAME_LEN] = "\0"; - - tsdbGetRootDir(repoid, tsdbDir); - if (tfsMkdir(tsdbDir) < 0) { - goto _err; - } - - tsdbGetDataDir(repoid, dataDir); - if (tfsMkdir(dataDir) < 0) { - goto _err; - } - - // TODO: need to create current file with nothing in - - return 0; - -_err: - tsdbError("vgId:%d failed to create TSDB repository since %s", repoid, tstrerror(terrno)); - return -1; -} - -int32_t tsdbDropRepo(int repoid) { - char tsdbDir[TSDB_FILENAME_LEN] = "\0"; - - tsdbGetRootDir(repoid, tsdbDir); - return tfsRmdir(tsdbDir); -} - -STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { - STsdbRepo *pRepo; - STsdbCfg config = *pCfg; - - terrno = TSDB_CODE_SUCCESS; - - // Check and set default configurations - if (tsdbCheckAndSetDefaultCfg(&config) < 0) { - tsdbError("vgId:%d failed to open TSDB repository since %s", config.tsdbId, tstrerror(terrno)); - return NULL; - } - - // Create new TSDB object - if ((pRepo = tsdbNewRepo(&config, pAppH)) == NULL) { - tsdbError("vgId:%d failed to open TSDB repository while creating TSDB object since %s", config.tsdbId, - tstrerror(terrno)); - return NULL; - } - - // Open meta - if (tsdbOpenMeta(pRepo) < 0) { - tsdbError("vgId:%d failed to open TSDB repository while opening Meta since %s", config.tsdbId, tstrerror(terrno)); - tsdbCloseRepo(pRepo, false); - return NULL; - } - - if (tsdbOpenBufPool(pRepo) < 0) { - tsdbError("vgId:%d failed to open TSDB repository while opening buffer pool since %s", config.tsdbId, - tstrerror(terrno)); - tsdbCloseRepo(pRepo, false); - return NULL; - } - - if (tsdbOpenFS(pRepo) < 0) { - tsdbError("vgId:%d failed to open TSDB repository while opening FS since %s", config.tsdbId, tstrerror(terrno)); - tsdbCloseRepo(pRepo, false); - return NULL; - } - - // TODO: Restore information from data - if ((!(pRepo->state & TSDB_STATE_BAD_DATA)) && tsdbRestoreInfo(pRepo) < 0) { - tsdbError("vgId:%d failed to open TSDB repository while restore info since %s", config.tsdbId, tstrerror(terrno)); - tsdbCloseRepo(pRepo, false); - return NULL; - } - - pRepo->mergeBuf = NULL; - - tsdbStartStream(pRepo); - - tsdbDebug("vgId:%d, TSDB repository opened", REPO_ID(pRepo)); - - return pRepo; -} - -// Note: all working thread and query thread must stopped when calling this function -int tsdbCloseRepo(STsdbRepo *repo, int toCommit) { - if (repo == NULL) return 0; - - STsdbRepo *pRepo = repo; - int vgId = REPO_ID(pRepo); - - terrno = TSDB_CODE_SUCCESS; - - tsdbStopStream(pRepo); - if(pRepo->pthread){ - taosDestoryThread(pRepo->pthread); - pRepo->pthread = NULL; - } - - if (toCommit) { - tsdbSyncCommit(repo); - } - - tsem_wait(&(pRepo->readyToCommit)); - - tsdbUnRefMemTable(pRepo, pRepo->mem); - tsdbUnRefMemTable(pRepo, pRepo->imem); - pRepo->mem = NULL; - pRepo->imem = NULL; - - tsdbCloseFS(pRepo); - tsdbCloseBufPool(pRepo); - tsdbCloseMeta(pRepo); - tsdbFreeRepo(pRepo); - tsdbDebug("vgId:%d repository is closed", vgId); - - if (terrno != TSDB_CODE_SUCCESS) { - return -1; - } else { - return 0; - } -} - -STsdbCfg *tsdbGetCfg(const STsdbRepo *repo) { - ASSERT(repo != NULL); - return &((STsdbRepo *)repo)->config; -} - -int tsdbLockRepo(STsdbRepo *pRepo) { - int code = pthread_mutex_lock(&pRepo->mutex); - if (code != 0) { - tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - pRepo->repoLocked = true; - return 0; -} - -int tsdbUnlockRepo(STsdbRepo *pRepo) { - ASSERT(IS_REPO_LOCKED(pRepo)); - pRepo->repoLocked = false; - int code = pthread_mutex_unlock(&pRepo->mutex); - if (code != 0) { - tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -int tsdbCheckCommit(STsdbRepo *pRepo) { - ASSERT(pRepo->mem != NULL); - STsdbCfg *pCfg = &(pRepo->config); - - STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); - ASSERT(pBufBlock != NULL); - if ((pRepo->mem->extraBuffList != NULL) || - ((listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 3) && (pBufBlock->remain < TSDB_BUFFER_RESERVE))) { - // trigger commit - if (tsdbAsyncCommit(pRepo) < 0) return -1; - } - - return 0; -} - -STsdbMeta *tsdbGetMeta(STsdbRepo *pRepo) { return pRepo->tsdbMeta; } - -STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo) { return NULL; } - -int tsdbGetState(STsdbRepo *repo) { return repo->state; } - -int8_t tsdbGetCompactState(STsdbRepo *repo) { return (int8_t)(repo->compactState); } - -void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) { - ASSERT(repo != NULL); - STsdbRepo *pRepo = repo; - *totalPoints = pRepo->stat.pointsWritten; - *totalStorage = pRepo->stat.totalStorage; - *compStorage = pRepo->stat.compStorage; -} - -int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { - // TODO: think about multithread cases - if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; - - STsdbCfg * pRCfg = &repo->config; - - ASSERT(pRCfg->tsdbId == pCfg->tsdbId); - ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); - ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); - ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); - ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); - ASSERT(pRCfg->precision == pCfg->precision); - - bool configChanged = false; - if (pRCfg->compression != pCfg->compression) { - configChanged = true; - } - if (pRCfg->keep != pCfg->keep) { - configChanged = true; - } - if (pRCfg->keep1 != pCfg->keep1) { - configChanged = true; - } - if (pRCfg->keep2 != pCfg->keep2) { - configChanged = true; - } - if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { - configChanged = true; - } - if (pRCfg->totalBlocks != pCfg->totalBlocks) { - configChanged = true; - } - - if (!configChanged) { - tsdbError("vgId:%d no config changed", REPO_ID(repo)); - } - - int code = pthread_mutex_lock(&repo->save_mutex); - if (code != 0) { - tsdbError("vgId:%d failed to lock tsdb save config mutex since %s", REPO_ID(repo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - - STsdbCfg * pSaveCfg = &repo->save_config; - *pSaveCfg = repo->config; - - pSaveCfg->compression = pCfg->compression; - pSaveCfg->keep = pCfg->keep; - pSaveCfg->keep1 = pCfg->keep1; - pSaveCfg->keep2 = pCfg->keep2; - pSaveCfg->cacheLastRow = pCfg->cacheLastRow; - pSaveCfg->totalBlocks = pCfg->totalBlocks; - - tsdbInfo("vgId:%d old config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d),totalBlocks(%d)", - REPO_ID(repo), - pRCfg->compression, pRCfg->keep, pRCfg->keep1,pRCfg->keep2, - pRCfg->cacheLastRow, pRCfg->totalBlocks); - tsdbInfo("vgId:%d new config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d),totalBlocks(%d)", - REPO_ID(repo), - pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, - pSaveCfg->cacheLastRow,pSaveCfg->totalBlocks); - - repo->config_changed = true; - - pthread_mutex_unlock(&repo->save_mutex); - - // schedule a commit msg and wait for the new config applied - tsdbSyncCommitConfig(repo); - - return 0; -#if 0 - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbCfg config = pRepo->config; - STsdbCfg * pRCfg = &pRepo->config; - - if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; - - ASSERT(pRCfg->tsdbId == pCfg->tsdbId); - ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); - ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); - ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); - ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); - ASSERT(pRCfg->precision == pCfg->precision); - - bool configChanged = false; - if (pRCfg->compression != pCfg->compression) { - tsdbAlterCompression(pRepo, pCfg->compression); - config.compression = pCfg->compression; - configChanged = true; - } - if (pRCfg->keep != pCfg->keep) { - if (tsdbAlterKeep(pRepo, pCfg->keep) < 0) { - tsdbError("vgId:%d failed to configure repo when alter keep since %s", REPO_ID(pRepo), tstrerror(terrno)); - config.keep = pCfg->keep; - return -1; - } - configChanged = true; - } - if (pRCfg->totalBlocks != pCfg->totalBlocks) { - tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks); - config.totalBlocks = pCfg->totalBlocks; - configChanged = true; - } - if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { - config.cacheLastRow = pCfg->cacheLastRow; - configChanged = true; - } - - if (configChanged) { - if (tsdbSaveConfig(pRepo->rootDir, &config) < 0) { - tsdbError("vgId:%d failed to configure repository while save config since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - } - - return 0; -#endif -} - -uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) { - // TODO - return 0; -#if 0 - STsdbRepo *pRepo = (STsdbRepo *)repo; - // STsdbMeta *pMeta = pRepo->tsdbMeta; - STsdbFileH *pFileH = pRepo->tsdbFileH; - uint32_t magic = 0; - char * fname = NULL; - - struct stat fState; - - tsdbDebug("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex); - ASSERT(*index <= eindex); - - if (name[0] == 0) { // get the file from index or after, but not larger than eindex - int fid = (*index) / TSDB_FILE_TYPE_MAX; - - if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { - if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { - fname = tsdbGetMetaFileName(pRepo->rootDir); - *index = TSDB_META_FILE_INDEX; - magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); - sprintf(name, "tsdb/%s", TSDB_META_FILE_NAME); - } else { - return 0; - } - } else { - SFileGroup *pFGroup = - taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); - if (pFGroup->fileId == fid) { - SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX]; - fname = strdup(TSDB_FILE_NAME(pFile)); - magic = pFile->info.magic; - char *tfname = strdup(fname); - sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); - tfree(tfname); - } else { - if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) { - SFile *pFile = &pFGroup->files[0]; - fname = strdup(TSDB_FILE_NAME(pFile)); - *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; - magic = pFile->info.magic; - char *tfname = strdup(fname); - sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); - tfree(tfname); - } else { - return 0; - } - } - } - } else { // get the named file at the specified index. If not there, return 0 - fname = malloc(256); - sprintf(fname, "%s/vnode/vnode%d/%s", TFS_PRIMARY_PATH(), REPO_ID(pRepo), name); - if (access(fname, F_OK) != 0) { - tfree(fname); - return 0; - } - if (*index == TSDB_META_FILE_INDEX) { // get meta file - tsdbGetStoreInfo(fname, &magic, size); - } else { - char tfname[TSDB_FILENAME_LEN] = "\0"; - sprintf(tfname, "vnode/vnode%d/tsdb/%s/%s", REPO_ID(pRepo), TSDB_DATA_DIR_NAME, basename(name)); - tsdbGetFileInfoImpl(tfname, &magic, size); - } - tfree(fname); - return magic; - } - - if (stat(fname, &fState) < 0) { - tfree(fname); - return 0; - } - - *size = fState.st_size; - // magic = *size; - - tfree(fname); - return magic; -#endif -} - -void tsdbGetRootDir(int repoid, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb", repoid); -} - -void tsdbGetDataDir(int repoid, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data", repoid); -} - -static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { - // Check tsdbId - if (pCfg->tsdbId < 0) { - tsdbError("vgId:%d invalid vgroup ID", pCfg->tsdbId); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - - // Check precision - if (pCfg->precision == -1) { - pCfg->precision = TSDB_DEFAULT_PRECISION; - } else { - if (!IS_VALID_PRECISION(pCfg->precision)) { - tsdbError("vgId:%d invalid precision configuration %d", pCfg->tsdbId, pCfg->precision); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - // Check compression - if (pCfg->compression == -1) { - pCfg->compression = TSDB_DEFAULT_COMPRESSION; - } else { - if (!IS_VALID_COMPRESSION(pCfg->compression)) { - tsdbError("vgId:%d invalid compression configuration %d", pCfg->tsdbId, pCfg->precision); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - // Check daysPerFile - if (pCfg->daysPerFile == -1) { - pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; - } else { - if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { - tsdbError( - "vgId:%d invalid daysPerFile configuration! daysPerFile %d TSDB_MIN_DAYS_PER_FILE %d TSDB_MAX_DAYS_PER_FILE " - "%d", - pCfg->tsdbId, pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - // Check minRowsPerFileBlock and maxRowsPerFileBlock - if (pCfg->minRowsPerFileBlock == -1) { - pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; - } else { - if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { - tsdbError( - "vgId:%d invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MIN_ROW_FBLOCK %d " - "TSDB_MAX_MIN_ROW_FBLOCK %d", - pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - if (pCfg->maxRowsPerFileBlock == -1) { - pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; - } else { - if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { - tsdbError( - "vgId:%d invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAX_ROW_FBLOCK %d " - "TSDB_MAX_MAX_ROW_FBLOCK %d", - pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { - tsdbError("vgId:%d invalid configuration! minRowsPerFileBlock %d maxRowsPerFileBlock %d", pCfg->tsdbId, - pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - - // Check keep - if (pCfg->keep == -1) { - pCfg->keep = TSDB_DEFAULT_KEEP; - } else { - if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) { - tsdbError( - "vgId:%d invalid keep configuration! keep %d TSDB_MIN_KEEP %d " - "TSDB_MAX_KEEP %d", - pCfg->tsdbId, pCfg->keep, TSDB_MIN_KEEP, TSDB_MAX_KEEP); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - if (pCfg->keep1 == 0) { - pCfg->keep1 = pCfg->keep; - } - - if (pCfg->keep2 == 0) { - pCfg->keep2 = pCfg->keep; - } - - // update check - if (pCfg->update < TD_ROW_DISCARD_UPDATE || pCfg->update > TD_ROW_PARTIAL_UPDATE) - pCfg->update = TD_ROW_DISCARD_UPDATE; - - // update cacheLastRow - if (pCfg->cacheLastRow != 0) { - if (pCfg->cacheLastRow > 3) - pCfg->cacheLastRow = 1; - } - return 0; -} - -static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { - STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(*pRepo)); - if (pRepo == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - pRepo->state = TSDB_STATE_OK; - pRepo->code = TSDB_CODE_SUCCESS; - pRepo->compactState = 0; - pRepo->config = *pCfg; - if (pAppH) { - pRepo->appH = *pAppH; - } - pRepo->repoLocked = false; - pRepo->pthread = NULL; - - int code = pthread_mutex_init(&(pRepo->mutex), NULL); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - tsdbFreeRepo(pRepo); - return NULL; - } - - code = pthread_mutex_init(&(pRepo->save_mutex), NULL); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - tsdbFreeRepo(pRepo); - return NULL; - } - pRepo->config_changed = false; - atomic_store_8(&pRepo->hasCachedLastColumn, 0); - - code = tsem_init(&(pRepo->readyToCommit), 0, 1); - if (code != 0) { - code = errno; - terrno = TAOS_SYSTEM_ERROR(code); - tsdbFreeRepo(pRepo); - return NULL; - } - - pRepo->tsdbMeta = tsdbNewMeta(pCfg); - if (pRepo->tsdbMeta == NULL) { - tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno)); - tsdbFreeRepo(pRepo); - return NULL; - } - - pRepo->pPool = tsdbNewBufPool(pCfg); - if (pRepo->pPool == NULL) { - tsdbError("vgId:%d failed to create buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno)); - tsdbFreeRepo(pRepo); - return NULL; - } - - pRepo->fs = tsdbNewFS(pCfg); - if (pRepo->fs == NULL) { - tsdbError("vgId:%d failed to TSDB file system since %s", REPO_ID(pRepo), tstrerror(terrno)); - tsdbFreeRepo(pRepo); - return NULL; - } - - return pRepo; -} - -static void tsdbFreeRepo(STsdbRepo *pRepo) { - if (pRepo) { - tsdbFreeFS(pRepo->fs); - tsdbFreeBufPool(pRepo->pPool); - tsdbFreeMeta(pRepo->tsdbMeta); - tsdbFreeMergeBuf(pRepo->mergeBuf); - // tsdbFreeMemTable(pRepo->mem); - // tsdbFreeMemTable(pRepo->imem); - tsem_destroy(&(pRepo->readyToCommit)); - pthread_mutex_destroy(&pRepo->mutex); - free(pRepo); - } -} - -static void tsdbStartStream(STsdbRepo *pRepo) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - - for (int i = 0; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable && pTable->type == TSDB_STREAM_TABLE) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, - tsdbGetTableSchemaImpl(pTable, false, false, -1), 0); - } - } -} - -static void tsdbStopStream(STsdbRepo *pRepo) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - - for (int i = 0; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable && pTable->type == TSDB_STREAM_TABLE) { - (*pRepo->appH.cqDropFunc)(pTable->cqhandle); - } - } -} - -static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh) { - //tsdbInfo("tsdbRestoreLastColumns of table %s", pTable->name->data); - - STSchema *pSchema = tsdbGetTableLatestSchema(pTable); - if (pSchema == NULL) { - tsdbError("tsdbGetTableLatestSchema of table %s fail", pTable->name->data); - return 0; - } - - SBlock* pBlock; - int numColumns; - int32_t blockIdx; - SDataStatis* pBlockStatis = NULL; - SMemRow row = NULL; - // restore last column data with last schema - - int err = 0; - - numColumns = schemaNCols(pSchema); - if (numColumns <= pTable->restoreColumnNum) { - pTable->hasRestoreLastColumn = true; - return 0; - } - if (pTable->lastColSVersion != schemaVersion(pSchema)) { - if (tsdbInitColIdCacheWithSchema(pTable, pSchema) < 0) { - return -1; - } - } - - row = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); - if (row == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - err = -1; - goto out; - } - - memRowSetType(row, SMEM_ROW_DATA); - tdInitDataRow(memRowDataBody(row), pSchema); - - // first load block index info - if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { - err = -1; - goto out; - } - - pBlockStatis = calloc(numColumns, sizeof(SDataStatis)); - if (pBlockStatis == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - err = -1; - goto out; - } - memset(pBlockStatis, 0, numColumns * sizeof(SDataStatis)); - for(int32_t i = 0; i < numColumns; ++i) { - STColumn *pCol = schemaColAt(pSchema, i); - pBlockStatis[i].colId = pCol->colId; - } - - // load block from backward - SBlockIdx *pIdx = pReadh->pBlkIdx; - blockIdx = (int32_t)(pIdx->numOfBlocks - 1); - - while (numColumns > pTable->restoreColumnNum && blockIdx >= 0) { - bool loadStatisData = false; - pBlock = pReadh->pBlkInfo->blocks + blockIdx; - blockIdx -= 1; - - // load block data - if (tsdbLoadBlockData(pReadh, pBlock, NULL) < 0) { - err = -1; - goto out; - } - - // file block with sub-blocks has no statistics data - if (pBlock->numOfSubBlocks <= 1) { - tsdbLoadBlockStatis(pReadh, pBlock); - tsdbGetBlockStatis(pReadh, pBlockStatis, (int)numColumns); - loadStatisData = true; - } - - for (int16_t i = 0; i < numColumns && numColumns > pTable->restoreColumnNum; ++i) { - STColumn *pCol = schemaColAt(pSchema, i); - // ignore loaded columns - if (pTable->lastCols[i].bytes != 0) { - continue; - } - - // ignore block which has no not-null colId column - if (loadStatisData && pBlockStatis[i].numOfNull == pBlock->numOfRows) { - continue; - } - - // OK,let's load row from backward to get not-null column - for (int32_t rowId = pBlock->numOfRows - 1; rowId >= 0; rowId--) { - SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; - const void* pColData = tdGetColDataOfRow(pDataCol, rowId); - tdAppendColVal(memRowDataBody(row), pColData, pCol->type, pCol->offset); - //SDataCol *pDataCol = readh.pDCols[0]->cols + j; - void *value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); - if (isNull(value, pCol->type)) { - continue; - } - - int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pCol->colId); - if (idx == -1) { - tsdbError("tsdbRestoreLastColumns restore vgId:%d,table:%s cache column %d fail", REPO_ID(pRepo), pTable->name->data, pCol->colId); - continue; - } - // save not-null column - uint16_t bytes = IS_VAR_DATA_TYPE(pCol->type) ? varDataTLen(pColData) : pCol->bytes; - SDataCol *pLastCol = &(pTable->lastCols[idx]); - pLastCol->pData = malloc(bytes); - pLastCol->bytes = bytes; - pLastCol->colId = pCol->colId; - memcpy(pLastCol->pData, value, bytes); - - // save row ts(in column 0) - pDataCol = pReadh->pDCols[0]->cols + 0; - pCol = schemaColAt(pSchema, 0); - tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->offset); - pLastCol->ts = memRowKey(row); - - pTable->restoreColumnNum += 1; - - tsdbDebug("tsdbRestoreLastColumns restore vgId:%d,table:%s cache column %d, %" PRId64, REPO_ID(pRepo), pTable->name->data, pLastCol->colId, pLastCol->ts); - break; - } - } - } - -out: - taosTZfree(row); - tfree(pBlockStatis); - - if (err == 0 && numColumns <= pTable->restoreColumnNum) { - pTable->hasRestoreLastColumn = true; - } - - return err; -} - -static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, SBlockIdx *pIdx) { - ASSERT(pTable->lastRow == NULL); - if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { - return -1; - } - - SBlock* pBlock = pReadh->pBlkInfo->blocks + pIdx->numOfBlocks - 1; - - if (tsdbLoadBlockData(pReadh, pBlock, NULL) < 0) { - return -1; - } - - // Get the data in row - - STSchema *pSchema = tsdbGetTableSchema(pTable); - pTable->lastRow = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); - if (pTable->lastRow == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - memRowSetType(pTable->lastRow, SMEM_ROW_DATA); - tdInitDataRow(memRowDataBody(pTable->lastRow), pSchema); - for (int icol = 0; icol < schemaNCols(pSchema); icol++) { - STColumn *pCol = schemaColAt(pSchema, icol); - SDataCol *pDataCol = pReadh->pDCols[0]->cols + icol; - tdAppendColVal(memRowDataBody(pTable->lastRow), tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, - pCol->offset); - } - - return 0; -} - -int tsdbRestoreInfo(STsdbRepo *pRepo) { - SFSIter fsiter; - SReadH readh; - SDFileSet *pSet; - STsdbMeta *pMeta = pRepo->tsdbMeta; - STsdbCfg * pCfg = REPO_CFG(pRepo); - - if (tsdbInitReadH(&readh, pRepo) < 0) { - return -1; - } - - tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); - - if (CACHE_LAST_NULL_COLUMN(pCfg)) { - for (int i = 1; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - pTable->restoreColumnNum = 0; - pTable->hasRestoreLastColumn = false; - } - } - - while ((pSet = tsdbFSIterNext(&fsiter)) != NULL) { - if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - if (tsdbLoadBlockIdx(&readh) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - for (int i = 1; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - - //tsdbInfo("tsdbRestoreInfo restore vgId:%d,table:%s", REPO_ID(pRepo), pTable->name->data); - - if (tsdbSetReadTable(&readh, pTable) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); - SBlockIdx *pIdx = readh.pBlkIdx; - if (pIdx && lastKey < pIdx->maxKey) { - pTable->lastKey = pIdx->maxKey; - - if (CACHE_LAST_ROW(pCfg) && tsdbRestoreLastRow(pRepo, pTable, &readh, pIdx) != 0) { - tsdbDestroyReadH(&readh); - return -1; - } - } - - // restore NULL columns - if (pIdx && CACHE_LAST_NULL_COLUMN(pCfg) && !pTable->hasRestoreLastColumn) { - if (tsdbRestoreLastColumns(pRepo, pTable, &readh) != 0) { - tsdbDestroyReadH(&readh); - return -1; - } - } - } - } - - tsdbDestroyReadH(&readh); - - if (CACHE_LAST_NULL_COLUMN(pCfg)) { - atomic_store_8(&pRepo->hasCachedLastColumn, 1); - } - - return 0; -} - -int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) { - bool cacheLastRow = false, cacheLastCol = false; - SFSIter fsiter; - SReadH readh; - SDFileSet *pSet; - STsdbMeta *pMeta = pRepo->tsdbMeta; - int tableNum = 0; - int maxTableIdx = 0; - int cacheLastRowTableNum = 0; - int cacheLastColTableNum = 0; - - bool need_free_last_row = CACHE_LAST_ROW(oldCfg) && !CACHE_LAST_ROW(&(pRepo->config)); - bool need_free_last_col = CACHE_LAST_NULL_COLUMN(oldCfg) && !CACHE_LAST_NULL_COLUMN(&(pRepo->config)); - - if (CACHE_LAST_ROW(&(pRepo->config)) || CACHE_LAST_NULL_COLUMN(&(pRepo->config))) { - tsdbInfo("tsdbCacheLastData cache last data since cacheLast option changed"); - cacheLastRow = !CACHE_LAST_ROW(oldCfg) && CACHE_LAST_ROW(&(pRepo->config)); - cacheLastCol = !CACHE_LAST_NULL_COLUMN(oldCfg) && CACHE_LAST_NULL_COLUMN(&(pRepo->config)); - } - - // calc max table idx and table num - for (int i = 1; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - tableNum += 1; - maxTableIdx = i; - if (cacheLastCol) { - pTable->restoreColumnNum = 0; - pTable->hasRestoreLastColumn = false; - } - } - - // if close last option,need to free data - if (need_free_last_row || need_free_last_col) { - if (need_free_last_col) { - atomic_store_8(&pRepo->hasCachedLastColumn, 0); - } - tsdbInfo("free cache last data since cacheLast option changed"); - for (int i = 1; i <= maxTableIdx; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - if (need_free_last_row) { - taosTZfree(pTable->lastRow); - pTable->lastRow = NULL; - } - if (need_free_last_col) { - tsdbFreeLastColumns(pTable); - pTable->hasRestoreLastColumn = false; - } - } - } - - if (!cacheLastRow && !cacheLastCol) { - return 0; - } - - cacheLastRowTableNum = cacheLastRow ? tableNum : 0; - cacheLastColTableNum = cacheLastCol ? tableNum : 0; - - if (tsdbInitReadH(&readh, pRepo) < 0) { - return -1; - } - - tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); - - while ((pSet = tsdbFSIterNext(&fsiter)) != NULL && (cacheLastRowTableNum > 0 || cacheLastColTableNum > 0)) { - if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - if (tsdbLoadBlockIdx(&readh) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - for (int i = 1; i <= maxTableIdx; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - - //tsdbInfo("tsdbRestoreInfo restore vgId:%d,table:%s", REPO_ID(pRepo), pTable->name->data); - - if (tsdbSetReadTable(&readh, pTable) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - SBlockIdx *pIdx = readh.pBlkIdx; - - if (pIdx && cacheLastRowTableNum > 0 && pTable->lastRow == NULL) { - pTable->lastKey = pIdx->maxKey; - - if (tsdbRestoreLastRow(pRepo, pTable, &readh, pIdx) != 0) { - tsdbDestroyReadH(&readh); - return -1; - } - cacheLastRowTableNum -= 1; - } - - // restore NULL columns - if (pIdx && cacheLastColTableNum > 0 && !pTable->hasRestoreLastColumn) { - if (tsdbRestoreLastColumns(pRepo, pTable, &readh) != 0) { - tsdbDestroyReadH(&readh); - return -1; - } - if (pTable->hasRestoreLastColumn) { - cacheLastColTableNum -= 1; - } - } - } - } - - tsdbDestroyReadH(&readh); - - if (cacheLastCol) { - atomic_store_8(&pRepo->hasCachedLastColumn, 1); - } - - return 0; -} diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c deleted file mode 100644 index a311868de6f7254d776f08a4f4a247293609aef5..0000000000000000000000000000000000000000 --- a/src/tsdb/src/tsdbMeta.c +++ /dev/null @@ -1,1509 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "tsdbint.h" - -#define TSDB_SUPER_TABLE_SL_LEVEL 5 -#define DEFAULT_TAG_INDEX_COLUMN 0 - -static char * getTagIndexKey(const void *pData); -static STable *tsdbNewTable(); -static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable); -static void tsdbFreeTable(STable *pTable); -static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, bool lock); -static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx, bool lock); -static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper); -static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable); -static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid); -static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup); -static int tsdbTableSetName(STableCfg *config, char *name, bool dup); -static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup); -static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); -static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid); -static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup); -static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup); -static int tsdbEncodeTableName(void **buf, tstr *name); -static void * tsdbDecodeTableName(void *buf, tstr **name); -static int tsdbEncodeTable(void **buf, STable *pTable); -static void * tsdbDecodeTable(void *buf, STable **pRTable); -static int tsdbGetTableEncodeSize(int8_t act, STable *pTable); -static void * tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STable *pTable); -static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable); -static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable); -static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid); -static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema); -static int tsdbInsertNewTableAction(STsdbRepo *pRepo, STable* pTable); -static int tsdbAddSchema(STable *pTable, STSchema *pSchema); -static void tsdbFreeTableSchema(STable *pTable); - -// ------------------ OUTER FUNCTIONS ------------------ -int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbMeta *pMeta = pRepo->tsdbMeta; - STable * super = NULL; - STable * table = NULL; - bool newSuper = false; - bool superChanged = false; - int tid = pCfg->tableId.tid; - STable * pTable = NULL; - - if (tid < 1 || tid > TSDB_MAX_TABLES) { - tsdbError("vgId:%d failed to create table since invalid tid %d", REPO_ID(pRepo), tid); - terrno = TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO; - goto _err; - } - - if (tid < pMeta->maxTables && pMeta->tables[tid] != NULL) { - if (TABLE_UID(pMeta->tables[tid]) == pCfg->tableId.uid) { - tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), - TABLE_CHAR_NAME(pMeta->tables[tid]), TABLE_TID(pMeta->tables[tid]), TABLE_UID(pMeta->tables[tid])); - return 0; - } else { - tsdbInfo("vgId:%d table %s at tid %d uid %" PRIu64 - " exists, replace it with new table, this can be not reasonable", - REPO_ID(pRepo), TABLE_CHAR_NAME(pMeta->tables[tid]), TABLE_TID(pMeta->tables[tid]), - TABLE_UID(pMeta->tables[tid])); - tsdbDropTable(pRepo, pMeta->tables[tid]->tableId); - } - } - - pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid); - if (pTable != NULL) { - tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - TABLE_TID(pTable), TABLE_UID(pTable)); - terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; - goto _err; - } - - if (pCfg->type == TSDB_CHILD_TABLE) { - super = tsdbGetTableByUid(pMeta, pCfg->superUid); - if (super == NULL) { // super table not exists, try to create it - newSuper = true; - super = tsdbCreateTableFromCfg(pCfg, true, NULL); - if (super == NULL) goto _err; - } else { - if (TABLE_TYPE(super) != TSDB_SUPER_TABLE || TABLE_UID(super) != pCfg->superUid) { - terrno = TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO; - goto _err; - } - - if (schemaVersion(pCfg->tagSchema) > schemaVersion(super->tagSchema)) { - // tag schema out of date, need to update super table tag version - STSchema *pOldSchema = super->tagSchema; - TSDB_WLOCK_TABLE(super); - super->tagSchema = tdDupSchema(pCfg->tagSchema); - TSDB_WUNLOCK_TABLE(super); - tdFreeSchema(pOldSchema); - - superChanged = true; - } - } - } - - table = tsdbCreateTableFromCfg(pCfg, false, super); - if (table == NULL) goto _err; - - // Register to meta - tsdbWLockRepoMeta(pRepo); - if (newSuper) { - if (tsdbAddTableToMeta(pRepo, super, true, false) < 0) { - tsdbUnlockRepoMeta(pRepo); - goto _err; - } - } - if (tsdbAddTableToMeta(pRepo, table, true, false) < 0) { - tsdbUnlockRepoMeta(pRepo); - goto _err; - } - tsdbUnlockRepoMeta(pRepo); - - // Write to memtable action - if (newSuper || superChanged) { - // add insert new super table action - if (tsdbInsertNewTableAction(pRepo, super) != 0) { - goto _err; - } - } - // add insert new table action - if (tsdbInsertNewTableAction(pRepo, table) != 0) { - goto _err; - } - - if (tsdbCheckCommit(pRepo) < 0) return -1; - - return 0; - -_err: - if (newSuper) { - tsdbFreeTable(super); - } - tsdbFreeTable(table); - return -1; -} - -int tsdbDropTable(STsdbRepo *repo, STableId tableId) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbMeta *pMeta = pRepo->tsdbMeta; - uint64_t uid = tableId.uid; - int tid = 0; - char * tbname = NULL; - - STable *pTable = tsdbGetTableByUid(pMeta, uid); - if (pTable == NULL) { - tsdbError("vgId:%d failed to drop table since table not exists! tid:%d uid %" PRIu64, REPO_ID(pRepo), tableId.tid, - uid); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - return -1; - } - - tsdbDebug("vgId:%d try to drop table %s type %d", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TYPE(pTable)); - - tid = TABLE_TID(pTable); - tbname = strdup(TABLE_CHAR_NAME(pTable)); - if (tbname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - // Write to KV store first - if (tsdbRemoveTableFromStore(pRepo, pTable) < 0) { - tsdbError("vgId:%d failed to drop table %s since %s", REPO_ID(pRepo), tbname, tstrerror(terrno)); - goto _err; - } - - // Remove table from Meta - if (tsdbRmTableFromMeta(pRepo, pTable) < 0) { - tsdbError("vgId:%d failed to drop table %s since %s", REPO_ID(pRepo), tbname, tstrerror(terrno)); - goto _err; - } - - tsdbDebug("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, tbname, tid, uid); - free(tbname); - - if (tsdbCheckCommit(pRepo) < 0) goto _err; - - return 0; - -_err: - tfree(tbname); - return -1; -} - -void *tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_t bytes) { - // TODO: this function should be changed also - - STSchema *pSchema = tsdbGetTableTagSchema((STable*) pTable); - STColumn *pCol = tdGetColOfID(pSchema, colId); - if (pCol == NULL) { - return NULL; // No matched tag volumn - } - - char *val = tdGetKVRowValOfCol(((STable*)pTable)->tagVal, colId); - assert(type == pCol->type && bytes >= pCol->bytes); - - // if (val != NULL && IS_VAR_DATA_TYPE(type)) { - // assert(varDataLen(val) < pCol->bytes); - // } - - return val; -} - -char *tsdbGetTableName(void* pTable) { - // TODO: need to change as thread-safe - - if (pTable == NULL) { - return NULL; - } else { - return (char*) (((STable *)pTable)->name); - } -} - -STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { - if (pMsg == NULL) return NULL; - - SSchema *pSchema = (SSchema *)pMsg->data; - int16_t numOfCols = htons(pMsg->numOfColumns); - int16_t numOfTags = htons(pMsg->numOfTags); - - STSchemaBuilder schemaBuilder = {0}; - - STableCfg *pCfg = (STableCfg *)calloc(1, sizeof(STableCfg)); - if (pCfg == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - if (tsdbInitTableCfg(pCfg, pMsg->tableType, htobe64(pMsg->uid), htonl(pMsg->tid)) < 0) goto _err; - if (tdInitTSchemaBuilder(&schemaBuilder, htonl(pMsg->sversion)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - for (int i = 0; i < numOfCols; i++) { - if (tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - } - if (tsdbTableSetSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err; - if (tsdbTableSetName(pCfg, pMsg->tableFname, true) < 0) goto _err; - - if (numOfTags > 0) { - // Decode tag schema - tdResetTSchemaBuilder(&schemaBuilder, htonl(pMsg->tversion)); - for (int i = numOfCols; i < numOfCols + numOfTags; i++) { - if (tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - } - if (tsdbTableSetTagSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err; - if (tsdbTableSetSName(pCfg, pMsg->stableFname, true) < 0) goto _err; - if (tsdbTableSetSuperUid(pCfg, htobe64(pMsg->superTableUid)) < 0) goto _err; - - int32_t tagDataLen = htonl(pMsg->tagDataLen); - if (tagDataLen) { - char *pTagData = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema); - tsdbTableSetTagValue(pCfg, pTagData, true); - } - } - - if (pMsg->tableType == TSDB_STREAM_TABLE) { - char *sql = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema); - tsdbTableSetStreamSql(pCfg, sql, true); - } - - tdDestroyTSchemaBuilder(&schemaBuilder); - - return pCfg; - -_err: - tdDestroyTSchemaBuilder(&schemaBuilder); - tsdbClearTableCfg(pCfg); - return NULL; -} - -static UNUSED_FUNC int32_t colIdCompar(const void* left, const void* right) { - int16_t colId = *(int16_t*) left; - STColumn* p2 = (STColumn*) right; - - if (colId == p2->colId) { - return 0; - } - - return (colId < p2->colId)? -1:1; -} - -int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbMeta *pMeta = pRepo->tsdbMeta; - STSchema * pNewSchema = NULL; - - pMsg->uid = htobe64(pMsg->uid); - pMsg->tid = htonl(pMsg->tid); - pMsg->tversion = htons(pMsg->tversion); - pMsg->colId = htons(pMsg->colId); - pMsg->bytes = htons(pMsg->bytes); - pMsg->tagValLen = htonl(pMsg->tagValLen); - pMsg->numOfTags = htons(pMsg->numOfTags); - pMsg->schemaLen = htonl(pMsg->schemaLen); - for (int i = 0; i < pMsg->numOfTags; i++) { - STColumn *pTCol = (STColumn *)pMsg->data + i; - pTCol->bytes = htons(pTCol->bytes); - pTCol->colId = htons(pTCol->colId); - } - - STable *pTable = tsdbGetTableByUid(pMeta, pMsg->uid); - if (pTable == NULL || TABLE_TID(pTable) != pMsg->tid) { - tsdbError("vgId:%d failed to update table tag value since invalid table id %d uid %" PRIu64, REPO_ID(pRepo), - pMsg->tid, pMsg->uid); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - return -1; - } - - if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { - tsdbError("vgId:%d try to update tag value of a non-child table, invalid action", REPO_ID(pRepo)); - terrno = TSDB_CODE_TDB_INVALID_ACTION; - return -1; - } - - if (schemaVersion(pTable->pSuper->tagSchema) > pMsg->tversion) { - tsdbError( - "vgId:%d failed to update tag value of table %s since version out of date, client tag version %d server tag " - "version %d", - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), pMsg->tversion, schemaVersion(pTable->tagSchema)); - terrno = TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE; - return -1; - } - - if (schemaVersion(pTable->pSuper->tagSchema) < pMsg->tversion) { // tag schema out of data, - tsdbDebug("vgId:%d need to update tag schema of table %s tid %d uid %" PRIu64 - " since out of date, current version %d new version %d", - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), - schemaVersion(pTable->pSuper->tagSchema), pMsg->tversion); - - STSchemaBuilder schemaBuilder = {0}; - - STColumn *pTCol = (STColumn *)pMsg->data; - ASSERT(pMsg->schemaLen % sizeof(STColumn) == 0 && pTCol[0].colId == colColId(schemaColAt(pTable->pSuper->tagSchema, 0))); - if (tdInitTSchemaBuilder(&schemaBuilder, pMsg->tversion) < 0) { - tsdbDebug("vgId:%d failed to update tag schema of table %s tid %d uid %" PRIu64 " since out of memory", - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - for (int i = 0; i < (pMsg->schemaLen / sizeof(STColumn)); i++) { - if (tdAddColToSchema(&schemaBuilder, pTCol[i].type, pTCol[i].colId, pTCol[i].bytes) < 0) { - tdDestroyTSchemaBuilder(&schemaBuilder); - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } - pNewSchema = tdGetSchemaFromBuilder(&schemaBuilder); - if (pNewSchema == NULL) { - tdDestroyTSchemaBuilder(&schemaBuilder); - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - tdDestroyTSchemaBuilder(&schemaBuilder); - } - - // Change in memory - if (pNewSchema != NULL) { // change super table tag schema - TSDB_WLOCK_TABLE(pTable->pSuper); - STSchema *pOldSchema = pTable->pSuper->tagSchema; - pTable->pSuper->tagSchema = pNewSchema; - tdFreeSchema(pOldSchema); - TSDB_WUNLOCK_TABLE(pTable->pSuper); - } - - bool isChangeIndexCol = (pMsg->colId == colColId(schemaColAt(pTable->pSuper->tagSchema, 0))); - // STColumn *pCol = bsearch(&(pMsg->colId), pMsg->data, pMsg->numOfTags, sizeof(STColumn), colIdCompar); - // ASSERT(pCol != NULL); - - if (isChangeIndexCol) { - tsdbWLockRepoMeta(pRepo); - tsdbRemoveTableFromIndex(pMeta, pTable); - } - TSDB_WLOCK_TABLE(pTable); - tdSetKVRowDataOfCol(&(pTable->tagVal), pMsg->colId, pMsg->type, POINTER_SHIFT(pMsg->data, pMsg->schemaLen)); - TSDB_WUNLOCK_TABLE(pTable); - if (isChangeIndexCol) { - tsdbAddTableIntoIndex(pMeta, pTable, false); - tsdbUnlockRepoMeta(pRepo); - } - - // Update on file - int tlen1 = (pNewSchema) ? tsdbGetTableEncodeSize(TSDB_UPDATE_META, pTable->pSuper) : 0; - int tlen2 = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pTable); - void *buf = tsdbAllocBytes(pRepo, tlen1+tlen2); - ASSERT(buf != NULL); - if (pNewSchema) { - void *pBuf = tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, pTable->pSuper); - ASSERT(POINTER_DISTANCE(pBuf, buf) == tlen1); - buf = pBuf; - } - tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, pTable); - - if (tsdbCheckCommit(pRepo) < 0) return -1; - - return 0; -} - -// ------------------ INTERNAL FUNCTIONS ------------------ -static int tsdbInsertNewTableAction(STsdbRepo *pRepo, STable* pTable) { - int tlen = 0; - void *pBuf = NULL; - - tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pTable); - pBuf = tsdbAllocBytes(pRepo, tlen); - if (pBuf == NULL) { - return -1; - } - void *tBuf = tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, pBuf, pTable); - ASSERT(POINTER_DISTANCE(tBuf, pBuf) == tlen); - - return 0; -} - -STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { - STsdbMeta *pMeta = (STsdbMeta *)calloc(1, sizeof(*pMeta)); - if (pMeta == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - int code = pthread_rwlock_init(&pMeta->rwLock, NULL); - if (code != 0) { - tsdbError("vgId:%d failed to init TSDB meta r/w lock since %s", pCfg->tsdbId, strerror(code)); - terrno = TAOS_SYSTEM_ERROR(code); - goto _err; - } - - pMeta->maxTables = TSDB_INIT_NTABLES + 1; - pMeta->tables = (STable **)calloc(pMeta->maxTables, sizeof(STable *)); - if (pMeta->tables == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - pMeta->superList = tdListNew(sizeof(STable *)); - if (pMeta->superList == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - pMeta->uidMap = taosHashInit((size_t)(TSDB_INIT_NTABLES * 1.1), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - if (pMeta->uidMap == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - return pMeta; - -_err: - tsdbFreeMeta(pMeta); - return NULL; -} - -void tsdbFreeMeta(STsdbMeta *pMeta) { - if (pMeta) { - taosHashCleanup(pMeta->uidMap); - tdListFree(pMeta->superList); - tfree(pMeta->tables); - pthread_rwlock_destroy(&pMeta->rwLock); - free(pMeta); - } -} - -int tsdbOpenMeta(STsdbRepo *pRepo) { - return 0; -#if 0 - char * fname = NULL; - STsdbMeta *pMeta = pRepo->tsdbMeta; - ASSERT(pMeta != NULL); - - fname = tsdbGetMetaFileName(pRepo->rootDir); - if (fname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - // pMeta->pStore = tdOpenKVStore(fname, tsdbRestoreTable, tsdbOrgMeta, (void *)pRepo); - // if (pMeta->pStore == NULL) { - // tsdbError("vgId:%d failed to open TSDB meta while open the kv store since %s", REPO_ID(pRepo), tstrerror(terrno)); - // goto _err; - // } - - tsdbDebug("vgId:%d open TSDB meta succeed", REPO_ID(pRepo)); - tfree(fname); - return 0; - -_err: - tfree(fname); - return -1; -#endif -} - -int tsdbCloseMeta(STsdbRepo *pRepo) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - SListNode *pNode = NULL; - STable * pTable = NULL; - - if (pMeta == NULL) return 0; - // tdCloseKVStore(pMeta->pStore); - for (int i = 1; i < pMeta->maxTables; i++) { - tsdbFreeTable(pMeta->tables[i]); - } - - while ((pNode = tdListPopHead(pMeta->superList)) != NULL) { - tdListNodeGetData(pMeta->superList, pNode, (void *)(&pTable)); - tsdbFreeTable(pTable); - listNodeFree(pNode); - } - - tsdbDebug("vgId:%d TSDB meta is closed", REPO_ID(pRepo)); - return 0; -} - -STable *tsdbGetTableByUid(STsdbMeta *pMeta, uint64_t uid) { - void *ptr = taosHashGet(pMeta->uidMap, (char *)(&uid), sizeof(uid)); - - if (ptr == NULL) return NULL; - - return *(STable **)ptr; -} - -STSchema *tsdbGetTableSchemaByVersion(STable *pTable, int16_t _version) { - return tsdbGetTableSchemaImpl(pTable, true, false, _version); -} - -int tsdbWLockRepoMeta(STsdbRepo *pRepo) { - int code = pthread_rwlock_wrlock(&(pRepo->tsdbMeta->rwLock)); - if (code != 0) { - tsdbError("vgId:%d failed to write lock TSDB meta since %s", REPO_ID(pRepo), strerror(code)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - - return 0; -} - -int tsdbRLockRepoMeta(STsdbRepo *pRepo) { - int code = pthread_rwlock_rdlock(&(pRepo->tsdbMeta->rwLock)); - if (code != 0) { - tsdbError("vgId:%d failed to read lock TSDB meta since %s", REPO_ID(pRepo), strerror(code)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - - return 0; -} - -int tsdbUnlockRepoMeta(STsdbRepo *pRepo) { - int code = pthread_rwlock_unlock(&(pRepo->tsdbMeta->rwLock)); - if (code != 0) { - tsdbError("vgId:%d failed to unlock TSDB meta since %s", REPO_ID(pRepo), strerror(code)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - - return 0; -} - -void tsdbRefTable(STable *pTable) { - int32_t ref = T_REF_INC(pTable); - UNUSED(ref); - tsdbDebug("ref table %s uid %" PRIu64 " tid:%d, refCount:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), ref); -} - -void tsdbUnRefTable(STable *pTable) { - uint64_t uid = TABLE_UID(pTable); - int32_t tid = TABLE_TID(pTable); - int32_t ref = T_REF_DEC(pTable); - - tsdbDebug("unref table, uid:%" PRIu64 " tid:%d, refCount:%d", uid, tid, ref); - - if (ref == 0) { - if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { - tsdbUnRefTable(pTable->pSuper); - } - tsdbFreeTable(pTable); - } -} - -void tsdbFreeLastColumns(STable* pTable) { - if (pTable->lastCols == NULL) { - return; - } - - for (int i = 0; i < pTable->maxColNum; ++i) { - if (pTable->lastCols[i].bytes == 0) { - continue; - } - tfree(pTable->lastCols[i].pData); - pTable->lastCols[i].bytes = 0; - pTable->lastCols[i].pData = NULL; - } - tfree(pTable->lastCols); - pTable->lastCols = NULL; - pTable->maxColNum = 0; - pTable->lastColSVersion = -1; - pTable->restoreColumnNum = 0; - pTable->hasRestoreLastColumn = false; -} - -int16_t tsdbGetLastColumnsIndexByColId(STable* pTable, int16_t colId) { - if (pTable->lastCols == NULL) { - return -1; - } - // TODO: use binary search instead - for (int16_t i = 0; i < pTable->maxColNum; ++i) { - if (pTable->lastCols[i].colId == colId) { - return i; - } - } - - return -1; -} - -int tsdbInitColIdCacheWithSchema(STable* pTable, STSchema* pSchema) { - ASSERT(pTable->lastCols == NULL); - - int16_t numOfColumn = pSchema->numOfCols; - - pTable->lastCols = (SDataCol*)malloc(numOfColumn * sizeof(SDataCol)); - if (pTable->lastCols == NULL) { - return -1; - } - - for (int16_t i = 0; i < numOfColumn; ++i) { - STColumn *pCol = schemaColAt(pSchema, i); - SDataCol* pDataCol = &(pTable->lastCols[i]); - pDataCol->bytes = 0; - pDataCol->pData = NULL; - pDataCol->colId = pCol->colId; - } - - pTable->lastColSVersion = schemaVersion(pSchema); - pTable->maxColNum = numOfColumn; - pTable->restoreColumnNum = 0; - pTable->hasRestoreLastColumn = false; - return 0; -} - -STSchema* tsdbGetTableLatestSchema(STable *pTable) { - return tsdbGetTableSchemaByVersion(pTable, -1); -} - -int tsdbUpdateLastColSchema(STable *pTable, STSchema *pNewSchema) { - if (pTable->lastColSVersion == schemaVersion(pNewSchema)) { - return 0; - } - - tsdbDebug("tsdbUpdateLastColSchema:%s,%d->%d", pTable->name->data, pTable->lastColSVersion, schemaVersion(pNewSchema)); - - int16_t numOfCols = pNewSchema->numOfCols; - SDataCol *lastCols = (SDataCol*)malloc(numOfCols * sizeof(SDataCol)); - if (lastCols == NULL) { - return -1; - } - - TSDB_WLOCK_TABLE(pTable); - - for (int16_t i = 0; i < numOfCols; ++i) { - STColumn *pCol = schemaColAt(pNewSchema, i); - int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pCol->colId); - - SDataCol* pDataCol = &(lastCols[i]); - if (idx != -1) { - // move col data to new last column array - SDataCol* pOldDataCol = &(pTable->lastCols[idx]); - memcpy(pDataCol, pOldDataCol, sizeof(SDataCol)); - } else { - // init new colid data - pDataCol->colId = pCol->colId; - pDataCol->bytes = 0; - pDataCol->pData = NULL; - } - } - - SDataCol *oldLastCols = pTable->lastCols; - int16_t oldLastColNum = pTable->maxColNum; - - pTable->lastColSVersion = schemaVersion(pNewSchema); - pTable->lastCols = lastCols; - pTable->maxColNum = numOfCols; - - if (oldLastCols == NULL) { - TSDB_WUNLOCK_TABLE(pTable); - return 0; - } - - // free old schema last column datas - for (int16_t i = 0; i < oldLastColNum; ++i) { - SDataCol* pDataCol = &(oldLastCols[i]); - if (pDataCol->bytes == 0) { - continue; - } - int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pDataCol->colId); - if (idx != -1) { - continue; - } - - // free not exist column data - tfree(pDataCol->pData); - } - TSDB_WUNLOCK_TABLE(pTable); - tfree(oldLastCols); - - return 0; -} - -void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema, bool insertAct) { - ASSERT(TABLE_TYPE(pTable) != TSDB_STREAM_TABLE && TABLE_TYPE(pTable) != TSDB_SUPER_TABLE); - STsdbMeta *pMeta = pRepo->tsdbMeta; - - STable *pCTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable; - ASSERT(schemaVersion(pSchema) > schemaVersion(*(STSchema **)taosArrayGetLast(pCTable->schema))); - - TSDB_WLOCK_TABLE(pCTable); - tsdbAddSchema(pCTable, pSchema); - - if (schemaNCols(pSchema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(pSchema); - if (schemaTLen(pSchema) > pMeta->maxRowBytes) pMeta->maxRowBytes = schemaTLen(pSchema); - TSDB_WUNLOCK_TABLE(pCTable); - - if (insertAct) { - if (tsdbInsertNewTableAction(pRepo, pCTable) != 0) { - tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " tsdbInsertNewTableAction fail", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - TABLE_TID(pTable), TABLE_UID(pTable)); - } - } -} - -int tsdbRestoreTable(STsdbRepo *pRepo, void *cont, int contLen) { - STable *pTable = NULL; - - if (!taosCheckChecksumWhole((uint8_t *)cont, contLen)) { - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - tsdbDecodeTable(cont, &pTable); - - if (tsdbAddTableToMeta(pRepo, pTable, false, false) < 0) { - tsdbFreeTable(pTable); - return -1; - } - - tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is restored from file", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - TABLE_TID(pTable), TABLE_UID(pTable)); - return 0; -} - -void tsdbOrgMeta(STsdbRepo *pRepo) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - - for (int i = 1; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable != NULL && pTable->type == TSDB_CHILD_TABLE) { - tsdbAddTableIntoIndex(pMeta, pTable, true); - } - } -} - -// ------------------ LOCAL FUNCTIONS ------------------ -static char *getTagIndexKey(const void *pData) { - STable *pTable = (STable *)pData; - - STSchema *pSchema = tsdbGetTableTagSchema(pTable); - STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN); - void * res = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId); - if (res == NULL) { - // treat the column as NULL if we cannot find it - res = (char*)getNullValue(pCol->type); - } - return res; -} - -static STable *tsdbNewTable() { - STable *pTable = (STable *)calloc(1, sizeof(*pTable)); - if (pTable == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - pTable->lastKey = TSKEY_INITIAL_VAL; - - pTable->lastCols = NULL; - pTable->restoreColumnNum = 0; - pTable->maxColNum = 0; - pTable->hasRestoreLastColumn = false; - pTable->lastColSVersion = -1; - return pTable; -} - -static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable) { - STable *pTable = NULL; - size_t tsize = 0; - - pTable = tsdbNewTable(); - if (pTable == NULL) goto _err; - - if (isSuper) { - pTable->type = TSDB_SUPER_TABLE; - tsize = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN - 1); - pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1); - if (pTable->name == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - STR_WITH_SIZE_TO_VARSTR(pTable->name, pCfg->sname, (VarDataLenT)tsize); - TABLE_UID(pTable) = pCfg->superUid; - TABLE_TID(pTable) = -1; - TABLE_SUID(pTable) = -1; - pTable->pSuper = NULL; - if (tsdbAddSchema(pTable, tdDupSchema(pCfg->schema)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - pTable->tagSchema = tdDupSchema(pCfg->tagSchema); - if (pTable->tagSchema == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - pTable->tagVal = NULL; - STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, - SL_ALLOW_DUP_KEY, getTagIndexKey); - if (pTable->pIndex == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - } else { - pTable->type = pCfg->type; - tsize = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN - 1); - pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1); - if (pTable->name == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - STR_WITH_SIZE_TO_VARSTR(pTable->name, pCfg->name, (VarDataLenT)tsize); - TABLE_UID(pTable) = pCfg->tableId.uid; - TABLE_TID(pTable) = pCfg->tableId.tid; - - if (pCfg->type == TSDB_CHILD_TABLE) { - TABLE_SUID(pTable) = pCfg->superUid; - if (tsdbCheckTableTagVal(pCfg->tagValues, pSTable->tagSchema) < 0) { - goto _err; - } - pTable->tagVal = tdKVRowDup(pCfg->tagValues); - if (pTable->tagVal == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - } else { - TABLE_SUID(pTable) = -1; - if (tsdbAddSchema(pTable, tdDupSchema(pCfg->schema)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { - pTable->sql = strdup(pCfg->sql); - if (pTable->sql == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - } - } - } - - T_REF_INC(pTable); - - tsdbDebug("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), - TABLE_UID(pTable)); - - return pTable; - -_err: - tsdbFreeTable(pTable); - return NULL; -} - -static void tsdbFreeTable(STable *pTable) { - if (pTable) { - if (pTable->name != NULL) - tsdbTrace("table %s tid %d uid %" PRIu64 " is freed", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), - TABLE_UID(pTable)); - tfree(TABLE_NAME(pTable)); - if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { - tsdbFreeTableSchema(pTable); - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tdFreeSchema(pTable->tagSchema); - } - } - - kvRowFree(pTable->tagVal); - - tSkipListDestroy(pTable->pIndex); - taosTZfree(pTable->lastRow); - tfree(pTable->sql); - - tsdbFreeLastColumns(pTable); - free(pTable); - } -} - -static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, bool lock) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - - if (lock && tsdbWLockRepoMeta(pRepo) < 0) { - tsdbError("vgId:%d failed to add table %s to meta since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - tstrerror(terrno)); - return -1; - } - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - if (tdListAppend(pMeta->superList, (void *)(&pTable)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to add table %s to meta since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - tstrerror(terrno)); - goto _err; - } - } else { - if (TABLE_TID(pTable) >= pMeta->maxTables) { - if (tsdbAdjustMetaTables(pRepo, TABLE_TID(pTable)) < 0) goto _err; - } - if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index - if (tsdbAddTableIntoIndex(pMeta, pTable, true) < 0) { - tsdbDebug("vgId:%d failed to add table %s to meta while add table to index since %s", REPO_ID(pRepo), - TABLE_CHAR_NAME(pTable), tstrerror(terrno)); - goto _err; - } - } - ASSERT(TABLE_TID(pTable) < pMeta->maxTables); - pMeta->tables[TABLE_TID(pTable)] = pTable; - pMeta->nTables++; - } - - if (taosHashPut(pMeta->uidMap, (char *)(&pTable->tableId.uid), sizeof(pTable->tableId.uid), (void *)(&pTable), - sizeof(pTable)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to add table %s to meta while put into uid map since %s", REPO_ID(pRepo), - TABLE_CHAR_NAME(pTable), tstrerror(terrno)); - goto _err; - } - - if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { - STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - if (schemaNCols(pSchema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(pSchema); - if (schemaTLen(pSchema) > pMeta->maxRowBytes) pMeta->maxRowBytes = schemaTLen(pSchema); - } - - if (lock && tsdbUnlockRepoMeta(pRepo) < 0) return -1; - if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE && addIdx) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, - tsdbGetTableSchemaImpl(pTable, false, false, -1), 1); - } - - tsdbDebug("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - TABLE_TID(pTable), TABLE_UID(pTable)); - return 0; - -_err: - tsdbRemoveTableFromMeta(pRepo, pTable, false, false); - if (lock) tsdbUnlockRepoMeta(pRepo); - return -1; -} - -static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx, bool lock) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - SListIter lIter = {0}; - SListNode *pNode = NULL; - STable * tTable = NULL; - - STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - int maxCols = schemaNCols(pSchema); - int maxRowBytes = schemaTLen(pSchema); - - if (lock) tsdbWLockRepoMeta(pRepo); - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tdListInitIter(pMeta->superList, &lIter, TD_LIST_BACKWARD); - - while ((pNode = tdListNext(&lIter)) != NULL) { - tdListNodeGetData(pMeta->superList, pNode, (void *)(&tTable)); - if (pTable == tTable) { - tdListPopNode(pMeta->superList, pNode); - free(pNode); - break; - } - } - } else { - pMeta->tables[pTable->tableId.tid] = NULL; - if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE && rmFromIdx) { - tsdbRemoveTableFromIndex(pMeta, pTable); - } - - pMeta->nTables--; - } - - taosHashRemove(pMeta->uidMap, (char *)(&(TABLE_UID(pTable))), sizeof(TABLE_UID(pTable))); - - if (maxCols == pMeta->maxCols || maxRowBytes == pMeta->maxRowBytes) { - maxCols = 0; - maxRowBytes = 0; - for (int i = 0; i < pMeta->maxTables; i++) { - STable *_pTable = pMeta->tables[i]; - if (_pTable != NULL) { - pSchema = tsdbGetTableSchemaImpl(_pTable, false, false, -1); - maxCols = MAX(maxCols, schemaNCols(pSchema)); - maxRowBytes = MAX(maxRowBytes, schemaTLen(pSchema)); - } - } - } - pMeta->maxCols = maxCols; - pMeta->maxRowBytes = maxRowBytes; - - if (lock) tsdbUnlockRepoMeta(pRepo); - tsdbDebug("vgId:%d table %s uid %" PRIu64 " is removed from meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_UID(pTable)); - tsdbUnRefTable(pTable); -} - -static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper) { - ASSERT(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); - STable *pSTable = tsdbGetTableByUid(pMeta, TABLE_SUID(pTable)); - ASSERT(pSTable != NULL); - - pTable->pSuper = pSTable; - - tSkipListPut(pSTable->pIndex, (void *)pTable); - - if (refSuper) T_REF_INC(pSTable); - return 0; -} - -static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { - ASSERT(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); - - STable *pSTable = pTable->pSuper; - ASSERT(pSTable != NULL); - - char* key = getTagIndexKey(pTable); - SArray *res = tSkipListGet(pSTable->pIndex, key); - - size_t size = taosArrayGetSize(res); - ASSERT(size > 0); - - for (int32_t i = 0; i < size; ++i) { - SSkipListNode *pNode = taosArrayGetP(res, i); - - // STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode); - if ((STable *)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need - tSkipListRemoveNode(pSTable->pIndex, pNode); - } - } - - taosArrayDestroy(res); - return 0; -} - -static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid) { - if (type != TSDB_CHILD_TABLE && type != TSDB_NORMAL_TABLE && type != TSDB_STREAM_TABLE) { - terrno = TSDB_CODE_TDB_INVALID_TABLE_TYPE; - return -1; - } - - memset((void *)config, 0, sizeof(*config)); - - config->type = type; - config->superUid = TSDB_INVALID_SUPER_TABLE_ID; - config->tableId.uid = uid; - config->tableId.tid = tid; - return 0; -} - -static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup) { - if (dup) { - config->schema = tdDupSchema(pSchema); - if (config->schema == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } else { - config->schema = pSchema; - } - return 0; -} - -static int tsdbTableSetName(STableCfg *config, char *name, bool dup) { - if (dup) { - config->name = strdup(name); - if (config->name == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } else { - config->name = name; - } - - return 0; -} - -static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup) { - if (config->type != TSDB_CHILD_TABLE) { - terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; - return -1; - } - - if (dup) { - config->tagSchema = tdDupSchema(pSchema); - if (config->tagSchema == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } else { - config->tagSchema = pSchema; - } - return 0; -} - -static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup) { - if (config->type != TSDB_CHILD_TABLE) { - terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; - return -1; - } - - if (dup) { - config->sname = strdup(sname); - if (config->sname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } else { - config->sname = sname; - } - return 0; -} - -static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid) { - if (config->type != TSDB_CHILD_TABLE || uid == TSDB_INVALID_SUPER_TABLE_ID) { - terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; - return -1; - } - - config->superUid = uid; - return 0; -} - -static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup) { - if (config->type != TSDB_CHILD_TABLE) { - terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; - return -1; - } - - if (dup) { - config->tagValues = tdKVRowDup(row); - if (config->tagValues == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } else { - config->tagValues = row; - } - - return 0; -} - -static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup) { - if (config->type != TSDB_STREAM_TABLE) { - terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; - return -1; - } - - if (dup) { - config->sql = strdup(sql); - if (config->sql == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } else { - config->sql = sql; - } - - return 0; -} - -void tsdbClearTableCfg(STableCfg *config) { - if (config) { - if (config->schema) tdFreeSchema(config->schema); - if (config->tagSchema) tdFreeSchema(config->tagSchema); - if (config->tagValues) kvRowFree(config->tagValues); - tfree(config->name); - tfree(config->sname); - tfree(config->sql); - free(config); - } -} - -static int tsdbEncodeTableName(void **buf, tstr *name) { - int tlen = 0; - - tlen += taosEncodeFixedI16(buf, name->len); - if (buf != NULL) { - memcpy(*buf, name->data, name->len); - *buf = POINTER_SHIFT(*buf, name->len); - } - tlen += name->len; - - return tlen; -} - -static void *tsdbDecodeTableName(void *buf, tstr **name) { - VarDataLenT len = 0; - - buf = taosDecodeFixedI16(buf, &len); - *name = calloc(1, sizeof(tstr) + len + 1); - if (*name == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - (*name)->len = len; - memcpy((*name)->data, buf, len); - - buf = POINTER_SHIFT(buf, len); - return buf; -} - -static int tsdbEncodeTable(void **buf, STable *pTable) { - ASSERT(pTable != NULL); - int tlen = 0; - - tlen += taosEncodeFixedU8(buf, pTable->type); - tlen += tsdbEncodeTableName(buf, pTable->name); - tlen += taosEncodeFixedU64(buf, TABLE_UID(pTable)); - tlen += taosEncodeFixedI32(buf, TABLE_TID(pTable)); - - if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { - tlen += taosEncodeFixedU64(buf, TABLE_SUID(pTable)); - tlen += tdEncodeKVRow(buf, pTable->tagVal); - } else { - uint32_t arraySize = (uint32_t)taosArrayGetSize(pTable->schema); - if(arraySize > UINT8_MAX) { - tlen += taosEncodeFixedU8(buf, 0); - tlen += taosEncodeFixedU32(buf, arraySize); - } else { - tlen += taosEncodeFixedU8(buf, (uint8_t)arraySize); - } - for (uint32_t i = 0; i < arraySize; i++) { - STSchema *pSchema = taosArrayGetP(pTable->schema, i); - tlen += tdEncodeSchema(buf, pSchema); - } - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tlen += tdEncodeSchema(buf, pTable->tagSchema); - } - - if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { - tlen += taosEncodeString(buf, pTable->sql); - } - } - - return tlen; -} - -static void *tsdbDecodeTable(void *buf, STable **pRTable) { - STable *pTable = tsdbNewTable(); - if (pTable == NULL) return NULL; - - uint8_t type = 0; - - buf = taosDecodeFixedU8(buf, &type); - pTable->type = type; - buf = tsdbDecodeTableName(buf, &(pTable->name)); - buf = taosDecodeFixedU64(buf, &TABLE_UID(pTable)); - buf = taosDecodeFixedI32(buf, &TABLE_TID(pTable)); - - if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { - buf = taosDecodeFixedU64(buf, &TABLE_SUID(pTable)); - buf = tdDecodeKVRow(buf, &(pTable->tagVal)); - } else { - uint32_t nSchemas = 0; - buf = taosDecodeFixedU8(buf, (uint8_t *)&nSchemas); - if(nSchemas == 0) { - buf = taosDecodeFixedU32(buf, &nSchemas); - } - for (int i = 0; i < nSchemas; i++) { - STSchema *pSchema; - buf = tdDecodeSchema(buf, &pSchema); - tsdbAddSchema(pTable, pSchema); - } - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - buf = tdDecodeSchema(buf, &(pTable->tagSchema)); - STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, - SL_ALLOW_DUP_KEY, getTagIndexKey); - if (pTable->pIndex == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbFreeTable(pTable); - return NULL; - } - } - - if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { - buf = taosDecodeString(buf, &(pTable->sql)); - } - } - - T_REF_INC(pTable); - - *pRTable = pTable; - - return buf; -} - -static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) { - int tlen = 0; - if (act == TSDB_UPDATE_META) { - tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM); - } else { - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_SIZE(pTable->pIndex) + 1)); - } else { - tlen = sizeof(SListNode) + sizeof(SActObj); - } - } - - return tlen; -} - -static void *tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STable *pTable) { - SListNode *pNode = (SListNode *)buf; - SActObj * pAct = (SActObj *)(pNode->data); - SActCont * pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(*pAct)); - void * pBuf = (void *)pCont; - - pNode->prev = pNode->next = NULL; - pAct->act = act; - pAct->uid = TABLE_UID(pTable); - - if (act == TSDB_UPDATE_META) { - pBuf = (void *)(pCont->cont); - pCont->len = tsdbEncodeTable(&pBuf, pTable) + sizeof(TSCKSUM); - taosCalcChecksumAppend(0, (uint8_t *)pCont->cont, pCont->len); - pBuf = POINTER_SHIFT(pBuf, sizeof(TSCKSUM)); - } - - tdListAppendNode(pRepo->mem->actList, pNode); - - return pBuf; -} - -static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable) { - int tlen = tsdbGetTableEncodeSize(TSDB_DROP_META, pTable); - void *buf = tsdbAllocBytes(pRepo, tlen); - if (buf == NULL) { - return -1; - } - - void *pBuf = buf; - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - SSkipListIterator *pIter = tSkipListCreateIter(pTable->pIndex); - if (pIter == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - while (tSkipListIterNext(pIter)) { - STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); - ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE); - pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, tTable); - } - - tSkipListDestroyIter(pIter); - } - pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, pTable); - - ASSERT(POINTER_DISTANCE(pBuf, buf) == tlen); - - return 0; -} - -static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) { - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - SSkipListIterator *pIter = tSkipListCreateIter(pTable->pIndex); - if (pIter == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - tsdbWLockRepoMeta(pRepo); - - while (tSkipListIterNext(pIter)) { - STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); - tsdbRemoveTableFromMeta(pRepo, tTable, false, false); - } - - tsdbRemoveTableFromMeta(pRepo, pTable, false, false); - - tsdbUnlockRepoMeta(pRepo); - - tSkipListDestroyIter(pIter); - - } else { - if ((TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) && pTable->cqhandle) pRepo->appH.cqDropFunc(pTable->cqhandle); - tsdbRemoveTableFromMeta(pRepo, pTable, true, true); - } - - return 0; -} - -static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - ASSERT(tid >= pMeta->maxTables); - - int maxTables = tsdbGetNextMaxTables(tid); - - STable **tables = (STable **)calloc(maxTables, sizeof(STable *)); - if (tables == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - memcpy((void *)tables, (void *)pMeta->tables, sizeof(STable *) * pMeta->maxTables); - pMeta->maxTables = maxTables; - - STable **tTables = pMeta->tables; - pMeta->tables = tables; - tfree(tTables); - tsdbDebug("vgId:%d tsdb meta maxTables is adjusted as %d", REPO_ID(pRepo), maxTables); - - return 0; -} - -static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema) { - for (size_t i = 0; i < kvRowNCols(pKVRow); i++) { - SColIdx * pColIdx = kvRowColIdxAt(pKVRow, i); - STColumn *pCol = tdGetColOfID(pSchema, pColIdx->colId); - - if ((pCol == NULL) || (!IS_VAR_DATA_TYPE(pCol->type))) continue; - - void *pValue = tdGetKVRowValOfCol(pKVRow, pCol->colId); - if (varDataTLen(pValue) > pCol->bytes) { - terrno = TSDB_CODE_TDB_IVLD_TAG_VAL; - return -1; - } - } - - return 0; -} - -static int tsdbAddSchema(STable *pTable, STSchema *pSchema) { - ASSERT(TABLE_TYPE(pTable) != TSDB_CHILD_TABLE); - - if (pTable->schema == NULL) { - pTable->schema = taosArrayInit(TSDB_MAX_TABLE_SCHEMAS, sizeof(SSchema *)); - if (pTable->schema == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - } - - ASSERT(taosArrayGetSize(pTable->schema) == 0 || - schemaVersion(pSchema) > schemaVersion(*(STSchema **)taosArrayGetLast(pTable->schema))); - - if (taosArrayPush(pTable->schema, &pSchema) == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return 0; -} - -static void tsdbFreeTableSchema(STable *pTable) { - ASSERT(pTable != NULL); - - if (pTable->schema) { - for (size_t i = 0; i < taosArrayGetSize(pTable->schema); i++) { - STSchema *pSchema = taosArrayGetP(pTable->schema, i); - tdFreeSchema(pSchema); - } - - taosArrayDestroy(pTable->schema); - } -} diff --git a/src/tsdb/src/tsdbRecover.c b/src/tsdb/src/tsdbRecover.c deleted file mode 100644 index 6dea4a4e57392be988126c579648f39a8270b9bf..0000000000000000000000000000000000000000 --- a/src/tsdb/src/tsdbRecover.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/src/tsdb/tests/CMakeLists.txt b/src/tsdb/tests/CMakeLists.txt deleted file mode 100644 index a3477aef95f9e54484c106ba2afb853b48456be0..0000000000000000000000000000000000000000 --- a/src/tsdb/tests/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - -add_executable(tsdbTests ${SOURCE_LIST}) -target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb tutil trpc) - -add_test(NAME unit COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests) \ No newline at end of file diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp deleted file mode 100644 index dc804856fdf1e288f2a70a2813639c1324f42851..0000000000000000000000000000000000000000 --- a/src/tsdb/tests/tsdbTests.cpp +++ /dev/null @@ -1,163 +0,0 @@ -#include -#include -#include - -#include "tsdb.h" -#include "tsdbMain.h" - -static double getCurTime() { - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec + tv.tv_usec * 1E-6; -} - -typedef struct { - STsdbRepo *pRepo; - bool isAscend; - int tid; - uint64_t uid; - int sversion; - TSKEY startTime; - TSKEY interval; - int totalRows; - int rowsPerSubmit; - STSchema * pSchema; -} SInsertInfo; - -static int insertData(SInsertInfo *pInfo) { - SSubmitMsg *pMsg = - (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + dataRowMaxBytesFromSchema(pInfo->pSchema) * pInfo->rowsPerSubmit); - if (pMsg == NULL) return -1; - TSKEY start_time = pInfo->startTime; - - // Loop to write data - double stime = getCurTime(); - - for (int k = 0; k < pInfo->totalRows/pInfo->rowsPerSubmit; k++) { - memset((void *)pMsg, 0, sizeof(SSubmitMsg)); - SSubmitBlk *pBlock = (SSubmitBlk *)pMsg->blocks; - pBlock->uid = pInfo->uid; - pBlock->tid = pInfo->tid; - pBlock->sversion = pInfo->sversion; - pBlock->dataLen = 0; - pBlock->schemaLen = 0; - pBlock->numOfRows = 0; - for (int i = 0; i < pInfo->rowsPerSubmit; i++) { - // start_time += 1000; - if (pInfo->isAscend) { - start_time += pInfo->interval; - } else { - start_time -= pInfo->interval; - } - SDataRow row = (SDataRow)(pBlock->data + pBlock->dataLen); - tdInitDataRow(row, pInfo->pSchema); - - for (int j = 0; j < schemaNCols(pInfo->pSchema); j++) { - STColumn *pTCol = schemaColAt(pInfo->pSchema, j); - if (j == 0) { // Just for timestamp - tdAppendColVal(row, (void *)(&start_time), pTCol->type, pTCol->offset); - } else { // For int - int val = 10; - tdAppendColVal(row, (void *)(&val), pTCol->type, pTCol->offset); - } - } - pBlock->dataLen += dataRowLen(row); - pBlock->numOfRows++; - } - pMsg->length = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + pBlock->dataLen; - pMsg->numOfBlocks = 1; - - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->numOfRows = htonl(pBlock->numOfRows); - pBlock->schemaLen = htonl(pBlock->schemaLen); - pBlock->uid = htobe64(pBlock->uid); - pBlock->tid = htonl(pBlock->tid); - - pBlock->sversion = htonl(pBlock->sversion); - pBlock->padding = htonl(pBlock->padding); - - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - - if (tsdbInsertData(pInfo->pRepo, pMsg, NULL) < 0) { - tfree(pMsg); - return -1; - } - } - - double etime = getCurTime(); - - printf("Spent %f seconds to write %d records\n", etime - stime, pInfo->totalRows); - tfree(pMsg); - return 0; -} - -static void tsdbSetCfg(STsdbCfg *pCfg, int32_t tsdbId, int32_t cacheBlockSize, int32_t totalBlocks, int32_t maxTables, - int32_t daysPerFile, int32_t keep, int32_t minRows, int32_t maxRows, int8_t precision, - int8_t compression) { - pCfg->tsdbId = tsdbId; - pCfg->cacheBlockSize = cacheBlockSize; - pCfg->totalBlocks = totalBlocks; - // pCfg->maxTables = maxTables; - pCfg->daysPerFile = daysPerFile; - pCfg->keep = keep; - pCfg->minRowsPerFileBlock = minRows; - pCfg->maxRowsPerFileBlock = maxRows; - pCfg->precision = precision; - pCfg->compression = compression; -} - -static void tsdbSetTableCfg(STableCfg *pCfg) { - STSchemaBuilder schemaBuilder = {0}; - - pCfg->type = TSDB_NORMAL_TABLE; - pCfg->superUid = TSDB_INVALID_SUPER_TABLE_ID; - pCfg->tableId.tid = 1; - pCfg->tableId.uid = 5849583783847394; - tdInitTSchemaBuilder(&schemaBuilder, 0); - - int colId = 0; - for (int i = 0; i < 5; i++) { - tdAddColToSchema(&schemaBuilder, (colId == 0) ? TSDB_DATA_TYPE_TIMESTAMP : TSDB_DATA_TYPE_INT, colId, 0); - colId++; - } - - pCfg->schema = tdGetSchemaFromBuilder(&schemaBuilder); - pCfg->name = strdup("t1"); - - tdDestroyTSchemaBuilder(&schemaBuilder); -} - -TEST(TsdbTest, testInsertSpeed) { - int vnode = 1; - int ret = 0; - STsdbCfg tsdbCfg; - STableCfg tableCfg; - std::string testDir = "./test"; - char * rootDir = strdup((testDir + "/vnode" + std::to_string(vnode)).c_str()); - - tsdbDebugFlag = 131; //NOTE: you must set the flag - - taosRemoveDir(rootDir); - - // Create and open repository - tsdbSetCfg(&tsdbCfg, 1, 16, 4, -1, -1, -1, -1, -1, -1, -1); - tsdbCreateRepo(rootDir, &tsdbCfg); - STsdbRepo *repo = tsdbOpenRepo(rootDir, NULL); - ASSERT_NE(repo, nullptr); - - // Create table - tsdbSetTableCfg(&tableCfg); - tsdbCreateTable(repo, &tableCfg); - - // Insert data - SInsertInfo iInfo = {repo, true, 1, 5849583783847394, 0, 1590000000000, 10, 10000000, 100, tableCfg.schema}; - - insertData(&iInfo); - - tsdbCloseRepo(repo, 1); -} - -static char *getTKey(const void *data) { - return (char *)data; -} \ No newline at end of file diff --git a/tests/script/general/user/monitor.sim b/tests/script/general/user/monitor.sim deleted file mode 100644 index 90aad59932115e04263da8cc1e8b91effb09eb07..0000000000000000000000000000000000000000 --- a/tests/script/general/user/monitor.sim +++ /dev/null @@ -1,32 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 - -print ========== step1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c monitor -v 1 -system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect - -print ========== step2 -#sql drop database log -x step21 -# return -1 -#step21: -sql drop table log.dn -x step22 -# return -1 -step22: -sql drop user log -x step23 -# return -1 -step23: - -print ========== step3 - -sleep 2000 -#sql select * from log.dn -#if $rows == 0 then -# return -1 -#endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 4dff6393798fb0c103048e18ab23b4f34cbff048..2408faefa0418b83ab7e1fb715e81ee4fa6a4caf 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -1,417 +1,7 @@ -cd ../../../debug; cmake .. -cd ../../../debug; make #======================b1-start=============== -./test.sh -f general/field/2.sim -./test.sh -f general/field/3.sim -./test.sh -f general/field/4.sim -./test.sh -f general/field/5.sim -./test.sh -f general/field/6.sim -./test.sh -f general/field/bigint.sim -./test.sh -f general/field/binary.sim -./test.sh -f general/field/bool.sim -./test.sh -f general/field/single.sim -./test.sh -f general/field/smallint.sim -./test.sh -f general/field/tinyint.sim +./test.sh -f general/user/basic1.sim -./test.sh -f general/http/autocreate.sim -./test.sh -f general/http/chunked.sim -./test.sh -f general/http/gzip.sim -./test.sh -f general/http/restful.sim -./test.sh -f general/http/restful_insert.sim -./test.sh -f general/http/restful_limit.sim -./test.sh -f general/http/restful_full.sim -./test.sh -f general/http/prepare.sim -./test.sh -f general/http/telegraf.sim -./test.sh -f general/http/grafana_bug.sim -./test.sh -f general/http/grafana.sim - -./test.sh -f general/insert/basic.sim -./test.sh -f general/insert/insert_drop.sim -./test.sh -f general/insert/query_block1_memory.sim -./test.sh -f general/insert/query_block2_memory.sim -./test.sh -f general/insert/query_block1_file.sim -./test.sh -f general/insert/query_block2_file.sim -./test.sh -f general/insert/query_file_memory.sim -./test.sh -f general/insert/query_multi_file.sim -./test.sh -f general/insert/tcp.sim - -./test.sh -f general/parser/alter.sim -./test.sh -f general/parser/alter1.sim -./test.sh -f general/parser/alter_stable.sim -./test.sh -f general/parser/auto_create_tb.sim -./test.sh -f general/parser/auto_create_tb_drop_tb.sim -./test.sh -f general/parser/col_arithmetic_operation.sim -./test.sh -f general/parser/columnValue.sim -./test.sh -f general/parser/commit.sim -./test.sh -f general/parser/create_db.sim -./test.sh -f general/parser/create_mt.sim -./test.sh -f general/parser/create_tb.sim -./test.sh -f general/parser/dbtbnameValidate.sim -./test.sh -f general/parser/import_commit1.sim -./test.sh -f general/parser/import_commit2.sim -./test.sh -f general/parser/import_commit3.sim -./test.sh -f general/parser/insert_tb.sim -./test.sh -f general/parser/first_last.sim -./test.sh -f general/parser/lastrow.sim -./test.sh -f general/parser/nchar.sim -./test.sh -f general/parser/null_char.sim -./test.sh -f general/parser/single_row_in_tb.sim -./test.sh -f general/parser/select_from_cache_disk.sim -./test.sh -f general/parser/mixed_blocks.sim -./test.sh -f general/parser/selectResNum.sim -./test.sh -f general/parser/limit.sim -./test.sh -f general/parser/limit1.sim -./test.sh -f general/parser/limit1_tblocks100.sim -./test.sh -f general/parser/select_across_vnodes.sim -./test.sh -f general/parser/slimit1.sim -./test.sh -f general/parser/tbnameIn.sim -./test.sh -f general/parser/projection_limit_offset.sim -./test.sh -f general/parser/limit2.sim -./test.sh -f general/parser/fill.sim -./test.sh -f general/parser/fill_stb.sim -./test.sh -f general/parser/where.sim -./test.sh -f general/parser/slimit.sim -./test.sh -f general/parser/select_with_tags.sim -./test.sh -f general/parser/interp.sim -./test.sh -f general/parser/tags_dynamically_specifiy.sim -./test.sh -f general/parser/groupby.sim -./test.sh -f general/parser/set_tag_vals.sim -./test.sh -f general/parser/tags_filter.sim -./test.sh -f general/parser/slimit_alter_tags.sim -./test.sh -f general/parser/join.sim -./test.sh -f general/parser/join_multivnode.sim -./test.sh -f general/parser/binary_escapeCharacter.sim -./test.sh -f general/parser/repeatAlter.sim -./test.sh -f general/parser/union.sim -./test.sh -f general/parser/topbot.sim -./test.sh -f general/db/nosuchfile.sim -./test.sh -f general/parser/function.sim -./test.sh -f unique/cluster/vgroup100.sim - -./test.sh -f unique/http/admin.sim -./test.sh -f unique/http/opentsdb.sim - -./test.sh -f unique/import/replica2.sim -./test.sh -f unique/import/replica3.sim - -./test.sh -f general/alter/cached_schema_after_alter.sim #======================b1-end=============== -#======================b2-start=============== - - -./test.sh -f general/wal/sync.sim -./test.sh -f general/wal/kill.sim -./test.sh -f general/wal/maxtables.sim - -./test.sh -f general/user/authority.sim -./test.sh -f general/user/monitor.sim -./test.sh -f general/user/pass_alter.sim -./test.sh -f general/user/pass_len.sim -./test.sh -f general/user/user_create.sim -./test.sh -f general/user/user_len.sim - -./test.sh -f general/vector/metrics_field.sim -./test.sh -f general/vector/metrics_mix.sim -./test.sh -f general/vector/metrics_query.sim -./test.sh -f general/vector/metrics_tag.sim -./test.sh -f general/vector/metrics_time.sim -./test.sh -f general/vector/multi.sim -./test.sh -f general/vector/single.sim -./test.sh -f general/vector/table_field.sim -./test.sh -f general/vector/table_mix.sim -./test.sh -f general/vector/table_query.sim -./test.sh -f general/vector/table_time.sim - -./test.sh -f unique/account/account_create.sim -./test.sh -f unique/account/account_delete.sim -./test.sh -f unique/account/account_len.sim -./test.sh -f unique/account/authority.sim -./test.sh -f unique/account/basic.sim -./test.sh -f unique/account/paras.sim -./test.sh -f unique/account/pass_alter.sim -./test.sh -f unique/account/pass_len.sim -./test.sh -f unique/account/usage.sim -./test.sh -f unique/account/user_create.sim -./test.sh -f unique/account/user_len.sim - -./test.sh -f unique/big/maxvnodes.sim -./test.sh -f unique/big/tcp.sim - -./test.sh -f unique/cluster/alter.sim -./test.sh -f unique/cluster/cache.sim -./test.sh -f unique/http/admin.sim -./test.sh -f unique/http/opentsdb.sim - -./test.sh -f unique/import/replica2.sim -./test.sh -f unique/import/replica3.sim - -./test.sh -f general/alter/cached_schema_after_alter.sim - - -#======================b2-end=============== -#======================b3-start=============== - -./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim -#./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim -./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim -./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim -#./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim -#./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim -./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim -./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim -#./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim -# lower the priority while file corruption -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim -./test.sh -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim -./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim -./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim - -./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim -./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim -./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim -./test.sh -f unique/arbitrator/offline_replica2_dropDb_online.sim -./test.sh -f unique/arbitrator/offline_replica2_dropTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_alterTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_alterTag_online.sim -./test.sh -f unique/arbitrator/offline_replica3_createTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_dropDb_online.sim -./test.sh -f unique/arbitrator/offline_replica3_dropTable_online.sim -./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim -./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim -./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim - -./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim -./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim -./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim -./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim -./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim -./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim - -./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim -./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim -./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim -./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim - -./test.sh -f unique/stable/balance_replica1.sim -./test.sh -f unique/stable/dnode2_stop.sim -./test.sh -f unique/stable/dnode2.sim -./test.sh -f unique/stable/dnode3.sim -./test.sh -f unique/stable/replica2_dnode4.sim -./test.sh -f unique/stable/replica2_vnode3.sim -./test.sh -f unique/stable/replica3_dnode6.sim -./test.sh -f unique/stable/replica3_vnode3.sim - -#======================b3-end=============== -#======================b4-start=============== - - -./test.sh -f general/alter/count.sim -./test.sh -f general/alter/dnode.sim -./test.sh -f general/alter/import.sim -./test.sh -f general/alter/insert1.sim -./test.sh -f general/alter/insert2.sim -./test.sh -f general/alter/metrics.sim -./test.sh -f general/alter/table.sim - -./test.sh -f general/cache/new_metrics.sim -./test.sh -f general/cache/restart_metrics.sim -./test.sh -f general/cache/restart_table.sim - -./test.sh -f general/connection/connection.sim - -./test.sh -f general/column/commit.sim -./test.sh -f general/column/metrics.sim -./test.sh -f general/column/table.sim - -./test.sh -f general/compress/commitlog.sim -./test.sh -f general/compress/compress.sim -./test.sh -f general/compress/compress2.sim -./test.sh -f general/compress/uncompress.sim - -./test.sh -f general/stable/disk.sim -./test.sh -f general/stable/dnode3.sim -./test.sh -f general/stable/metrics.sim -./test.sh -f general/stable/refcount.sim -./test.sh -f general/stable/show.sim -./test.sh -f general/stable/values.sim -./test.sh -f general/stable/vnode3.sim - -./test.sh -f unique/column/replica3.sim -./test.sh -f issue/TD-2713.sim -./test.sh -f general/parser/select_distinct_tag.sim -./test.sh -f unique/mnode/mgmt30.sim -./test.sh -f issue/TD-2677.sim -./test.sh -f issue/TD-2680.sim -./test.sh -f unique/dnode/lossdata.sim - -#======================b4-end=============== -#======================b5-start=============== - -./test.sh -f unique/dnode/alternativeRole.sim -./test.sh -f unique/dnode/balance1.sim -./test.sh -f unique/dnode/balance2.sim -./test.sh -f unique/dnode/balance3.sim -./test.sh -f unique/dnode/balancex.sim -./test.sh -f unique/dnode/offline1.sim -./test.sh -f unique/dnode/offline2.sim - -./test.sh -f general/stream/metrics_del.sim -./test.sh -f general/stream/metrics_replica1_vnoden.sim -./test.sh -f general/stream/restart_stream.sim -./test.sh -f general/stream/stream_3.sim -./test.sh -f general/stream/stream_restart.sim -./test.sh -f general/stream/table_del.sim -./test.sh -f general/stream/table_replica1_vnoden.sim - -./test.sh -f general/connection/test_old_data.sim -./test.sh -f unique/dnode/datatrans_3node.sim -./test.sh -f unique/dnode/datatrans_3node_2.sim -./test.sh -f general/db/alter_tables_d2.sim -./test.sh -f general/db/alter_tables_v1.sim -./test.sh -f general/db/alter_tables_v4.sim - -#======================b5-end=============== -#======================b6-start=============== - -./test.sh -f unique/dnode/reason.sim -./test.sh -f unique/dnode/remove1.sim -./test.sh -f unique/dnode/remove2.sim -./test.sh -f unique/dnode/vnode_clean.sim - -./test.sh -f unique/db/commit.sim -./test.sh -f unique/db/delete.sim -./test.sh -f unique/db/delete_part.sim -./test.sh -f unique/db/replica_add12.sim -./test.sh -f unique/db/replica_add13.sim -./test.sh -f unique/db/replica_add23.sim -./test.sh -f unique/db/replica_reduce21.sim -./test.sh -f unique/db/replica_reduce32.sim -./test.sh -f unique/db/replica_reduce31.sim -./test.sh -f unique/db/replica_part.sim - -./test.sh -f unique/vnode/many.sim -./test.sh -f unique/vnode/replica2_basic2.sim -./test.sh -f unique/vnode/replica2_repeat.sim -./test.sh -f unique/vnode/replica3_basic.sim -./test.sh -f unique/vnode/replica3_repeat.sim -./test.sh -f unique/vnode/replica3_vgroup.sim - -./test.sh -f unique/dnode/monitor.sim -./test.sh -f unique/dnode/monitor_bug.sim -./test.sh -f unique/dnode/simple.sim -./test.sh -f unique/dnode/data1.sim -./test.sh -f unique/dnode/m2.sim -./test.sh -f unique/dnode/m3.sim -./test.sh -f unique/dnode/offline3.sim -./test.sh -f general/wal/kill.sim -./test.sh -f general/wal/maxtables.sim - -./test.sh -f general/import/basic.sim -./test.sh -f general/import/commit.sim -./test.sh -f general/import/large.sim -./test.sh -f general/import/replica1.sim -./test.sh -f unique/cluster/balance1.sim -./test.sh -f unique/cluster/balance2.sim -./test.sh -f unique/cluster/balance3.sim - -#======================b6-end=============== -#======================b7-start=============== - -./test.sh -f general/compute/avg.sim -./test.sh -f general/compute/bottom.sim -./test.sh -f general/compute/count.sim -./test.sh -f general/compute/diff.sim -./test.sh -f general/compute/diff2.sim -./test.sh -f general/compute/first.sim -./test.sh -f general/compute/interval.sim -./test.sh -f general/compute/last.sim -./test.sh -f general/compute/leastsquare.sim -./test.sh -f general/compute/max.sim -./test.sh -f general/compute/min.sim -./test.sh -f general/compute/null.sim -./test.sh -f general/compute/percentile.sim -./test.sh -f general/compute/stddev.sim -./test.sh -f general/compute/sum.sim -./test.sh -f general/compute/top.sim - -./test.sh -f general/db/alter_option.sim -./test.sh -f general/db/alter_vgroups.sim -./test.sh -f general/db/basic.sim -./test.sh -f general/db/basic1.sim -./test.sh -f general/db/basic2.sim -./test.sh -f general/db/basic3.sim -./test.sh -f general/db/basic4.sim -./test.sh -f general/db/basic5.sim -./test.sh -f general/db/delete_reuse1.sim -./test.sh -f general/db/delete_reuse2.sim -./test.sh -f general/db/delete_reusevnode.sim -./test.sh -f general/db/delete_reusevnode2.sim -./test.sh -f general/db/delete_writing1.sim -./test.sh -f general/db/delete_writing2.sim -./test.sh -f general/db/delete.sim -./test.sh -f general/db/len.sim -./test.sh -f general/db/repeat.sim -./test.sh -f general/db/tables.sim -./test.sh -f general/db/vnodes.sim -./test.sh -f general/db/topic1.sim -./test.sh -f general/db/topic2.sim -./test.sh -f general/table/autocreate.sim -./test.sh -f general/table/basic1.sim -./test.sh -f general/table/basic2.sim -./test.sh -f general/table/basic3.sim -./test.sh -f general/table/bigint.sim -./test.sh -f general/table/binary.sim -./test.sh -f general/table/bool.sim -./test.sh -f general/table/column_name.sim -./test.sh -f general/table/column_num.sim -./test.sh -f general/table/column_value.sim -./test.sh -f general/table/column2.sim -./test.sh -f general/table/date.sim -./test.sh -f general/table/db.table.sim -./test.sh -f general/table/delete_reuse1.sim -./test.sh -f general/table/delete_reuse2.sim -./test.sh -f general/table/delete_writing.sim -./test.sh -f general/table/describe.sim -./test.sh -f general/table/double.sim -./test.sh -f general/table/float.sim -./test.sh -f general/table/int.sim -./test.sh -f general/table/limit.sim -./test.sh -f general/table/smallint.sim -./test.sh -f general/table/table_len.sim -./test.sh -f general/table/table.sim -./test.sh -f general/table/tinyint.sim -./test.sh -f general/table/vgroup.sim -./test.sh -f general/table/createmulti.sim - -./test.sh -f unique/mnode/mgmt20.sim -./test.sh -f unique/mnode/mgmt21.sim -./test.sh -f unique/mnode/mgmt22.sim -./test.sh -f unique/mnode/mgmt23.sim -./test.sh -f unique/mnode/mgmt24.sim -./test.sh -f unique/mnode/mgmt25.sim -./test.sh -f unique/mnode/mgmt26.sim -./test.sh -f unique/mnode/mgmt33.sim -./test.sh -f unique/mnode/mgmt34.sim -./test.sh -f unique/mnode/mgmtr2.sim - -./test.sh -f unique/arbitrator/insert_duplicationTs.sim -./test.sh -f general/parser/join_manyblocks.sim -./test.sh -f general/parser/stableOp.sim -./test.sh -f general/parser/timestamp.sim -./test.sh -f general/parser/sliding.sim -./test.sh -f general/parser/having.sim -./test.sh -f general/parser/having_child.sim -./test.sh -f general/parser/between_and.sim -./test.sh -f general/parser/last_cache.sim -./test.sh -f unique/big/balance.sim - -#======================b7-end=============== diff --git a/tests/test-all.sh b/tests/test-all.sh index eea623b27e482d67e0d3e94a27c7f4376449d556..29aecd2f0880744fdc1a2f026d3dbbf7f32d9ff8 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -41,7 +41,7 @@ function dohavecore(){ else cd ../../ if [[ $1 == 1 ]];then - tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug/build/bin/taosd debug/build/bin/tsim debug/build/lib/libtaos*so* + #tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` fi fi @@ -67,8 +67,8 @@ function runSimCaseOneByOne { else echo -n $case ./test.sh -f $case > /dev/null 2>&1 && \ - ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ - ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ( grep -q 'script.*success.*m$' ../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ echo -e "${RED} failed${NC}" | tee -a out.log fi out_log=`tail -1 out.log ` @@ -99,11 +99,12 @@ function runSimCaseOneByOnefq { ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) else + pwd echo -n $case ./test.sh -f $case > ../../sim/case.log 2>&1 && \ ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ - ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) + ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && pwd && cat ../../sim/case.log ) fi out_log=`tail -1 out.log `