diff --git a/Jenkinsfile b/Jenkinsfile index 5faa02d0c3162c9b65aa1e9d5eac44de5118a582..c47d37a9a072d707eabc45cfa7099b7342b5c184 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -106,6 +106,10 @@ pipeline { abortPreviousBuilds() } pre_test() + sh''' + cd ${WKC}/tests + ./test-all.sh b1fq + ''' } } // stage('Parallel test stage') { diff --git a/cmake/stub_CMakeLists.txt.in b/cmake/stub_CMakeLists.txt.in new file mode 100644 index 0000000000000000000000000000000000000000..e227e820d639865c9b440e3137b3b98f2cf7aade --- /dev/null +++ b/cmake/stub_CMakeLists.txt.in @@ -0,0 +1,12 @@ + +# stub +ExternalProject_Add(stub + GIT_REPOSITORY https://github.com/coolxv/cpp-stub.git + GIT_SUBMODULES "src" + SOURCE_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub" + BINARY_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub/src" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index dfdde96172d30d7773e5841c1c261e7d52a87028..c08f894fe7e8f7dc83a0269ba606607974e4f1bd 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -12,6 +12,7 @@ configure_file("${CMAKE_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE} # googletest if(${BUILD_TEST}) cat("${CMAKE_SUPPORT_DIR}/gtest_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) + cat("${CMAKE_SUPPORT_DIR}/stub_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_TEST}) # lz4 @@ -79,6 +80,11 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build . # googletest if(${BUILD_TEST}) add_subdirectory(googletest) + target_include_directories( + gtest + PUBLIC $ + PUBLIC $ + ) endif(${BUILD_TEST}) # cJson diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index a788df2e32306e7c1fc5ca97e42bcc78a7acb4b5..cf5c5c5637c26d76f181a626990ad12e65f3ffbe 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -51,7 +51,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET_CUR, "mq-set-cur" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RSP_READY, "rsp-ready" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RES_READY, "res-ready" ) // message from client to mnode TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" ) @@ -78,8 +78,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB, "create-stb" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB, "alter-stb" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB, "drop-stb" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_VGROUP_LIST, "vgroup-list" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_STREAM, "kill-stream" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_HEARTBEAT, "heartbeat" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW, "show" ) @@ -94,22 +93,22 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" ) // message from vnode to dnode // message from mnode to vnode -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB_IN, "create-stb-in" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB_IN, "alter-stb-in" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB_IN, "drop-stb-in" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB_IN, "create-stb-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB_IN, "alter-stb-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB_IN, "drop-stb-internal" ) // message from mnode to mnode // message from mnode to qnode // message from mnode to dnode -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_VNODE_IN, "create-vnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_VNODE_IN, "alter-vnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_VNODE_IN, "drop-vnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_AUTH_VNODE_IN, "auth-vnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SYNC_VNODE_IN, "sync-vnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_VNODE_IN, "compact-vnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_MNODE_IN, "create-mnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_MNODE_IN, "alter-mnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_MNODE_IN, "drop-mnode" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONFIG_DNODE_IN, "config-dnode" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_VNODE_IN, "create-vnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_VNODE_IN, "alter-vnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_VNODE_IN, "drop-vnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_AUTH_VNODE_IN, "auth-vnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SYNC_VNODE_IN, "sync-vnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_VNODE_IN, "compact-vnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_MNODE_IN, "create-mnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_MNODE_IN, "alter-mnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_MNODE_IN, "drop-mnode-internal" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONFIG_DNODE_IN, "config-dnode-internal" ) // message from qnode to vnode // message from qnode to mnode @@ -312,11 +311,11 @@ typedef struct { typedef struct { int32_t len; // one create table message char tableName[TSDB_TABLE_FNAME_LEN]; - int8_t igExists; - int8_t getMeta; int16_t numOfTags; int16_t numOfColumns; int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string + int8_t igExists; + int8_t rspMeta; int8_t reserved[16]; char schema[]; } SCreateTableMsg; @@ -341,16 +340,37 @@ typedef struct { } SAlterStbMsg; typedef struct { - char tableFname[TSDB_TABLE_FNAME_LEN]; - char db[TSDB_FULL_DB_NAME_LEN]; - int16_t type; /* operation type */ - int16_t numOfCols; /* number of schema */ - int32_t tagValLen; - SSchema schema[]; - // tagVal is padded after schema - // char tagVal[]; + SMsgHead head; + char name[TSDB_TABLE_FNAME_LEN]; + uint64_t suid; + int32_t sverson; + uint32_t ttl; + uint32_t keep; + int32_t numOfTags; + int32_t numOfColumns; + SSchema pSchema[]; +} SCreateStbInternalMsg; + +typedef struct { + SMsgHead head; + char name[TSDB_TABLE_FNAME_LEN]; + uint64_t suid; +} SDropStbInternalMsg; + +typedef struct { + SMsgHead head; + char name[TSDB_TABLE_FNAME_LEN]; + int8_t type; /* operation type */ + int32_t numOfCols; /* number of schema */ + int32_t numOfTags; + char data[]; } SAlterTableMsg; +typedef struct { + SMsgHead head; + char name[TSDB_TABLE_FNAME_LEN]; +} SDropTableMsg; + typedef struct { SMsgHead head; int64_t uid; @@ -591,8 +611,8 @@ typedef struct { int32_t daysToKeep0; int32_t daysToKeep1; int32_t daysToKeep2; - int32_t minRowsPerFileBlock; - int32_t maxRowsPerFileBlock; + int32_t minRows; + int32_t maxRows; int32_t commitTime; int32_t fsyncPeriod; int8_t walLevel; @@ -706,7 +726,7 @@ typedef struct { SVnodeLoad data[]; } SVnodeLoads; -typedef struct SStatusMsg { +typedef struct { int32_t sver; int32_t dnodeId; int32_t clusterId; @@ -756,6 +776,7 @@ typedef struct { int32_t dnodeId; char db[TSDB_FULL_DB_NAME_LEN]; uint64_t dbUid; + int32_t vgVersion; int32_t cacheBlockSize; int32_t totalBlocks; int32_t daysPerFile; @@ -942,18 +963,6 @@ typedef struct { char subSqlInfo[TSDB_SHOW_SUBQUERY_LEN]; // include subqueries' index, Obj IDs and states(C-complete/I-imcomplete) } SQueryDesc; -typedef struct { - char sql[TSDB_SHOW_SQL_LEN]; - char dstTable[TSDB_TABLE_NAME_LEN]; - int32_t streamId; - int64_t num; // number of computing/cycles - int64_t useconds; - int64_t ctime; - int64_t stime; - int64_t slidingTime; - int64_t interval; -} SStreamDesc; - typedef struct { int32_t connId; int32_t pid; @@ -1090,6 +1099,7 @@ typedef struct { } SUpdateTagValRsp; typedef struct SSchedulerQueryMsg { + uint64_t schedulerId; uint64_t queryId; uint64_t taskId; uint32_t contentLen; @@ -1097,15 +1107,39 @@ typedef struct SSchedulerQueryMsg { } SSchedulerQueryMsg; typedef struct SSchedulerReadyMsg { + uint64_t schedulerId; uint64_t queryId; uint64_t taskId; } SSchedulerReadyMsg; typedef struct SSchedulerFetchMsg { + uint64_t schedulerId; uint64_t queryId; uint64_t taskId; } SSchedulerFetchMsg; +typedef struct SSchedulerStatusMsg { + uint64_t schedulerId; +} SSchedulerStatusMsg; + +typedef struct STaskStatus { + uint64_t queryId; + uint64_t taskId; + int8_t status; +} STaskStatus; + +typedef struct SSchedulerStatusRsp { + uint32_t num; + STaskStatus status[]; +} SSchedulerStatusRsp; + + +typedef struct SSchedulerCancelMsg { + uint64_t schedulerId; + uint64_t queryId; + uint64_t taskId; +} SSchedulerCancelMsg; + #pragma pack(pop) diff --git a/include/common/tmsgtype.h b/include/common/tmsgtype.h index 1fb10ae15b692d21f8a07937393399fe49664ce3..8e7ad87a0ac9184e18a6e62837bbf821192b1ec0 100644 --- a/include/common/tmsgtype.h +++ b/include/common/tmsgtype.h @@ -35,7 +35,7 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SELECT, "select" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_FETCH, "fetch" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_INSERT, "insert" ) - TSDB_DEFINE_SQL_TYPE( TSDB_SQL_UPDATE_TAGS_VAL, "update-tag-val" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_UPDATE_TAG_VAL, "update-tag-val" ) // the SQL below is for mgmt node TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" ) @@ -54,7 +54,7 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_TABLE, "alter-table" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_DB, "alter-db" ) - TSDB_DEFINE_SQL_TYPE(TSDB_SQL_SYNC_DB_REPLICA, "sync db-replica") + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SYNC_DB_REPLICA, "sync db-replica") TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_MNODE, "create-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_MNODE, "drop-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DNODE, "create-dnode" ) @@ -102,14 +102,6 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MAX, "max" ) }; -// create table operation type -enum TSQL_CREATE_TABLE_TYPE { - TSQL_CREATE_TABLE = 0x1, - TSQL_CREATE_STABLE = 0x2, - TSQL_CREATE_CTABLE = 0x3, - TSQL_CREATE_STREAM = 0x4, -}; - #ifdef __cplusplus } #endif diff --git a/include/common/tname.h b/include/common/tname.h index e31bfd38a67dd5e01e6a3246e490d998bf50331f..de9e309b5508aeb682c09d6df4766700900fdb16 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -16,8 +16,6 @@ #ifndef TDENGINE_TNAME_H #define TDENGINE_TNAME_H -//#include "taosmsg.h" - #define TSDB_DB_NAME_T 1 #define TSDB_TABLE_NAME_T 2 diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index 65779b209938bc9766f2d0d11881c2354d0c5092..aa4c3af392dfb2e94198d0062801fe405624752c 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif - +#include "trpc.h" typedef struct { uint64_t numOfStartTask; @@ -32,48 +32,6 @@ typedef struct { uint64_t numOfErrors; } SQnodeStat; -/* start Task msg */ -typedef struct { - uint32_t schedulerIp; - uint16_t schedulerPort; - int64_t taskId; - int64_t queryId; - uint32_t srcIp; - uint16_t srcPort; -} SQnodeStartTaskMsg; - -/* stop Task msg */ -typedef struct { - int64_t taskId; -} SQnodeStopTaskMsg; - -/* start/stop Task msg response */ -typedef struct { - int64_t taskId; - int32_t code; -} SQnodeTaskRespMsg; - -/* Task status msg */ -typedef struct { - int64_t taskId; - int32_t status; - int64_t queryId; -} SQnodeTaskStatusMsg; - -/* Qnode/Scheduler heartbeat msg */ -typedef struct { - int32_t status; - int32_t load; - -} SQnodeHeartbeatMsg; - -/* Qnode sent/received msg */ -typedef struct { - int8_t msgType; - int32_t msgLen; - char msg[]; -} SQnodeMsg; - /** * Start one Qnode in Dnode. diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index a373828ebddf7d365d470bdcfccd49af8b4ac4e7..e708d7b545ac2f644755d7345e0af943bdee77ca 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -142,6 +142,36 @@ int vnodeProcessCMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); */ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +/** + * @brief Process a query message. + * + * @param pVnode The vnode object. + * @param pMsg The request message + * @param pRsp The response message + * @return int 0 for success, -1 for failure + */ +int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); + +/** + * @brief Process a fetch message. + * + * @param pVnode The vnode object. + * @param pMsg The request message + * @param pRsp The response message + * @return int 0 for success, -1 for failure + */ +int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); + +/** + * @brief Process a consume message. + * + * @param pVnode The vnode object. + * @param pMsg The request message + * @param pRsp The response message + * @return int 0 for success, -1 for failure + */ +int vnodeProcessConsumeReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); + /* ------------------------ SVnodeCfg ------------------------ */ /** * @brief Initialize VNODE options. @@ -180,73 +210,52 @@ typedef struct { char info[]; } SVnodeRsp; -#define VNODE_INIT_CREATE_STB_REQ(NAME, TTL, KEEP, SUID, PSCHEMA, PTAGSCHEMA) \ - { .ver = 0, .ctReq = META_INIT_STB_CFG(NAME, TTL, KEEP, SUID, PSCHEMA, PTAGSCHEMA) } +static FORCE_INLINE void vnodeSetCreateStbReq(SVnodeReq *pReq, char *name, uint32_t ttl, uint32_t keep, tb_uid_t suid, + STSchema *pSchema, STSchema *pTagSchema) { + pReq->ver = 0; + + pReq->ctReq.name = name; + pReq->ctReq.ttl = ttl; + pReq->ctReq.keep = keep; + pReq->ctReq.type = META_SUPER_TABLE; + pReq->ctReq.stbCfg.suid = suid; + pReq->ctReq.stbCfg.pSchema = pSchema; + pReq->ctReq.stbCfg.pTagSchema = pTagSchema; +} + +static FORCE_INLINE void vnodeSetCreateCtbReq(SVnodeReq *pReq, char *name, uint32_t ttl, uint32_t keep, tb_uid_t suid, + SKVRow pTag) { + pReq->ver = 0; -#define VNODE_INIT_CREATE_CTB_REQ(NAME, TTL, KEEP, SUID, PTAG) \ - { .ver = 0, .ctReq = META_INIT_CTB_CFG(NAME, TTL, KEEP, SUID, PTAG) } + pReq->ctReq.name = name; + pReq->ctReq.ttl = ttl; + pReq->ctReq.keep = keep; + pReq->ctReq.type = META_CHILD_TABLE; + pReq->ctReq.ctbCfg.suid = suid; + pReq->ctReq.ctbCfg.pTag = pTag; +} -#define VNODE_INIT_CREATE_NTB_REQ(NAME, TTL, KEEP, SUID, PSCHEMA) \ - { .ver = 0, .ctReq = META_INIT_NTB_CFG(NAME, TTL, KEEP, SUID, PSCHEMA) } +static FORCE_INLINE void vnodeSetCreateNtbReq(SVnodeReq *pReq, char *name, uint32_t ttl, uint32_t keep, + STSchema *pSchema) { + pReq->ver = 0; + + pReq->ctReq.name = name; + pReq->ctReq.ttl = ttl; + pReq->ctReq.keep = keep; + pReq->ctReq.type = META_NORMAL_TABLE; + pReq->ctReq.ntbCfg.pSchema = pSchema; +} int vnodeBuildReq(void **buf, const SVnodeReq *pReq, uint8_t type); void *vnodeParseReq(void *buf, SVnodeReq *pReq, uint8_t type); /* ------------------------ FOR COMPILE ------------------------ */ -#if 1 - -#include "taosmsg.h" -#include "trpc.h" - -// typedef struct { -// char db[TSDB_FULL_DB_NAME_LEN]; -// int32_t cacheBlockSize; // MB -// int32_t totalBlocks; -// int32_t daysPerFile; -// int32_t daysToKeep0; -// int32_t daysToKeep1; -// int32_t daysToKeep2; -// int32_t minRowsPerFileBlock; -// int32_t maxRowsPerFileBlock; -// int8_t precision; // time resolution -// int8_t compression; -// int8_t cacheLastRow; -// int8_t update; -// int8_t quorum; -// int8_t replica; -// int8_t selfIndex; -// int8_t walLevel; -// int32_t fsyncPeriod; // millisecond -// SReplica replicas[TSDB_MAX_REPLICA]; -// } SVnodeCfg; - -typedef enum { - VN_MSG_TYPE_WRITE = 1, - VN_MSG_TYPE_APPLY, - VN_MSG_TYPE_SYNC, - VN_MSG_TYPE_QUERY, - VN_MSG_TYPE_FETCH -} EVnMsgType; - -typedef struct { - int32_t curNum; - int32_t allocNum; - SRpcMsg rpcMsg[]; -} SVnodeMsg; - int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg); int32_t vnodeCompact(SVnode *pVnode); int32_t vnodeSync(SVnode *pVnode); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); -SVnodeMsg *vnodeInitMsg(int32_t msgNum); -int32_t vnodeAppendMsg(SVnodeMsg *pMsg, SRpcMsg *pRpcMsg); -void vnodeCleanupMsg(SVnodeMsg *pMsg); -void vnodeProcessMsg(SVnode *pVnode, SVnodeMsg *pMsg, EVnMsgType msgType); - -#endif - #ifdef __cplusplus } #endif diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 2535ec8a5b31f460dc19befb8274652a75942112..f93c46da0c412e3529dd0c016ef7c42dc8de1686 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -23,63 +23,67 @@ extern "C" { #endif -typedef struct SIndex SIndex; -typedef struct SIndexTerm SIndexTerm; -typedef struct SIndexOpts SIndexOpts; +typedef struct SIndex SIndex; +typedef struct SIndexTerm SIndexTerm; +typedef struct SIndexOpts SIndexOpts; typedef struct SIndexMultiTermQuery SIndexMultiTermQuery; typedef struct SArray SIndexMultiTerm; -typedef enum { - ADD_VALUE, // add index colume value - DEL_VALUE, // delete index column value - UPDATE_VALUE, // update index column value - ADD_INDEX, // add index on specify column - DROP_INDEX, // drop existed index - DROP_SATBLE // drop stable +typedef enum { + ADD_VALUE, // add index colume value + DEL_VALUE, // delete index column value + UPDATE_VALUE, // update index column value + ADD_INDEX, // add index on specify column + DROP_INDEX, // drop existed index + DROP_SATBLE // drop stable } SIndexOperOnColumn; -typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; -typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = 3} EIndexQueryType; +typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; +typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3 } EIndexQueryType; /* - * @param: oper + * @param: oper * -*/ -SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType oper); -void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery); -int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type); -/* - * @param: + */ +SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType oper); +void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery); +int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType type); +/* + * @param: * @param: */ -int indexOpen(SIndexOpts *opt, const char *path, SIndex **index); -void indexClose(SIndex *index); -int indexPut(SIndex *index, SIndexMultiTerm *terms, uint64_t uid); -int indexDelete(SIndex *index, SIndexMultiTermQuery *query); -int indexSearch(SIndex *index, SIndexMultiTermQuery *query, SArray *result); -int indexRebuild(SIndex *index, SIndexOpts *opt); +int indexOpen(SIndexOpts* opt, const char* path, SIndex** index); +void indexClose(SIndex* index); +int indexPut(SIndex* index, SIndexMultiTerm* terms, uint64_t uid); +int indexDelete(SIndex* index, SIndexMultiTermQuery* query); +int indexSearch(SIndex* index, SIndexMultiTermQuery* query, SArray* result); +int indexRebuild(SIndex* index, SIndexOpts* opt); /* * @param * @param */ -SIndexMultiTerm *indexMultiTermCreate(); -int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term); -void indexMultiTermDestroy(SIndexMultiTerm *terms); +SIndexMultiTerm* indexMultiTermCreate(); +int indexMultiTermAdd(SIndexMultiTerm* terms, SIndexTerm* term); +void indexMultiTermDestroy(SIndexMultiTerm* terms); /* - * @param: + * @param: * @param: */ -SIndexOpts *indexOptsCreate(); -void indexOptsDestroy(SIndexOpts *opts); +SIndexOpts* indexOptsCreate(); +void indexOptsDestroy(SIndexOpts* opts); /* * @param: * @param: */ -SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, - const char *colName, int32_t nColName, const char *colVal, int32_t nColVal); -void indexTermDestroy(SIndexTerm *p); - +SIndexTerm* indexTermCreate(int64_t suid, + SIndexOperOnColumn operType, + uint8_t colType, + const char* colName, + int32_t nColName, + const char* colVal, + int32_t nColVal); +void indexTermDestroy(SIndexTerm* p); #ifdef __cplusplus } diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h new file mode 100644 index 0000000000000000000000000000000000000000..62f36abcee9c45c0c254848c57e8d24a108999ba --- /dev/null +++ b/include/libs/parser/parsenodes.h @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_PARSENODES_H_ +#define _TD_PARSENODES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "catalog.h" +#include "common.h" +#include "function.h" +#include "tmsgtype.h" +#include "tname.h" +#include "tvariant.h" + +/* + * The first field of a node of any type is guaranteed to be the int16_t. + * Hence the type of any node can be gotten by casting it to SQueryNode. + */ +typedef struct SQueryNode { + int16_t type; +} SQueryNode; + +#define nodeType(nodeptr) (((const SQueryNode*)(nodeptr))->type) + +typedef struct SField { + char name[TSDB_COL_NAME_LEN]; + uint8_t type; + int16_t bytes; +} SField; + +typedef struct SParseBasicCtx { + const char *db; + int32_t acctId; + uint64_t requestId; +} SParseBasicCtx; + +typedef struct SFieldInfo { + int16_t numOfOutput; // number of column in result + SField *final; + SArray *internalField; // SArray +} SFieldInfo; + +typedef struct SCond { + uint64_t uid; + int32_t len; // length of tag query condition data + char * cond; +} SCond; + +typedef struct SJoinNode { + uint64_t uid; + int16_t tagColId; + SArray* tsJoin; + SArray* tagJoin; +} SJoinNode; + +typedef struct SJoinInfo { + bool hasJoin; + SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM]; +} SJoinInfo; + +typedef struct STagCond { + int16_t relType; // relation between tbname list and query condition, including : TK_AND or TK_OR + SCond tbnameCond; // tbname query condition, only support tbname query condition on one table + SJoinInfo joinInfo; // join condition, only support two tables join currently + SArray *pCond; // for different table, the query condition must be seperated +} STagCond; + +typedef struct STableMetaInfo { + STableMeta *pTableMeta; // table meta, cached in client side and acquired by name + SVgroupsInfo *vgroupList; + SName name; + char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql + SArray *tagColList; // SArray, involved tag columns +} STableMetaInfo; + +typedef struct SColumnIndex { + int16_t tableIndex; + int16_t columnIndex; + int16_t type; // normal column/tag/ user input constant column +} SColumnIndex; + +// select statement +typedef struct SQueryStmtInfo { + int16_t command; // the command may be different for each subclause, so keep it seperately. + uint32_t type; // query/insert type + STimeWindow window; // the whole query time window + SInterval interval; // tumble time window + SSessionWindow sessionWindow; // session time window + SStateWindow stateWindow; // state window query + SGroupbyExpr groupbyExpr; // groupby tags info + SArray * colList; // SArray + SFieldInfo fieldsInfo; + SArray** exprList; // SArray + SLimit limit; + SLimit slimit; + STagCond tagCond; + SArray * colCond; + SArray * order; + int16_t numOfTables; + int16_t curTableIdx; + STableMetaInfo **pTableMetaInfo; + struct STSBuf *tsBuf; + + int16_t fillType; // final result fill type + int64_t * fillVal; // default value for fill + int32_t numOfFillVal; // fill value size + + char * msg; // pointer to the pCmd->payload to keep error message temporarily + int64_t clauseLimit; // limit for current sub clause + + int64_t prjOffset; // offset value in the original sql expression, only applied at client side + int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit + + int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX + int32_t bufLen; + char* buf; + SArray *pUdfInfo; + + struct SQueryStmtInfo *sibling; // sibling + struct SQueryStmtInfo *pDownstream; + SMultiFunctionsDesc info; + SArray *pUpstream; // SArray + int32_t havingFieldNum; + int32_t exprListLevelIndex; +} SQueryStmtInfo; + +typedef enum { + PAYLOAD_TYPE_KV = 0, + PAYLOAD_TYPE_RAW = 1, +} EPayloadType; + +typedef struct SVgDataBlocks { + SVgroupInfo vg; + int32_t numOfTables; // number of tables in current submit block + uint32_t size; + char *pData; // SMsgDesc + SSubmitMsg + SSubmitBlk + ... +} SVgDataBlocks; + +typedef struct SInsertStmtInfo { + int16_t nodeType; + SArray* pDataBlocks; // data block for each vgroup, SArray. + int8_t schemaAttache; // denote if submit block is built with table schema or not + uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert + uint32_t insertType; // insert data from [file|sql statement| bound statement] + const char* sql; // current sql statement position +} SInsertStmtInfo; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_PARSENODES_H_*/ diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 8e26fa98c945973bc6ef90aa262f2b9be20fe2b8..03750a16a9e8cb3bce6acafe91d0305fff3ed49e 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -20,108 +20,7 @@ extern "C" { #endif -#include "catalog.h" -#include "common.h" -#include "tname.h" -#include "tvariant.h" -#include "function.h" - -typedef struct SField { - char name[TSDB_COL_NAME_LEN]; - uint8_t type; - int16_t bytes; -} SField; - -typedef struct SFieldInfo { - int16_t numOfOutput; // number of column in result - SField *final; - SArray *internalField; // SArray -} SFieldInfo; - -typedef struct SCond { - uint64_t uid; - int32_t len; // length of tag query condition data - char * cond; -} SCond; - -typedef struct SJoinNode { - uint64_t uid; - int16_t tagColId; - SArray* tsJoin; - SArray* tagJoin; -} SJoinNode; - -typedef struct SJoinInfo { - bool hasJoin; - SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM]; -} SJoinInfo; - -typedef struct STagCond { - int16_t relType; // relation between tbname list and query condition, including : TK_AND or TK_OR - SCond tbnameCond; // tbname query condition, only support tbname query condition on one table - SJoinInfo joinInfo; // join condition, only support two tables join currently - SArray *pCond; // for different table, the query condition must be seperated -} STagCond; - -typedef struct STableMetaInfo { - STableMeta *pTableMeta; // table meta, cached in client side and acquired by name - SVgroupsInfo *vgroupList; - SName name; - char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql - SArray *tagColList; // SArray, involved tag columns -} STableMetaInfo; - -typedef struct SQueryStmtInfo { - int16_t command; // the command may be different for each subclause, so keep it seperately. - uint32_t type; // query/insert type - STimeWindow window; // the whole query time window - SInterval interval; // tumble time window - SSessionWindow sessionWindow; // session time window - SStateWindow stateWindow; // state window query - SGroupbyExpr groupbyExpr; // groupby tags info - SArray * colList; // SArray - SFieldInfo fieldsInfo; - SArray** exprList; // SArray - SLimit limit; - SLimit slimit; - STagCond tagCond; - SArray * colCond; - SArray * order; - int16_t numOfTables; - int16_t curTableIdx; - STableMetaInfo **pTableMetaInfo; - struct STSBuf *tsBuf; - - int16_t fillType; // final result fill type - int64_t * fillVal; // default value for fill - int32_t numOfFillVal; // fill value size - - char * msg; // pointer to the pCmd->payload to keep error message temporarily - int64_t clauseLimit; // limit for current sub clause - - int64_t prjOffset; // offset value in the original sql expression, only applied at client side - int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit - - int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX - int32_t bufLen; - char* buf; - SArray *pUdfInfo; - - struct SQueryStmtInfo *sibling; // sibling - struct SQueryStmtInfo *pDownstream; - SMultiFunctionsDesc info; - SArray *pUpstream; // SArray - int32_t havingFieldNum; - int32_t exprListLevelIndex; -} SQueryStmtInfo; - -typedef struct SColumnIndex { - int16_t tableIndex; - int16_t columnIndex; - int16_t type; // normal column/tag/ user input constant column -} SColumnIndex; - -struct SInsertStmtInfo; +#include "parsenodes.h" /** * True will be returned if the input sql string is insert, false otherwise. @@ -132,17 +31,16 @@ struct SInsertStmtInfo; bool qIsInsertSql(const char* pStr, size_t length); typedef struct SParseContext { - const char* pAcctId; - const char* pDbname; - void *pRpc; - const char* pClusterId; - const SEpSet* pEpSet; - int64_t id; // query id, generated by uuid generator - int8_t schemaAttached; // denote if submit block is built with table schema or not - const char* pSql; // sql string - size_t sqlLen; // length of the sql string - char* pMsg; // extended error message if exists to help avoid the problem in sql statement. - int32_t msgLen; // max length of the msg + SParseBasicCtx ctx; + void *pRpc; + struct SCatalog *pCatalog; + const SEpSet *pEpSet; + int64_t id; // query id, generated by uuid generator + int8_t schemaAttached; // denote if submit block is built with table schema or not + const char *pSql; // sql string + size_t sqlLen; // length of the sql string + char *pMsg; // extended error message if exists to help avoid the problem in sql statement. + int32_t msgLen; // max length of the msg } SParseContext; /** @@ -153,27 +51,7 @@ typedef struct SParseContext { * @param msg extended error message if exists. * @return error code */ -int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t* type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen); - -typedef enum { - PAYLOAD_TYPE_KV = 0, - PAYLOAD_TYPE_RAW = 1, -} EPayloadType; - -typedef struct SVgDataBlocks { - int64_t vgId; // virtual group id - int32_t numOfTables; // number of tables in current submit block - uint32_t size; - char *pData; // SMsgDesc + SSubmitMsg + SSubmitBlk + ... -} SVgDataBlocks; - -typedef struct SInsertStmtInfo { - SArray* pDataBlocks; // data block for each vgroup, SArray. - int8_t schemaAttache; // denote if submit block is built with table schema or not - uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert - uint32_t insertType; // insert data from [file|sql statement| bound statement] - const char* sql; // current sql statement position -} SInsertStmtInfo; +int32_t qParseQuerySql(const char* pStr, size_t length, SParseBasicCtx* pParseCtx, int32_t* type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen); /** * Parse the insert sql statement. @@ -210,9 +88,9 @@ typedef struct SSourceParam { SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, const char* funcName, SSourceParam* pSource, SSchema* pResSchema, int16_t interSize); int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy); -int32_t getExprFunctionLevel(SQueryStmtInfo* pQueryInfo); +int32_t getExprFunctionLevel(const SQueryStmtInfo* pQueryInfo); -STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex); +STableMetaInfo* getMetaInfo(const SQueryStmtInfo* pQueryInfo, int32_t tableIndex); SSchema *getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex); SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* name); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 8b54b88b28fd908bf2ad9040918cdcf9a4c4eea0..153cf0bb3ec2e4ad33f1e04ebff53a709e3e555c 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -25,6 +25,7 @@ extern "C" { #define QUERY_TYPE_MERGE 1 #define QUERY_TYPE_PARTIAL 2 #define QUERY_TYPE_SCAN 3 +#define QUERY_TYPE_MODIFY 4 enum OPERATOR_TYPE_E { OP_Unknown, @@ -58,18 +59,17 @@ typedef struct SQueryNodeBasicInfo { typedef struct SDataSink { SQueryNodeBasicInfo info; - SDataBlockSchema schema; } SDataSink; typedef struct SDataDispatcher { SDataSink sink; - // todo } SDataDispatcher; typedef struct SDataInserter { SDataSink sink; - uint64_t uid; // unique id of the table - // todo data field + int32_t numOfTables; + uint32_t size; + char *pData; } SDataInserter; typedef struct SPhyNode { @@ -119,12 +119,13 @@ typedef struct SSubplanId { typedef struct SSubplan { SSubplanId id; // unique id of the subplan - int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN - int32_t level; // the execution level of current subplan, starting from 0. - SEpSet execEpSet; // for the scan sub plan, the optional execution node - SArray *pChildern; // the datasource subplan,from which to fetch the result - SArray *pParents; // the data destination subplan, get data from current subplan - SPhyNode *pNode; // physical plan of current subplan + int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN + int32_t level; // the execution level of current subplan, starting from 0. + SEpSet execEpSet; // for the scan sub plan, the optional execution node + SArray *pChildern; // the datasource subplan,from which to fetch the result + SArray *pParents; // the data destination subplan, get data from current subplan + SPhyNode *pNode; // physical plan of current subplan + SDataSink *pDataSink; // data of the subplan flow into the datasink } SSubplan; typedef struct SQueryDag { @@ -133,10 +134,12 @@ typedef struct SQueryDag { SArray *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. } SQueryDag; +struct SQueryNode; + /** * Create the physical plan for the query, according to the AST. */ -int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); +int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); // Set datasource of this subplan, multiple calls may be made to a subplan. // @subplan subplan to be schedule @@ -144,7 +147,7 @@ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* // @ep one execution location of this group of datasource subplans int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SEpAddr* ep); -int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str); +int32_t qExplainQuery(const struct SQueryNode* pQueryInfo, struct SEpSet* pQnode, char** str); /** * Convert to subplan to string for the scheduler to send to the executor diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 254d572149b6c01a439cd9e23470ff90c5a65936..002e736c37f1992af6a7693c07b1f64fa879ccb3 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -24,6 +24,15 @@ extern "C" { #include "thash.h" #include "tlog.h" +enum { + JOB_TASK_STATUS_NOT_START = 1, + JOB_TASK_STATUS_EXECUTING, + JOB_TASK_STATUS_SUCCEED, + JOB_TASK_STATUS_FAILED, + JOB_TASK_STATUS_CANCELLING, + JOB_TASK_STATUS_CANCELLED +}; + typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema uint8_t precision; // the number of precision @@ -81,15 +90,27 @@ typedef struct STableMetaOutput { STableMeta *tbMeta; } STableMetaOutput; +typedef int32_t __async_exec_fn_t(void* param); + bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); -extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); -extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); +int32_t initTaskQueue(); +int32_t cleanupTaskQueue(); + +/** + * + * @param execFn The asynchronously execution function + * @param execParam The parameters of the execFn + * @param code The response code during execution the execFn + * @return + */ +int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); SSchema* tGetTbnameColumnSchema(); -extern void msgInit(); +void msgInit(); -extern int32_t qDebugFlag; +extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); +extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); #define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) #define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h new file mode 100644 index 0000000000000000000000000000000000000000..63a6b6f89bee7100abd91426b979163a99cfd641 --- /dev/null +++ b/include/libs/qworker/qworker.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_QWORKER_H_ +#define _TD_QWORKER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "trpc.h" + +typedef struct SQWorkerCfg { + uint32_t maxSchedulerNum; + uint32_t maxResCacheNum; + uint32_t maxSchTaskNum; +} SQWorkerCfg; + +typedef struct { + uint64_t numOfStartTask; + uint64_t numOfStopTask; + uint64_t numOfRecvedFetch; + uint64_t numOfSentHb; + uint64_t numOfSentFetch; + uint64_t numOfTaskInQueue; + uint64_t numOfFetchInQueue; + uint64_t numOfErrors; +} SQWorkerStat; + + +int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt); + +int32_t qWorkerProcessQueryMsg(void *qWorkerMgmt, SSchedulerQueryMsg *msg, SRpcMsg *rsp); + +int32_t qWorkerProcessReadyMsg(void *qWorkerMgmt, SSchedulerReadyMsg *msg, SRpcMsg *rsp); + +int32_t qWorkerProcessStatusMsg(void *qWorkerMgmt, SSchedulerStatusMsg *msg, SRpcMsg *rsp); + +int32_t qWorkerProcessFetchMsg(void *qWorkerMgmt, SSchedulerFetchMsg *msg, SRpcMsg *rsp); + +int32_t qWorkerProcessCancelMsg(void *qWorkerMgmt, SSchedulerCancelMsg *msg, SRpcMsg *rsp); + +void qWorkerDestroy(void **qWorkerMgmt); + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_QWORKER_H_*/ diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 2cbf26f8772927f67c3a882465cd66414be20a93..975b10353871f269abecb70d4974836d3e4a44b2 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -25,6 +25,7 @@ extern "C" { typedef struct SSchedulerCfg { int32_t clusterType; + int32_t maxJobNum; } SSchedulerCfg; typedef struct SQueryProfileSummary { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 95824df5ac9adea0a260012369df2dd0fe4fe73e..7e8df3add27cd16ca7db47f3e996185e9da49fc1 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -230,6 +230,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x03C5) #define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x03C6) +// mnode-trans +#define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) +#define TSDB_CODE_MND_TRANS_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D1) + // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) #define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0401) diff --git a/include/util/thash.h b/include/util/thash.h index d5038802c079f3ed3cddcef6c30c219459fc9917..ebdc91f0546a7ef9de3801f4eabf13f18c2cdf4c 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -193,11 +193,10 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p); /** * Get the corresponding key information for a given data in hash table - * @param pHashObj * @param data * @return */ -int32_t taosHashGetKey(SHashObj *pHashObj, void *data, void** key, size_t* keyLen); +int32_t taosHashGetKey(void *data, void** key, size_t* keyLen); #ifdef __cplusplus } diff --git a/include/util/tthread.h b/include/util/tthread.h index 0ff267dd1fef8e2e52d5032d15441604110655c0..7a5fd1f4c877024f2e3b8ade79a4094c1ef91ebc 100644 --- a/include/util/tthread.h +++ b/include/util/tthread.h @@ -24,8 +24,8 @@ extern "C" { #include "tdef.h" // create new thread -pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param); -// destory thread +pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param); +// destory thread bool taosDestoryThread(pthread_t* pthread); // thread running return true bool taosThreadRunning(pthread_t* pthread); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 4e7fff06b6bd60f015764745d8d093046e812fd6..83ff0d29fb99e527a3acb6355f27728b8cb5f169 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -86,26 +86,32 @@ typedef struct STscObj { SAppInstInfo *pAppInfo; } STscObj; -typedef struct SClientResultInfo { - const char *pMsg; +typedef struct SReqResultInfo { + const char *pRspMsg; const char *pData; TAOS_FIELD *fields; - int32_t numOfCols; - int32_t numOfRows; - int32_t current; + uint32_t numOfCols; + int32_t *length; TAOS_ROW row; char **pCol; -} SClientResultInfo; -typedef struct SReqBody { - tsem_t rspSem; // not used now - void* fp; - void* param; - int32_t paramLen; - int64_t execId; // showId/queryId - SClientResultInfo* pResInfo; -} SRequestBody; + uint32_t numOfRows; + uint32_t current; +} SReqResultInfo; + +typedef struct SReqMsg { + void *pMsg; + uint32_t len; +} SReqMsgInfo; + +typedef struct SRequestSendRecvBody { + tsem_t rspSem; // not used now + void* fp; + int64_t execId; // showId/queryId + SReqMsgInfo requestMsg; + SReqResultInfo resInfo; +} SRequestSendRecvBody; #define ERROR_MSG_BUF_DEFAULT_SIZE 512 @@ -115,7 +121,7 @@ typedef struct SRequestObj { STscObj *pTscObj; SQueryExecMetric metric; char *sqlstr; // sql string - SRequestBody body; + SRequestSendRecvBody body; int64_t self; char *msgBuf; int32_t code; @@ -123,11 +129,10 @@ typedef struct SRequestObj { } SRequestObj; typedef struct SRequestMsgBody { - int32_t msgType; - void *pData; - int32_t msgLen; - uint64_t requestId; - uint64_t requestObjRefId; + int32_t msgType; + SReqMsgInfo msgInfo; + uint64_t requestId; + uint64_t requestObjRefId; } SRequestMsgBody; extern SAppInfo appInfo; @@ -143,13 +148,16 @@ int taos_init(); void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo); void destroyTscObj(void*pObj); -void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); -void destroyRequest(SRequestObj* pRequest); +void *createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); +void destroyRequest(SRequestObj* pRequest); + +char *getConnectionDB(STscObj* pObj); +void setConnectionDB(STscObj* pTscObj, const char* db); void taos_init_imp(void); int taos_options_imp(TSDB_OPTION option, const char *str); -void* openTransporter(const char *user, const char *auth); +void* openTransporter(const char *user, const char *auth, int32_t numOfThreads); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); void initMsgHandleFp(); @@ -158,7 +166,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); void* doFetchRow(SRequestObj* pRequest); -void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); +void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); #ifdef __cplusplus } diff --git a/source/client/src/tscEnv.c b/source/client/src/clientEnv.c similarity index 94% rename from source/client/src/tscEnv.c rename to source/client/src/clientEnv.c index 023bd6ebe977c956f201d45fb63da31066d33ee2..b7a622c2e8f0c06859154b2c212f9a4a9f19b659 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/clientEnv.c @@ -13,17 +13,17 @@ * along with this program. If not, see . */ -#include "clientInt.h" -#include "clientLog.h" #include "os.h" #include "taosmsg.h" +#include "query.h" +#include "clientInt.h" +#include "clientLog.h" #include "tcache.h" #include "tconfig.h" #include "tglobal.h" #include "tnote.h" #include "tref.h" #include "trpc.h" -#include "tsched.h" #include "ttime.h" #include "ttimezone.h" @@ -33,10 +33,8 @@ SAppInfo appInfo; int32_t tscReqRef = -1; int32_t tscConnRef = -1; -void *tscQhandle = NULL; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; -int32_t tsNumOfThreads = 1; volatile int32_t tscInitRes = 0; static void registerRequest(SRequestObj* pRequest) { @@ -98,12 +96,12 @@ void closeTransporter(STscObj* pTscObj) { } // TODO refactor -void* openTransporter(const char *user, const char *auth) { +void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; rpcInit.label = "TSC"; - rpcInit.numOfThreads = tsNumOfThreads; + rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; rpcInit.sessions = tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; @@ -170,7 +168,7 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty pRequest->type = type; pRequest->pTscObj = pObj; pRequest->body.fp = fp; - pRequest->body.param = param; +// pRequest->body.requestMsg. = param; pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); tsem_init(&pRequest->body.rspSem, 0, 0); @@ -188,11 +186,7 @@ static void doDestroyRequest(void* p) { tfree(pRequest->sqlstr); tfree(pRequest->pInfo); - if (pRequest->body.pResInfo != NULL) { - tfree(pRequest->body.pResInfo->pData); - tfree(pRequest->body.pResInfo->pMsg); - tfree(pRequest->body.pResInfo); - } + tfree(pRequest->body.resInfo.pRspMsg); deregisterRequest(pRequest); tfree(pRequest); @@ -233,18 +227,8 @@ void taos_init_imp(void) { taosSetCoreDump(true); - double factor = 4.0; - int32_t numOfThreads = MAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); - - int32_t queueSize = tsMaxConnections * 2; - tscQhandle = taosInitScheduler(queueSize, numOfThreads, "tsc"); - if (NULL == tscQhandle) { - tscError("failed to init task queue"); - tscInitRes = -1; - return; - } + initTaskQueue(); - tscDebug("client task queue is initialized, numOfThreads: %d", numOfThreads); tscConnRef = taosOpenRef(200, destroyTscObj); tscReqRef = taosOpenRef(40960, doDestroyRequest); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index b6200de824822170e2827b041b1c768c3e7a48c1..42de97fe6a2387e824355cd7d491a717cc2d1744 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -102,9 +102,8 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, SAppInstInfo** pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); if (pInst == NULL) { SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); - p->mgmtEp = epSet; - p->pTransporter = openTransporter(user, secretEncrypt); + p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); pInst = &p; @@ -152,11 +151,14 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { int32_t type = 0; void* output = NULL; int32_t outputLen = 0; - code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE); - if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_CREATE_DB) { + + SParseBasicCtx c = {.requestId = pRequest->requestId, .acctId = pTscObj->acctId, .db = getConnectionDB(pTscObj)}; + code = qParseQuerySql(pRequest->sqlstr, sqlLen, &c, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE); + if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || + type == TSDB_SQL_DROP_ACCT || type == TSDB_SQL_CREATE_DB || type == TSDB_SQL_CREATE_ACCT || + type == TSDB_SQL_CREATE_TABLE || type == TSDB_SQL_USE_DB) { pRequest->type = type; - pRequest->body.param = output; - pRequest->body.paramLen = outputLen; + pRequest->body.requestMsg = (SReqMsgInfo){.pMsg = output, .len = outputLen}; SRequestMsgBody body = {0}; buildRequestMsgFp[type](pRequest, &body); @@ -169,6 +171,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { } else { assert(0); } + + tfree(c.db); } if (code != TSDB_CODE_SUCCESS) { @@ -255,7 +259,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { pMsgBody->msgType = TSDB_MSG_TYPE_CONNECT; - pMsgBody->msgLen = sizeof(SConnectMsg); + pMsgBody->msgInfo.len = sizeof(SConnectMsg); pMsgBody->requestObjRefId = pRequest->self; SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg)); @@ -279,28 +283,28 @@ static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) pConnect->startTime = htobe64(appInfo.startTime); tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app)); - pMsgBody->pData = pConnect; + pMsgBody->msgInfo.pMsg = pConnect; return 0; } static void destroyRequestMsgBody(SRequestMsgBody* pMsgBody) { assert(pMsgBody != NULL); - tfree(pMsgBody->pData); + tfree(pMsgBody->msgInfo.pMsg); } int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId) { - char *pMsg = rpcMallocCont(pBody->msgLen); + char *pMsg = rpcMallocCont(pBody->msgInfo.len); if (NULL == pMsg) { tscError("0x%"PRIx64" msg:%s malloc failed", pBody->requestId, taosMsg[pBody->msgType]); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return -1; } - memcpy(pMsg, pBody->pData, pBody->msgLen); + memcpy(pMsg, pBody->msgInfo.pMsg, pBody->msgInfo.len); SRpcMsg rpcMsg = { .msgType = pBody->msgType, .pCont = pMsg, - .contLen = pBody->msgLen, + .contLen = pBody->msgInfo.len, .ahandle = (void*) pBody->requestObjRefId, .handle = NULL, .code = 0 @@ -388,7 +392,7 @@ TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, c void* doFetchRow(SRequestObj* pRequest) { assert(pRequest != NULL); - SClientResultInfo* pResultInfo = pRequest->body.pResInfo; + SReqResultInfo* pResultInfo = &pRequest->body.resInfo; if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { pRequest->type = TSDB_SQL_RETRIEVE_MNODE; @@ -421,7 +425,7 @@ void* doFetchRow(SRequestObj* pRequest) { return pResultInfo->row; } -void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) { +void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) { assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL); if (numOfRows == 0) { return; @@ -436,8 +440,19 @@ void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 } } -const char *taos_get_client_info() { return version; } +char* getConnectionDB(STscObj* pObj) { + char *p = NULL; + pthread_mutex_lock(&pObj->mutex); + p = strndup(pObj->db, tListLen(pObj->db)); + pthread_mutex_unlock(&pObj->mutex); + + return p; +} -int taos_affected_rows(TAOS_RES *res) { return 1; } +void setConnectionDB(STscObj* pTscObj, const char* db) { + assert(db != NULL && pTscObj != NULL); + pthread_mutex_lock(&pTscObj->mutex); + tstrncpy(pTscObj->db, db, tListLen(pTscObj->db)); + pthread_mutex_unlock(&pTscObj->mutex); +} -int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 21e632db8d5d3e5a5cb1885e85a0c50e12e9f3e7..f50765d37a82a2836cd671286d9bd64cdfc4f837 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -1,16 +1,11 @@ +#include "os.h" #include "clientInt.h" #include "clientLog.h" -#include "os.h" +#include "query.h" #include "taosmsg.h" -#include "tcache.h" -#include "tconfig.h" #include "tglobal.h" -#include "tnote.h" #include "tref.h" #include "trpc.h" -#include "tsched.h" -#include "ttime.h" -#include "ttimezone.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -44,9 +39,7 @@ void taos_cleanup(void) { tscReqRef = -1; taosCloseRef(id); - void* p = tscQhandle; - tscQhandle = NULL; - taosCleanUpScheduler(p); + cleanupTaskQueue(); id = tscConnRef; tscConnRef = -1; @@ -115,12 +108,7 @@ int taos_field_count(TAOS_RES *res) { } SRequestObj* pRequest = (SRequestObj*) res; - - SClientResultInfo* pResInfo = pRequest->body.pResInfo; - if (pResInfo == NULL) { - return 0; - } - + SReqResultInfo* pResInfo = &pRequest->body.resInfo; return pResInfo->numOfCols; } @@ -133,7 +121,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { return NULL; } - SClientResultInfo* pResInfo = ((SRequestObj*) res)->body.pResInfo; + SReqResultInfo* pResInfo = &(((SRequestObj*) res)->body.resInfo); return pResInfo->fields; } @@ -248,7 +236,7 @@ int* taos_fetch_lengths(TAOS_RES *res) { return NULL; } - return ((SRequestObj*) res)->body.pResInfo->length; + return ((SRequestObj*) res)->body.resInfo.length; } const char *taos_data_type(int type) { @@ -267,3 +255,9 @@ const char *taos_data_type(int type) { default: return "UNKNOWN"; } } + +const char *taos_get_client_info() { return version; } + +int taos_affected_rows(TAOS_RES *res) { return 1; } + +int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 36fa013f27e0b31f7d0124a3f6e8feccd19d345a..2f996c6ced0499068fe9fda7c5f19803fe487f31 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -13,3089 +13,20 @@ * along with this program. If not, see . */ -#include -#include -#include "clientInt.h" -#include "clientLog.h" -#include "os.h" -#include "tmsgtype.h" -#include "trpc.h" - -int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody) = {0}; -int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); - -#if 0 -void tscProcessActivityTimer(void *handle, void *tmrId); - -static int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo); - -static int32_t minMsgSize() { return tsRpcHeadSize + 100; } -static int32_t getWaitingTimeInterval(int32_t count) { - int32_t initial = 100; // 100 ms by default - if (count <= 1) { - return 0; - } - - return initial * ((2u)<<(count - 2)); -} - -static int32_t vgIdCompare(const void *lhs, const void *rhs) { - int32_t left = *(int32_t *)lhs; - int32_t right = *(int32_t *)rhs; - - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; - } -} -static int32_t removeDupVgid(int32_t *src, int32_t sz) { - if (src == NULL || sz <= 0) { - return 0; - } - qsort(src, sz, sizeof(src[0]), vgIdCompare); - - int32_t ret = 1; - for (int i = 1; i < sz; i++) { - if (src[i] != src[i - 1]) { - src[ret++] = src[i]; - } - } - return ret; -} - -static void tscSetDnodeEpSet(SRpcEpSet* pEpSet, SVgroupMsg* pVgroupInfo) { - assert(pEpSet != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); - - // Issue the query to one of the vnode among a vgroup randomly. - // change the inUse property would not affect the isUse attribute of STableMeta - pEpSet->inUse = rand() % pVgroupInfo->numOfEps; - - // apply the FQDN string length check here - bool existed = false; - - pEpSet->numOfEps = pVgroupInfo->numOfEps; - for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - pEpSet->port[i] = pVgroupInfo->epAddr[i].port; - - int32_t len = (int32_t) strnlen(pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); - if (len > 0) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); - existed = true; - } - } - - assert(existed); -} - -static void tscDumpMgmtEpSet(SSqlObj *pSql) { - SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - taosCorBeginRead(&pCorEpSet->version); - pSql->epSet = pCorEpSet->epSet; - taosCorEndRead(&pCorEpSet->version); -} -static void tscEpSetHtons(SRpcEpSet *s) { - for (int32_t i = 0; i < s->numOfEps; i++) { - s->port[i] = htons(s->port[i]); - } -} - -bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { - if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { - return false; - } - - for (int32_t i = 0; i < s1->numOfEps; i++) { - if (s1->port[i] != s2->port[i] - || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) - return false; - } - return true; -} - -void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { - // no need to update if equal - SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - taosCorBeginWrite(&pCorEpSet->version); - pCorEpSet->epSet = *pEpSet; - taosCorEndWrite(&pCorEpSet->version); -} - -static void tscDumpEpSetFromVgroupInfo(SRpcEpSet *pEpSet, SNewVgroupInfo *pVgroupInfo) { - if (pVgroupInfo == NULL) { return;} - int8_t inUse = pVgroupInfo->inUse; - pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; - pEpSet->numOfEps = pVgroupInfo->numOfEps; - for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - tstrncpy(pEpSet->fqdn[i], pVgroupInfo->ep[i].fqdn, sizeof(pEpSet->fqdn[i])); - pEpSet->port[i] = pVgroupInfo->ep[i].port; - } -} - -static void tscUpdateVgroupInfo(SSqlObj *pSql, SRpcEpSet *pEpSet) { - SSqlCmd *pCmd = &pSql->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { - return; - } - - int32_t vgId = -1; - if (pTableMetaInfo->pTableMeta->tableType == TSDB_SUPER_TABLE) { - vgId = extractSTableQueryVgroupId(pTableMetaInfo); - } else { - vgId = pTableMetaInfo->pTableMeta->vgId; - } - - assert(vgId > 0); - - SNewVgroupInfo vgroupInfo = {.vgId = -1}; - taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); - assert(vgroupInfo.numOfEps > 0 && vgroupInfo.vgId > 0); - - tscDebug("before: Endpoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); - vgroupInfo.inUse = pEpSet->inUse; - vgroupInfo.numOfEps = pEpSet->numOfEps; - for (int32_t i = 0; i < vgroupInfo.numOfEps; i++) { - tstrncpy(vgroupInfo.ep[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); - vgroupInfo.ep[i].port = pEpSet->port[i]; - } - - tscDebug("after: EndPoint in use:%d, numOfEps:%d", vgroupInfo.inUse, vgroupInfo.numOfEps); - taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(SNewVgroupInfo)); - - // Update the local cached epSet info cached by SqlObj - int32_t inUse = pSql->epSet.inUse; - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - tscDebug("0x%"PRIx64" update the epSet in SqlObj, in use before:%d, after:%d", pSql->self, inUse, pSql->epSet.inUse); - -} - -int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo) { - assert(pTableMetaInfo != NULL); - - int32_t vgIndex = pTableMetaInfo->vgroupIndex; - int32_t vgId = -1; - - if (pTableMetaInfo->pVgroupTables == NULL) { - SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; - assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); - vgId = pVgroupInfo->vgroups[vgIndex].vgId; - } else { - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - assert(vgIndex >= 0 && vgIndex < numOfVgroups); - - SVgroupTableInfo *pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); - vgId = pTableIdList->vgInfo.vgId; - } - - return vgId; -} - -void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { - STscObj *pObj = (STscObj *)param; - if (pObj == NULL) return; - - if (pObj != pObj->signature) { - tscError("heartbeat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature); - return; - } - - SSqlObj *pSql = tres; - SSqlRes *pRes = &pSql->res; - - if (code == TSDB_CODE_SUCCESS) { - SHeartBeatRsp *pRsp = (SHeartBeatRsp *)pRes->pRsp; - SRpcEpSet *epSet = &pRsp->epSet; - if (epSet->numOfEps > 0) { - tscEpSetHtons(epSet); - - //SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - //if (!tscEpSetIsEqual(&pCorEpSet->epSet, epSet)) { - // tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse); - // for (int8_t i = 0; i < epSet->numOfEps; i++) { - // tscTrace("endpoint %d: fqdn=%s, port=%d", i, epSet->fqdn[i], epSet->port[i]); - // } - //} - //concurrency problem, update mgmt epset anyway - tscUpdateMgmtEpSet(pSql, epSet); - } - - pSql->pTscObj->connId = htonl(pRsp->connId); - - if (pRsp->killConnection) { - tscKillConnection(pObj); - return; - } else { - if (pRsp->queryId) { - tscKillQuery(pObj, htonl(pRsp->queryId)); - } - - if (pRsp->streamId) { - tscKillStream(pObj, htonl(pRsp->streamId)); - } - } - - int32_t total = htonl(pRsp->totalDnodes); - int32_t online = htonl(pRsp->onlineDnodes); - assert(online <= total); - - if (online < total) { - tscError("0x%"PRIx64", HB, total dnode:%d, online dnode:%d", pSql->self, total, online); - pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - } - - if (pRes->length == NULL) { - pRes->length = calloc(2, sizeof(int32_t)); - } - - pRes->length[0] = total; - pRes->length[1] = online; - } else { - tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code)); - if (pRes->length == NULL) { - pRes->length = calloc(2, sizeof(int32_t)); - } - - pRes->length[1] = 0; - if (pRes->length[0] == 0) { - pRes->length[0] = 1; // make sure that the value of the total node is greater than the online node - } - } - - if (pObj->hbrid != 0) { - int32_t waitingDuring = tsShellActivityTimer * 500; - tscDebug("0x%"PRIx64" send heartbeat in %dms", pSql->self, waitingDuring); - - taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer); - } else { - tscDebug("0x%"PRIx64" start to close tscObj:%p, not send heartbeat again", pSql->self, pObj); - } -} - -void tscProcessActivityTimer(void *handle, void *tmrId) { - int64_t rid = (int64_t) handle; - STscObj *pObj = taosAcquireRef(tscRefId, rid); - if (pObj == NULL) { - return; - } - - SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid); - if (pHB == NULL) { - taosReleaseRef(tscRefId, rid); - return; - } - - assert(pHB->self == pObj->hbrid); - - pHB->retry = 0; - int32_t code = tscBuildAndSendRequest(pHB, NULL); - taosReleaseRef(tscObjRef, pObj->hbrid); - - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" failed to sent HB to server, reason:%s", pHB->self, tstrerror(code)); - } - - taosReleaseRef(tscRefId, rid); -} - -int tscSendMsgToServer(SSqlObj *pSql) { - STscObj* pObj = pSql->pTscObj; - SSqlCmd* pCmd = &pSql->cmd; - - char *pMsg = rpcMallocCont(pCmd->payloadLen); - if (NULL == pMsg) { - tscError("0x%"PRIx64" msg:%s malloc failed", pSql->self, taosMsg[pSql->cmd.msgType]); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - // set the mgmt ip list - if (pSql->cmd.command >= TSDB_SQL_MGMT) { - tscDumpMgmtEpSet(pSql); - } - - memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); - - SRpcMsg rpcMsg = { - .msgType = pSql->cmd.msgType, - .pCont = pMsg, - .contLen = pSql->cmd.payloadLen, - .ahandle = (void*)pSql->self, - .handle = NULL, - .code = 0 - }; - - rpcSendRequest(pObj->pRpcObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid); - return TSDB_CODE_SUCCESS; -} - -// handle three situation -// 1. epset retry, only return last failure ep -// 2. no epset retry, like 'taos -h invalidFqdn', return invalidFqdn -// 3. other situation, no expected -void tscSetFqdnErrorMsg(SSqlObj* pSql, SRpcEpSet* pEpSet) { - SSqlCmd* pCmd = &pSql->cmd; - SSqlRes* pRes = &pSql->res; - - char* msgBuf = tscGetErrorMsgPayload(pCmd); - - if (pEpSet) { - sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),pEpSet->fqdn[(pEpSet->inUse)%(pEpSet->numOfEps)]); - } else if (pCmd->command >= TSDB_SQL_MGMT) { - SRpcEpSet tEpset; - - SCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; - taosCorBeginRead(&pCorEpSet->version); - tEpset = pCorEpSet->epSet; - taosCorEndRead(&pCorEpSet->version); - - sprintf(msgBuf, "%s\"%s\"", tstrerror(pRes->code),tEpset.fqdn[(tEpset.inUse)%(tEpset.numOfEps)]); - } else { - sprintf(msgBuf, "%s", tstrerror(pRes->code)); - } -} - -void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { - TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle; - SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle); - if (pSql == NULL) { - rpcFreeCont(rpcMsg->pCont); - return; - } - - assert(pSql->self == handle); - - STscObj *pObj = pSql->pTscObj; - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - pSql->rpcRid = -1; - if (pObj->signature != pObj) { - tscDebug("0x%"PRIx64" DB connection is closed, cmd:%d pObj:%p signature:%p", pSql->self, pCmd->command, pObj, pObj->signature); - - taosRemoveRef(tscObjRef, handle); - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); - return; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) { - tscDebug("0x%"PRIx64" sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", - pSql->self, pCmd->command, pQueryInfo->type, pObj, pObj->signature); - - taosRemoveRef(tscObjRef, handle); - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); - return; - } - - if (pEpSet) { - if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { - if (pCmd->command < TSDB_SQL_MGMT) { - tscUpdateVgroupInfo(pSql, pEpSet); - } else { - tscUpdateMgmtEpSet(pSql, pEpSet); - } - } - } - - int32_t cmd = pCmd->command; - - // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema - if (cmd == TSDB_SQL_INSERT && rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - pSql->cmd.insertParam.schemaAttached = 1; - } - - // single table query error need to be handled here. - if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && - (((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) || - rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { - - // 1. super table subquery - // 2. nest queries are all not updated the tablemeta and retry parse the sql after cleanup local tablemeta/vgroup id buffer - if ((TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY | - TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) || - (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY)) || (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->distinct)) { - // do nothing in case of super table subquery - } else { - pSql->retry += 1; - tscWarn("0x%" PRIx64 " it shall renew table meta, code:%s, retry:%d", pSql->self, tstrerror(rpcMsg->code), pSql->retry); - - pSql->res.code = rpcMsg->code; // keep the previous error code - if (pSql->retry > pSql->maxRetry) { - tscError("0x%" PRIx64 " max retry %d reached, give up", pSql->self, pSql->maxRetry); - } else { - // wait for a little bit moment and then retry - // todo do not sleep in rpc callback thread, add this process into queue to process - if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { - int32_t duration = getWaitingTimeInterval(pSql->retry); - taosMsleep(duration); - } - - pSql->retryReason = rpcMsg->code; - rpcMsg->code = tscRenewTableMeta(pSql, 0); - // if there is an error occurring, proceed to the following error handling procedure. - if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); - return; - } - } - } - } - - pRes->rspLen = 0; - - if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { - tscDebug("0x%"PRIx64" query is cancelled, code:%s", pSql->self, tstrerror(pRes->code)); - } else { - pRes->code = rpcMsg->code; - } - - if (pRes->code == TSDB_CODE_SUCCESS) { - tscDebug("0x%"PRIx64" reset retry counter to be 0 due to success rsp, old:%d", pSql->self, pSql->retry); - pSql->retry = 0; - } - - if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { - assert(rpcMsg->msgType == pCmd->msgType + 1); - pRes->code = rpcMsg->code; - pRes->rspType = rpcMsg->msgType; - pRes->rspLen = rpcMsg->contLen; - - if (pRes->rspLen > 0 && rpcMsg->pCont) { - char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen); - if (tmp == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - } else { - pRes->pRsp = tmp; - memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen); - } - } else { - tfree(pRes->pRsp); - } - - /* - * There is not response callback function for submit response. - * The actual inserted number of points is the first number. - */ - if (rpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT_RSP && pRes->pRsp != NULL) { - SShellSubmitRspMsg *pMsg = (SShellSubmitRspMsg*)pRes->pRsp; - pMsg->code = htonl(pMsg->code); - pMsg->numOfRows = htonl(pMsg->numOfRows); - pMsg->affectedRows = htonl(pMsg->affectedRows); - pMsg->failedRows = htonl(pMsg->failedRows); - pMsg->numOfFailedBlocks = htonl(pMsg->numOfFailedBlocks); - - pRes->numOfRows += pMsg->affectedRows; - tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s inserted rows:%d rspLen:%d", pSql->self, sqlCmd[pCmd->command], - tstrerror(pRes->code), pMsg->affectedRows, pRes->rspLen); - } else { - tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s rspLen:%d", pSql->self, sqlCmd[pCmd->command], tstrerror(pRes->code), pRes->rspLen); - } - } - - if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) { - rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql); - } - - bool shouldFree = tscShouldBeFreed(pSql); - if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - if (rpcMsg->code != TSDB_CODE_SUCCESS) { - pRes->code = rpcMsg->code; - } - - rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; - if (rpcMsg->code == TSDB_CODE_RPC_FQDN_ERROR) { - tscAllocPayload(pCmd, TSDB_FQDN_LEN + 64); - tscSetFqdnErrorMsg(pSql, pEpSet); - } - - (*pSql->fp)(pSql->param, pSql, rpcMsg->code); - } - - if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it - tscDebug("0x%"PRIx64" sqlObj is automatically freed", pSql->self); - taosRemoveRef(tscObjRef, handle); - } - - taosReleaseRef(tscObjRef, handle); - rpcFreeCont(rpcMsg->pCont); -} - -int doBuildAndSendMsg(SSqlObj *pSql) { - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - - if (pCmd->command == TSDB_SQL_SELECT || - pCmd->command == TSDB_SQL_FETCH || - pCmd->command == TSDB_SQL_RETRIEVE_MNODE || - pCmd->command == TSDB_SQL_INSERT || - pCmd->command == TSDB_SQL_CONNECT || - pCmd->command == TSDB_SQL_HB || - pCmd->command == TSDB_SQL_RETRIEVE_FUNC || - pCmd->command == TSDB_SQL_STABLEVGROUP) { - pRes->code = tscBuildMsg[pCmd->command](pSql, NULL); - } - - if (pRes->code != TSDB_CODE_SUCCESS) { - tscAsyncResultOnError(pSql); - return TSDB_CODE_SUCCESS; - } - - int32_t code = tscSendMsgToServer(pSql); - - // NOTE: if code is TSDB_CODE_SUCCESS, pSql may have been released here already by other threads. - if (code != TSDB_CODE_SUCCESS) { - pRes->code = code; - tscAsyncResultOnError(pSql); - return TSDB_CODE_SUCCESS; - } - - return TSDB_CODE_SUCCESS; -} - -int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - - SSqlCmd *pCmd = &pSql->cmd; - uint32_t type = 0; - - if (pQueryInfo == NULL) { - pQueryInfo = tscGetQueryInfo(pCmd); - } - - STableMetaInfo *pTableMetaInfo = NULL; - - if (pQueryInfo != NULL) { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo != NULL) { - tNameExtractFullName(&pTableMetaInfo->name, name); - } - - type = pQueryInfo->type; - - // while numOfTables equals to 0, it must be Heartbeat - assert((pQueryInfo->numOfTables == 0 && (pQueryInfo->command == TSDB_SQL_HB || pSql->cmd.command == TSDB_SQL_RETRIEVE_FUNC)) || pQueryInfo->numOfTables > 0); - } - - tscDebug("0x%"PRIx64" SQL cmd:%s will be processed, name:%s, type:%d", pSql->self, sqlCmd[pCmd->command], name, type); - if (pCmd->command < TSDB_SQL_MGMT) { // the pTableMetaInfo cannot be NULL - if (pTableMetaInfo == NULL) { - pSql->res.code = TSDB_CODE_TSC_APP_ERROR; - return pSql->res.code; - } - } else if (pCmd->command >= TSDB_SQL_LOCAL) { - return (*tscProcessMsgRsp[pCmd->command])(pSql); - } - - return doBuildAndSendMsg(pSql); -} - -int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg *) pSql->cmd.payload; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - - pRetrieveMsg->free = htons(pQueryInfo->type); - pRetrieveMsg->qId = htobe64(pSql->res.qId); - - // todo valid the vgroupId at the client side - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - int32_t vgIndex = pTableMetaInfo->vgroupIndex; - int32_t vgId = -1; - - if (pTableMetaInfo->pVgroupTables == NULL) { - SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; - assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); - vgId = pVgroupInfo->vgroups[vgIndex].vgId; - } else { - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - assert(vgIndex >= 0 && vgIndex < numOfVgroups); - - SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); - vgId = pTableIdList->vgInfo.vgId; - } - - pRetrieveMsg->header.vgId = htonl(vgId); - tscDebug("0x%"PRIx64" build fetch msg from vgId:%d, vgIndex:%d, qId:0x%" PRIx64, pSql->self, vgId, vgIndex, pSql->res.qId); - } else { - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); - tscDebug("0x%"PRIx64" build fetch msg from only one vgroup, vgId:%d, qId:0x%" PRIx64, pSql->self, pTableMeta->vgId, - pSql->res.qId); - } - - pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg); - pSql->cmd.msgType = TSDB_MSG_TYPE_FETCH; - - pRetrieveMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SQueryInfo *pQueryInfo = tscGetQueryInfo(&pSql->cmd); - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - - // pSql->cmd.payloadLen is set during copying data into payload - pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - - SNewVgroupInfo vgroupInfo = {0}; - taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - - tscDebug("0x%"PRIx64" submit msg built, numberOfEP:%d", pSql->self, pSql->epSet.numOfEps); - - return TSDB_CODE_SUCCESS; -} - -/* - * for table query, simply return the size <= 1k - */ -static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql) { - const static int32_t MIN_QUERY_MSG_PKT_SIZE = TSDB_MAX_BYTES_PER_ROW * 5; - - SSqlCmd* pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - int32_t srcColListSize = (int32_t)(taosArrayGetSize(pQueryInfo->colList) * sizeof(SColumnInfo)); - int32_t srcColFilterSize = 0; - int32_t srcTagFilterSize = tscGetTagFilterSerializeLen(pQueryInfo); - - size_t numOfExprs = tscNumOfExprs(pQueryInfo); - int32_t exprSize = (int32_t)(sizeof(SSqlExpr) * numOfExprs * 2); - - int32_t tsBufSize = (pQueryInfo->tsBuf != NULL) ? pQueryInfo->tsBuf->fileSize : 0; - int32_t sqlLen = (int32_t) strlen(pSql->sqlstr) + 1; - - int32_t tableSerialize = 0; - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - if (pTableMetaInfo->pVgroupTables != NULL) { - size_t numOfGroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); - - int32_t totalTables = 0; - for (int32_t i = 0; i < numOfGroups; ++i) { - SVgroupTableInfo *pTableInfo = taosArrayGet(pTableMetaInfo->pVgroupTables, i); - totalTables += (int32_t) taosArrayGetSize(pTableInfo->itemList); - } - - tableSerialize = totalTables * sizeof(STableIdInfo); - } - - if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) { - STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); - if (pCond != NULL && pCond->cond != NULL) { - srcColFilterSize = pCond->len; - } - } - - SCond* pCond = &pQueryInfo->tagCond.tbnameCond; - if (pCond->len > 0) { - srcColListSize += pCond->len; - } - - return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + srcColFilterSize + srcTagFilterSize + - exprSize + tsBufSize + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; -} - -static char *doSerializeTableInfo(SQueryTableMsg *pQueryMsg, SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, char *pMsg, - int32_t *succeed) { - TSKEY dfltKey = htobe64(pQueryMsg->window.skey); - - STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo) || pTableMetaInfo->pVgroupTables == NULL) { - - int32_t vgId = -1; - if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - int32_t index = pTableMetaInfo->vgroupIndex; - assert(index >= 0); - - SVgroupMsg* pVgroupInfo = NULL; - if (pTableMetaInfo->vgroupList && pTableMetaInfo->vgroupList->numOfVgroups > 0) { - assert(index < pTableMetaInfo->vgroupList->numOfVgroups); - pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; - } else { - tscError("0x%"PRIx64" No vgroup info found", pSql->self); - - *succeed = 0; - return pMsg; - } - - vgId = pVgroupInfo->vgId; - tscSetDnodeEpSet(&pSql->epSet, pVgroupInfo); - tscDebug("0x%"PRIx64" query on stable, vgIndex:%d, numOfVgroups:%d", pSql->self, index, pTableMetaInfo->vgroupList->numOfVgroups); - } else { - vgId = pTableMeta->vgId; - - SNewVgroupInfo vgroupInfo = {0}; - taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - } - - if (pSql->epSet.numOfEps > 0){ - pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; - } - pQueryMsg->head.vgId = htonl(vgId); - - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->tid = htonl(pTableMeta->id.tid); - pTableIdInfo->uid = htobe64(pTableMeta->id.uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->id.uid, dfltKey)); - - pQueryMsg->numOfTables = htonl(1); // set the number of tables - pMsg += sizeof(STableIdInfo); - } else { // it is a subquery of the super table query, this EP info is acquired from vgroupInfo - int32_t index = pTableMetaInfo->vgroupIndex; - int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); - assert(index >= 0 && index < numOfVgroups); - - SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); - - // set the vgroup info - tscSetDnodeEpSet(&pSql->epSet, &pTableIdList->vgInfo); - pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); - - int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList); - pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables - - tscDebug("0x%"PRIx64" query on stable, vgId:%d, numOfTables:%d, vgIndex:%d, numOfVgroups:%d", pSql->self, - pTableIdList->vgInfo.vgId, numOfTables, index, numOfVgroups); - - // serialize each table id info - for(int32_t i = 0; i < numOfTables; ++i) { - STableIdInfo* pItem = taosArrayGet(pTableIdList->itemList, i); - - STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->tid = htonl(pItem->tid); - pTableIdInfo->uid = htobe64(pItem->uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pItem->uid, dfltKey)); - pMsg += sizeof(STableIdInfo); - } - } - - char n[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, n); - - tscDebug("0x%"PRIx64" vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql->self, htonl(pQueryMsg->head.vgId), n, pTableMeta->id.tid, pTableMeta->id.uid); - return pMsg; -} - -// TODO refactor -static int32_t serializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t numOfFilters, char** pMsg) { - // append the filter information after the basic column information - for (int32_t f = 0; f < numOfFilters; ++f) { - SColumnFilterInfo *pColFilter = &pColFilters[f]; - - SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)(*pMsg); - pFilterMsg->filterstr = htons(pColFilter->filterstr); - - (*pMsg) += sizeof(SColumnFilterInfo); - - if (pColFilter->filterstr) { - pFilterMsg->len = htobe64(pColFilter->len); - memcpy(*pMsg, (void *)pColFilter->pz, (size_t)(pColFilter->len + 1)); - (*pMsg) += (pColFilter->len + 1); // append the additional filter binary info - } else { - pFilterMsg->lowerBndi = htobe64(pColFilter->lowerBndi); - pFilterMsg->upperBndi = htobe64(pColFilter->upperBndi); - } - - pFilterMsg->lowerRelOptr = htons(pColFilter->lowerRelOptr); - pFilterMsg->upperRelOptr = htons(pColFilter->upperRelOptr); - - if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { - tscError("invalid filter info"); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t serializeSqlExpr(SSqlExpr* pExpr, STableMetaInfo* pTableMetaInfo, char** pMsg, int64_t id, bool validateColumn) { - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - // the queried table has been removed and a new table with the same name has already been created already - // return error msg - if (pExpr->uid != pTableMeta->id.uid) { - tscError("0x%"PRIx64" table has already been destroyed", id); - return TSDB_CODE_TSC_INVALID_TABLE_NAME; - } - - if (validateColumn && !tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { - tscError("0x%"PRIx64" table schema is not matched with parsed sql", id); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - assert(pExpr->resColId < 0); - SSqlExpr* pSqlExpr = (SSqlExpr *)(*pMsg); - - SColIndex* pIndex = &pSqlExpr->colInfo; - - pIndex->colId = htons(pExpr->colInfo.colId); - pIndex->colIndex = htons(pExpr->colInfo.colIndex); - pIndex->flag = htons(pExpr->colInfo.flag); - pSqlExpr->uid = htobe64(pExpr->uid); - pSqlExpr->colType = htons(pExpr->colType); - pSqlExpr->colBytes = htons(pExpr->colBytes); - pSqlExpr->resType = htons(pExpr->resType); - pSqlExpr->resBytes = htons(pExpr->resBytes); - pSqlExpr->interBytes = htonl(pExpr->interBytes); - pSqlExpr->functionId = htons(pExpr->functionId); - pSqlExpr->numOfParams = htons(pExpr->numOfParams); - pSqlExpr->resColId = htons(pExpr->resColId); - pSqlExpr->flist.numOfFilters = htons(pExpr->flist.numOfFilters); - - (*pMsg) += sizeof(SSqlExpr); - for (int32_t j = 0; j < pExpr->numOfParams; ++j) { // todo add log - pSqlExpr->param[j].nType = htonl(pExpr->param[j].nType); - pSqlExpr->param[j].nLen = htonl(pExpr->param[j].nLen); - - if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) { - memcpy((*pMsg), pExpr->param[j].pz, pExpr->param[j].nLen); - (*pMsg) += pExpr->param[j].nLen; - } else { - pSqlExpr->param[j].i64 = htobe64(pExpr->param[j].i64); - } - } - - serializeColFilterInfo(pExpr->flist.filterInfo, pExpr->flist.numOfFilters, pMsg); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = NULL; - STableMeta *pTableMeta = NULL; - STableMetaInfo *pTableMetaInfo = NULL; - - int32_t code = TSDB_CODE_SUCCESS; - int32_t size = tscEstimateQueryMsgSize(pSql); - assert(size > 0); - - if (TSDB_CODE_SUCCESS != tscAllocPayloadFast(pCmd, size)) { - tscError("%p failed to malloc for query msg", pSql); - return TSDB_CODE_TSC_INVALID_OPERATION; // todo add test for this - } - - pQueryInfo = tscGetQueryInfo(pCmd); - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - pTableMeta = pTableMetaInfo->pTableMeta; - - SQueryAttr query = {{0}}; - tscCreateQueryFromQueryInfo(pQueryInfo, &query, pSql); - query.vgId = pTableMeta->vgId; - - SArray* tableScanOperator = createTableScanPlan(&query); - SArray* queryOperator = createExecOperatorPlan(&query); - - SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; - tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); - - int32_t numOfTags = query.numOfTags; - int32_t sqlLen = (int32_t) strlen(pSql->sqlstr); - - if (taosArrayGetSize(tableScanOperator) == 0) { - pQueryMsg->tableScanOperator = htonl(-1); - } else { - int32_t* tablescanOp = taosArrayGet(tableScanOperator, 0); - pQueryMsg->tableScanOperator = htonl(*tablescanOp); - } - - pQueryMsg->window.skey = htobe64(query.window.skey); - pQueryMsg->window.ekey = htobe64(query.window.ekey); - - pQueryMsg->order = htons(query.order.order); - pQueryMsg->orderColId = htons(query.order.orderColId); - pQueryMsg->fillType = htons(query.fillType); - pQueryMsg->limit = htobe64(query.limit.limit); - pQueryMsg->offset = htobe64(query.limit.offset); - pQueryMsg->numOfCols = htons(query.numOfCols); - - pQueryMsg->interval.interval = htobe64(query.interval.interval); - pQueryMsg->interval.sliding = htobe64(query.interval.sliding); - pQueryMsg->interval.offset = htobe64(query.interval.offset); - pQueryMsg->interval.intervalUnit = query.interval.intervalUnit; - pQueryMsg->interval.slidingUnit = query.interval.slidingUnit; - pQueryMsg->interval.offsetUnit = query.interval.offsetUnit; - - pQueryMsg->stableQuery = query.stableQuery; - pQueryMsg->topBotQuery = query.topBotQuery; - pQueryMsg->groupbyColumn = query.groupbyColumn; - pQueryMsg->hasTagResults = query.hasTagResults; - pQueryMsg->timeWindowInterpo = query.timeWindowInterpo; - pQueryMsg->queryBlockDist = query.queryBlockDist; - pQueryMsg->stabledev = query.stabledev; - pQueryMsg->tsCompQuery = query.tsCompQuery; - pQueryMsg->simpleAgg = query.simpleAgg; - pQueryMsg->pointInterpQuery = query.pointInterpQuery; - pQueryMsg->needReverseScan = query.needReverseScan; - pQueryMsg->stateWindow = query.stateWindow; - pQueryMsg->numOfTags = htonl(numOfTags); - pQueryMsg->sqlstrLen = htonl(sqlLen); - pQueryMsg->sw.gap = htobe64(query.sw.gap); - pQueryMsg->sw.primaryColId = htonl(PRIMARYKEY_TIMESTAMP_COL_INDEX); - - pQueryMsg->secondStageOutput = htonl(query.numOfExpr2); - pQueryMsg->numOfOutput = htons((int16_t)query.numOfOutput); // this is the stage one output column number - - pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); - pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); - pQueryMsg->tbnameCondLen = htonl(pQueryInfo->tagCond.tbnameCond.len); - pQueryMsg->queryType = htonl(pQueryInfo->type); - pQueryMsg->prevResultLen = htonl(pQueryInfo->bufLen); - - // set column list ids - size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); - char *pMsg = (char *)(pQueryMsg->tableCols) + numOfCols * sizeof(SColumnInfo); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfo *pCol = &query.tableCols[i]; - - pQueryMsg->tableCols[i].colId = htons(pCol->colId); - pQueryMsg->tableCols[i].bytes = htons(pCol->bytes); - pQueryMsg->tableCols[i].type = htons(pCol->type); - //pQueryMsg->tableCols[i].flist.numOfFilters = htons(pCol->flist.numOfFilters); - pQueryMsg->tableCols[i].flist.numOfFilters = 0; - pQueryMsg->tableCols[i].flist.filterInfo = 0; - // append the filter information after the basic column information - //serializeColFilterInfo(pCol->flist.filterInfo, pCol->flist.numOfFilters, &pMsg); - } - - if (pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0 && !onlyQueryTags(&query) ) { - STblCond *pCond = tsGetTableFilter(pQueryInfo->colCond, pTableMeta->id.uid, 0); - if (pCond != NULL && pCond->cond != NULL) { - pQueryMsg->colCondLen = htons(pCond->len); - memcpy(pMsg, pCond->cond, pCond->len); - - pMsg += pCond->len; - } - } else { - pQueryMsg->colCondLen = 0; - } - - for (int32_t i = 0; i < query.numOfOutput; ++i) { - code = serializeSqlExpr(&query.pExpr1[i].base, pTableMetaInfo, &pMsg, pSql->self, true); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - } - - for (int32_t i = 0; i < query.numOfExpr2; ++i) { - code = serializeSqlExpr(&query.pExpr2[i].base, pTableMetaInfo, &pMsg, pSql->self, false); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - } - - int32_t succeed = 1; - - // serialize the table info (sid, uid, tags) - pMsg = doSerializeTableInfo(pQueryMsg, pSql, pTableMetaInfo, pMsg, &succeed); - if (succeed == 0) { - code = TSDB_CODE_TSC_APP_ERROR; - goto _end; - } - - SGroupbyExpr *pGroupbyExpr = query.pGroupbyExpr; - if (pGroupbyExpr != NULL && pGroupbyExpr->numOfGroupCols > 0) { - pQueryMsg->orderByIdx = htons(pGroupbyExpr->orderIndex); - pQueryMsg->orderType = htons(pGroupbyExpr->orderType); - - for (int32_t j = 0; j < pGroupbyExpr->numOfGroupCols; ++j) { - SColIndex* pCol = taosArrayGet(pGroupbyExpr->columnInfo, j); - - *((int16_t *)pMsg) = htons(pCol->colId); - pMsg += sizeof(pCol->colId); - - *((int16_t *)pMsg) += htons(pCol->colIndex); - pMsg += sizeof(pCol->colIndex); - - *((int16_t *)pMsg) += htons(pCol->flag); - pMsg += sizeof(pCol->flag); - - memcpy(pMsg, pCol->name, tListLen(pCol->name)); - pMsg += tListLen(pCol->name); - } - } - - if (query.fillType != TSDB_FILL_NONE) { - for (int32_t i = 0; i < query.numOfOutput; ++i) { - *((int64_t *)pMsg) = htobe64(query.fillVal[i]); - pMsg += sizeof(query.fillVal[0]); - } - } - - if (query.numOfTags > 0 && query.tagColList != NULL) { - for (int32_t i = 0; i < query.numOfTags; ++i) { - SColumnInfo* pTag = &query.tagColList[i]; - - SColumnInfo* pTagCol = (SColumnInfo*) pMsg; - pTagCol->colId = htons(pTag->colId); - pTagCol->bytes = htons(pTag->bytes); - pTagCol->type = htons(pTag->type); - pTagCol->flist.numOfFilters = 0; - - pMsg += sizeof(SColumnInfo); - } - } - - // serialize tag column query condition - if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) { - STagCond* pTagCond = &pQueryInfo->tagCond; - - SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid); - if (pCond != NULL && pCond->cond != NULL) { - pQueryMsg->tagCondLen = htons(pCond->len); - memcpy(pMsg, pCond->cond, pCond->len); - - pMsg += pCond->len; - } - } else { - pQueryMsg->tagCondLen = 0; - } - - if (pQueryInfo->bufLen > 0) { - memcpy(pMsg, pQueryInfo->buf, pQueryInfo->bufLen); - pMsg += pQueryInfo->bufLen; - } - - SCond* pCond = &pQueryInfo->tagCond.tbnameCond; - if (pCond->len > 0) { - strncpy(pMsg, pCond->cond, pCond->len); - pMsg += pCond->len; - } - - // compressed ts block - pQueryMsg->tsBuf.tsOffset = htonl((int32_t)(pMsg - pCmd->payload)); - - if (pQueryInfo->tsBuf != NULL) { - // note: here used the index instead of actual vnode id. - int32_t vnodeIndex = pTableMetaInfo->vgroupIndex; - code = dumpFileBlockByGroupId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsBuf.tsLen, &pQueryMsg->tsBuf.tsNumOfBlocks); - if (code != TSDB_CODE_SUCCESS) { - goto _end; - } - - pMsg += pQueryMsg->tsBuf.tsLen; - - pQueryMsg->tsBuf.tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); - pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); - pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); - } else { - pQueryMsg->tsBuf.tsLen = 0; - pQueryMsg->tsBuf.tsNumOfBlocks = 0; - } - - int32_t numOfOperator = (int32_t) taosArrayGetSize(queryOperator); - pQueryMsg->numOfOperator = htonl(numOfOperator); - for(int32_t i = 0; i < numOfOperator; ++i) { - int32_t *operator = taosArrayGet(queryOperator, i); - *(int32_t*)pMsg = htonl(*operator); - - pMsg += sizeof(int32_t); - } - - // support only one udf - if (pQueryInfo->pUdfInfo != NULL && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) { - pQueryMsg->udfContentOffset = htonl((int32_t) (pMsg - pCmd->payload)); - for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i); - *(int8_t*) pMsg = pUdfInfo->resType; - pMsg += sizeof(pUdfInfo->resType); - - *(int16_t*) pMsg = htons(pUdfInfo->resBytes); - pMsg += sizeof(pUdfInfo->resBytes); - - STR_TO_VARSTR(pMsg, pUdfInfo->name); - - pMsg += varDataTLen(pMsg); - - *(int32_t*) pMsg = htonl(pUdfInfo->funcType); - pMsg += sizeof(pUdfInfo->funcType); - - *(int32_t*) pMsg = htonl(pUdfInfo->bufSize); - pMsg += sizeof(pUdfInfo->bufSize); - - pQueryMsg->udfContentLen = htonl(pUdfInfo->contLen); - memcpy(pMsg, pUdfInfo->content, pUdfInfo->contLen); - - pMsg += pUdfInfo->contLen; - } - } else { - pQueryMsg->udfContentOffset = 0; - pQueryMsg->udfContentLen = 0; - } - - memcpy(pMsg, pSql->sqlstr, sqlLen); - pMsg += sqlLen; - - int32_t msgLen = (int32_t)(pMsg - pCmd->payload); - - tscDebug("0x%"PRIx64" msg built success, len:%d bytes", pSql->self, msgLen); - pCmd->payloadLen = msgLen; - pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY; - - pQueryMsg->head.contLen = htonl(msgLen); - assert(msgLen + minMsgSize() <= (int32_t)pCmd->allocSize); - - _end: - freeQueryAttr(&query); - taosArrayDestroy(tableScanOperator); - taosArrayDestroy(queryOperator); - return code; -} - -int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateDbMsg); - - pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CREATE_DB : TSDB_MSG_TYPE_CREATE_TP; - - SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload; - -// assert(pCmd->numOfClause == 1); - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateDbMsg->db); - assert(code == TSDB_CODE_SUCCESS); - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildCreateFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - SCreateFuncMsg *pCreateFuncMsg = (SCreateFuncMsg *)pCmd->payload; - - pCmd->msgType = TSDB_MSG_TYPE_CREATE_FUNCTION; - - pCmd->payloadLen = sizeof(SCreateFuncMsg) + htonl(pCreateFuncMsg->codeLen); - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateDnodeMsg); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload; - - SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); - strncpy(pCreate->ep, t0->z, t0->n); - - pCmd->msgType = TSDB_MSG_TYPE_CREATE_DNODE; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateAcctMsg); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateAcctMsg *pAlterMsg = (SCreateAcctMsg *)pCmd->payload; - - SStrToken *pName = &pInfo->pMiscInfo->user.user; - SStrToken *pPwd = &pInfo->pMiscInfo->user.passwd; - - strncpy(pAlterMsg->user, pName->z, pName->n); - strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); - - SCreateAcctInfo *pAcctOpt = &pInfo->pMiscInfo->acctOpt; - - pAlterMsg->cfg.maxUsers = htonl(pAcctOpt->maxUsers); - pAlterMsg->cfg.maxDbs = htonl(pAcctOpt->maxDbs); - pAlterMsg->cfg.maxTimeSeries = htonl(pAcctOpt->maxTimeSeries); - pAlterMsg->cfg.maxStreams = htonl(pAcctOpt->maxStreams); - pAlterMsg->cfg.maxPointsPerSecond = htonl(pAcctOpt->maxPointsPerSecond); - pAlterMsg->cfg.maxStorage = htobe64(pAcctOpt->maxStorage); - pAlterMsg->cfg.maxQueryTime = htobe64(pAcctOpt->maxQueryTime); - pAlterMsg->cfg.maxConnections = htonl(pAcctOpt->maxConnections); - - if (pAcctOpt->stat.n == 0) { - pAlterMsg->cfg.accessState = -1; - } else { - if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { - pAlterMsg->cfg.accessState = TSDB_VN_READ_ACCCESS; - } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { - pAlterMsg->cfg.accessState = TSDB_VN_WRITE_ACCCESS; - } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { - pAlterMsg->cfg.accessState = TSDB_VN_ALL_ACCCESS; - } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { - pAlterMsg->cfg.accessState = 0; - } - } - - pCmd->msgType = TSDB_MSG_TYPE_CREATE_ACCT; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCreateUserMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCreateUserMsg *pAlterMsg = (SCreateUserMsg *)pCmd->payload; - - SUserInfo *pUser = &pInfo->pMiscInfo->user; - strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n); - pAlterMsg->flag = (int8_t)pUser->type; - - if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - pAlterMsg->privilege = (char)pCmd->count; - } else if (pUser->type == TSDB_ALTER_USER_PASSWD) { - strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); - } else { // create user password info - strncpy(pAlterMsg->pass, pUser->passwd.z, pUser->passwd.n); - } - - if (pUser->type == TSDB_ALTER_USER_PASSWD || pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - pCmd->msgType = TSDB_MSG_TYPE_ALTER_USER; - } else { - pCmd->msgType = TSDB_MSG_TYPE_CREATE_USER; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildCfgDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCfgDnodeMsg); - pCmd->msgType = TSDB_MSG_TYPE_CONFIG_DNODE; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SDropDbMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pDropDbMsg->db); - assert(code == TSDB_CODE_SUCCESS && pTableMetaInfo->name.type == TSDB_DB_NAME_T); - - pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; - - pCmd->msgType = (pInfo->pMiscInfo->dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_DROP_DB : TSDB_MSG_TYPE_DROP_TP; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - pCmd->msgType = TSDB_MSG_TYPE_DROP_FUNCTION; - - pCmd->payloadLen = sizeof(SDropFuncMsg); - - return TSDB_CODE_SUCCESS; -} - - -int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SCMDropTableMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCMDropTableMsg *pDropTableMsg = (SCMDropTableMsg*)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pDropTableMsg->name); - - pDropTableMsg->supertable = (pInfo->pMiscInfo->tableType == TSDB_SUPER_TABLE)? 1:0; - pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; - pCmd->msgType = TSDB_MSG_TYPE_DROP_TABLE; - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char dnodeEp[TSDB_EP_LEN] = {0}; - tstrncpy(dnodeEp, pCmd->payload, TSDB_EP_LEN); - - pCmd->payloadLen = sizeof(SDropDnodeMsg); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload; - tstrncpy(pDrop->ep, dnodeEp, tListLen(pDrop->ep)); - pCmd->msgType = TSDB_MSG_TYPE_DROP_DNODE; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildDropUserAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char user[TSDB_USER_LEN] = {0}; - tstrncpy(user, pCmd->payload, TSDB_USER_LEN); - - pCmd->payloadLen = sizeof(SDropUserMsg); - pCmd->msgType = (pInfo->type == TSDB_SQL_DROP_USER)? TSDB_MSG_TYPE_DROP_USER:TSDB_MSG_TYPE_DROP_ACCT; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SDropUserMsg *pDropMsg = (SDropUserMsg *)pCmd->payload; - tstrncpy(pDropMsg->user, user, tListLen(user)); - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SUseDbMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pUseDbMsg->db); - pCmd->msgType = TSDB_MSG_TYPE_USE_DB; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildSyncDbReplicaMsg(SSqlObj* pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SSyncDbMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SSyncDbMsg *pSyncMsg = (SSyncDbMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pSyncMsg->db); - pCmd->msgType = TSDB_MSG_TYPE_SYNC_DB; - - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TSDB_MSG_TYPE_SHOW; - pCmd->payloadLen = sizeof(SShowMsg) + 100; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt; - SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - if (pShowInfo->showType == TSDB_MGMT_TABLE_FUNCTION) { - pShowMsg->type = pShowInfo->showType; - pShowMsg->payloadLen = 0; - pCmd->payloadLen = sizeof(SShowMsg); - - return TSDB_CODE_SUCCESS; - } - - if (tNameIsEmpty(&pTableMetaInfo->name)) { - char *p = cloneCurrentDBName(pSql); - tstrncpy(pShowMsg->db, p, sizeof(pShowMsg->db)); - tfree(p); - } else { - tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db); - } - - pShowMsg->type = pShowInfo->showType; - - if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { - SStrToken *pPattern = &pShowInfo->pattern; - if (pPattern->type > 0) { // only show tables support wildcard query - strncpy(pShowMsg->payload, pPattern->z, pPattern->n); - pShowMsg->payloadLen = htons(pPattern->n); - } - } else { - SStrToken *pEpAddr = &pShowInfo->prefix; - assert(pEpAddr->n > 0 && pEpAddr->type > 0); - - strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); - pShowMsg->payloadLen = htons(pEpAddr->n); - } - - pCmd->payloadLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen); - return TSDB_CODE_SUCCESS; -} - -int32_t tscBuildKillMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SKillQueryMsg); - - switch (pCmd->command) { - case TSDB_SQL_KILL_QUERY: - pCmd->msgType = TSDB_MSG_TYPE_KILL_QUERY; - break; - case TSDB_SQL_KILL_CONNECTION: - pCmd->msgType = TSDB_MSG_TYPE_KILL_CONN; - break; - case TSDB_SQL_KILL_STREAM: - pCmd->msgType = TSDB_MSG_TYPE_KILL_STREAM; - break; - } - return TSDB_CODE_SUCCESS; -} - -int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &(pSql->cmd); - int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg) + sizeof(SCreateTableMsg); - - SCreateTableSql *pCreateTableInfo = pInfo->pCreateTableInfo; - if (pCreateTableInfo->type == TSQL_CREATE_TABLE_FROM_STABLE) { - int32_t numOfTables = (int32_t)taosArrayGetSize(pInfo->pCreateTableInfo->childTableInfo); - size += numOfTables * (sizeof(SCreateTableMsg) + TSDB_MAX_TAGS_LEN); - } else { - size += sizeof(SSchema) * (pCmd->numOfCols + pCmd->count); - } - - if (pCreateTableInfo->pSelect != NULL) { - size += (pCreateTableInfo->pSelect->sqlstr.n + 1); - } - - return size + TSDB_EXTRA_PAYLOAD_SIZE; -} - -int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - int msgLen = 0; - int size = 0; - SSchema *pSchema; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - // Reallocate the payload size - size = tscEstimateCreateTableMsgLength(pSql, pInfo); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - tscError("0x%"PRIx64" failed to malloc for create table msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SCMCreateTableMsg *pCreateTableMsg = (SCMCreateTableMsg *)pCmd->payload; - - SCreateTableMsg* pCreateMsg = (SCreateTableMsg*)((char*) pCreateTableMsg + sizeof(SCMCreateTableMsg)); - char* pMsg = NULL; - - int8_t type = pInfo->pCreateTableInfo->type; - if (type == TSQL_CREATE_TABLE_FROM_STABLE) { // create by using super table, tags value - SArray* list = pInfo->pCreateTableInfo->childTableInfo; - - int32_t numOfTables = (int32_t) taosArrayGetSize(list); - pCreateTableMsg->numOfTables = htonl(numOfTables); - - pMsg = (char*) pCreateMsg; - for(int32_t i = 0; i < numOfTables; ++i) { - SCreateTableMsg* pCreate = (SCreateTableMsg*) pMsg; - - pCreate->numOfColumns = htons(pCmd->numOfCols); - pCreate->numOfTags = htons(pCmd->count); - pMsg += sizeof(SCreateTableMsg); - - SCreatedTableInfo* p = taosArrayGet(list, i); - strcpy(pCreate->tableName, p->fullname); - pCreate->igExists = (p->igExist)? 1 : 0; - - // use dbinfo from table id without modifying current db info - pMsg = serializeTagData(&p->tagdata, pMsg); - - int32_t len = (int32_t)(pMsg - (char*) pCreate); - pCreate->len = htonl(len); - } - } else { // create (super) table - pCreateTableMsg->numOfTables = htonl(1); // only one table will be created - - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateMsg->tableName); - assert(code == 0); - - SCreateTableSql *pCreateTable = pInfo->pCreateTableInfo; - - pCreateMsg->igExists = pCreateTable->existCheck ? 1 : 0; - pCreateMsg->numOfColumns = htons(pCmd->numOfCols); - pCreateMsg->numOfTags = htons(pCmd->count); - - pCreateMsg->sqlLen = 0; - pMsg = (char *)pCreateMsg->schema; - - pSchema = (SSchema *)pCreateMsg->schema; - - for (int i = 0; i < pCmd->numOfCols + pCmd->count; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - pSchema->type = pField->type; - strcpy(pSchema->name, pField->name); - pSchema->bytes = htons(pField->bytes); - - pSchema++; - } - - pMsg = (char *)pSchema; - if (type == TSQL_CREATE_STREAM) { // check if it is a stream sql - SSqlNode *pQuerySql = pInfo->pCreateTableInfo->pSelect; - - strncpy(pMsg, pQuerySql->sqlstr.z, pQuerySql->sqlstr.n + 1); - pCreateMsg->sqlLen = htons(pQuerySql->sqlstr.n + 1); - pMsg += pQuerySql->sqlstr.n + 1; - } - } - - tscFieldInfoClear(&pQueryInfo->fieldsInfo); - - msgLen = (int32_t)(pMsg - (char*)pCreateTableMsg); - pCreateTableMsg->contLen = htonl(msgLen); - pCmd->payloadLen = msgLen; - pCmd->msgType = TSDB_MSG_TYPE_CREATE_TABLE; - - assert(msgLen + minMsgSize() <= size); - return TSDB_CODE_SUCCESS; -} - -int tscEstimateAlterTableMsgLength(SSqlCmd *pCmd) { - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - return minMsgSize() + sizeof(SAlterTableMsg) + sizeof(SSchema) * tscNumOfFields(pQueryInfo) + TSDB_EXTRA_PAYLOAD_SIZE; -} - -int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - char *pMsg; - int msgLen = 0; - - SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SAlterTableInfo *pAlterInfo = pInfo->pAlterInfo; - int size = tscEstimateAlterTableMsgLength(pCmd); - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - tscError("0x%"PRIx64" failed to malloc for alter table msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload; - - tNameExtractFullName(&pTableMetaInfo->name, pAlterTableMsg->tableFname); - pAlterTableMsg->type = htons(pAlterInfo->type); - - pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo)); - SSchema *pSchema = pAlterTableMsg->schema; - for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - pSchema->type = pField->type; - strcpy(pSchema->name, pField->name); - pSchema->bytes = htons(pField->bytes); - pSchema++; - } - - pMsg = (char *)pSchema; - pAlterTableMsg->tagValLen = htonl(pAlterInfo->tagData.dataLen); - if (pAlterInfo->tagData.dataLen > 0) { - memcpy(pMsg, pAlterInfo->tagData.data, pAlterInfo->tagData.dataLen); - } - pMsg += pAlterInfo->tagData.dataLen; - - msgLen = (int32_t)(pMsg - (char*)pAlterTableMsg); - - pCmd->payloadLen = msgLen; - pCmd->msgType = TSDB_MSG_TYPE_ALTER_TABLE; - - assert(msgLen + minMsgSize() <= size); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { - SSqlCmd* pCmd = &pSql->cmd; - pCmd->msgType = TSDB_MSG_TYPE_UPDATE_TAG_VAL; - - SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; - pCmd->payloadLen = htonl(pUpdateMsg->head.contLen); - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - STableMeta *pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; - - SNewVgroupInfo vgroupInfo = {.vgId = -1}; - taosHashGetClone(tscVgroupMap, &pTableMeta->vgId, sizeof(pTableMeta->vgId), NULL, &vgroupInfo); - assert(vgroupInfo.vgId > 0); - - tscDumpEpSetFromVgroupInfo(&pSql->epSet, &vgroupInfo); - - return TSDB_CODE_SUCCESS; -} - -int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SAlterDbMsg); - pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_ALTER_DB : TSDB_MSG_TYPE_ALTER_TP; - - SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload; - pAlterDbMsg->dbType = -1; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - tNameExtractFullName(&pTableMetaInfo->name, pAlterDbMsg->db); - - return TSDB_CODE_SUCCESS; -} -int tscBuildCompactMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - if (pInfo->list == NULL || taosArrayGetSize(pInfo->list) <= 0) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - STscObj *pObj = pSql->pTscObj; - SSqlCmd *pCmd = &pSql->cmd; - SArray *pList = pInfo->list; - int32_t size = (int32_t)taosArrayGetSize(pList); - - int32_t *result = malloc(sizeof(int32_t) * size); - if (result == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < size; i++) { - tSqlExprItem* pSub = taosArrayGet(pList, i); - tVariant* pVar = &pSub->pNode->value; - if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) { - result[i] = (int32_t)(pVar->i64); - } else { - free(result); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } - - int count = removeDupVgid(result, size); - pCmd->payloadLen = sizeof(SCompactMsg) + count * sizeof(int32_t); - pCmd->msgType = TSDB_MSG_TYPE_COMPACT_VNODE; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - free(result); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - SCompactMsg *pCompactMsg = (SCompactMsg *)pCmd->payload; - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - if (tNameIsEmpty(&pTableMetaInfo->name)) { - pthread_mutex_lock(&pObj->mutex); - tstrncpy(pCompactMsg->db, pObj->db, sizeof(pCompactMsg->db)); - pthread_mutex_unlock(&pObj->mutex); - } else { - tNameGetFullDbName(&pTableMetaInfo->name, pCompactMsg->db); - } - - pCompactMsg->numOfVgroup = htons(count); - for (int32_t i = 0; i < count; i++) { - pCompactMsg->vgid[i] = htons(result[i]); - } - free(result); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - pCmd->payloadLen = sizeof(SRetrieveTableMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg*)pCmd->payload; - pRetrieveMsg->qId = htobe64(pSql->res.qId); - pRetrieveMsg->free = htons(pQueryInfo->type); - - return TSDB_CODE_SUCCESS; -} - -/* - * this function can only be called once. - * by using pRes->rspType to denote its status - * - * if pRes->rspType is 1, no more result - */ -static int tscLocalResultCommonBuilder(SSqlObj *pSql, int32_t numOfRes) { - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - pRes->code = TSDB_CODE_SUCCESS; - if (pRes->rspType == 0) { - pRes->numOfRows = numOfRes; - pRes->row = 0; - pRes->rspType = 1; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { - return pRes->code; - } - - tscSetResRawPtr(pRes, pQueryInfo); - } else { - tscResetForNextRetrieve(pRes); - } - - uint8_t code = pSql->res.code; - if (pSql->fp) { - if (code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, pSql->res.numOfRows); - } else { - tscAsyncResultOnError(pSql); - } - } - - return code; -} - -int tscProcessDescribeTableRsp(SSqlObj *pSql) { - SSqlCmd * pCmd = &pSql->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0); - - STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - int32_t numOfRes = tinfo.numOfColumns + tinfo.numOfTags; - return tscLocalResultCommonBuilder(pSql, numOfRes); -} - -int tscProcessLocalRetrieveRsp(SSqlObj *pSql) { - int32_t numOfRes = 1; - pSql->res.completed = true; - return tscLocalResultCommonBuilder(pSql, numOfRes); -} - -int tscProcessRetrieveGlobalMergeRsp(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - SSqlCmd* pCmd = &pSql->cmd; - - int32_t code = pRes->code; - if (pRes->code != TSDB_CODE_SUCCESS) { - tscAsyncResultOnError(pSql); - return code; - } - - if (pRes->pMerger == NULL) { // no result from subquery, so abort here directly. - (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); - return code; - } - - // global aggregation may be the upstream for parent query - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - if (pQueryInfo->pQInfo == NULL) { - STableGroupInfo tableGroupInfo = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES),}; - tableGroupInfo.map = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - - STableKeyInfo tableKeyInfo = {.pTable = NULL, .lastKey = INT64_MIN}; - - SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); - taosArrayPush(group, &tableKeyInfo); - taosArrayPush(tableGroupInfo.pGroupList, &group); - - tscDebug("0x%"PRIx64" create QInfo 0x%"PRIx64" to execute query processing", pSql->self, pSql->self); - pQueryInfo->pQInfo = createQInfoFromQueryNode(pQueryInfo, &tableGroupInfo, NULL, NULL, pRes->pMerger, MERGE_STAGE, pSql->self); - } - - uint64_t localQueryId = pSql->self; - qTableQuery(pQueryInfo->pQInfo, &localQueryId); - convertQueryResult(pRes, pQueryInfo, pSql->self, true); - - code = pRes->code; - if (pRes->code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); - } else { - tscAsyncResultOnError(pSql); - } - - return code; -} - -int tscProcessEmptyResultRsp(SSqlObj *pSql) { return tscLocalResultCommonBuilder(pSql, 0); } - -int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - STscObj *pObj = pSql->pTscObj; - SSqlCmd *pCmd = &pSql->cmd; - pCmd->msgType = TSDB_MSG_TYPE_CONNECT; - pCmd->payloadLen = sizeof(SConnectMsg); - - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { - tscError("0x%"PRIx64" failed to malloc for query msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SConnectMsg *pConnect = (SConnectMsg*)pCmd->payload; - - // TODO refactor full_name - char *db; // ugly code to move the space - - pthread_mutex_lock(&pObj->mutex); - db = strstr(pObj->db, TS_PATH_DELIMITER); - - db = (db == NULL) ? pObj->db : db + 1; - tstrncpy(pConnect->db, db, sizeof(pConnect->db)); - pthread_mutex_unlock(&pObj->mutex); - - tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion)); - tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion)); - - pConnect->pid = htonl(taosGetPId()); - taosGetCurrentAPPName(pConnect->appName, NULL); - - return TSDB_CODE_SUCCESS; -} - -/** - * multi table meta req pkg format: - * |SMultiTableInfoMsg | tableId0 | tableId1 | tableId2 | ...... - * 4B - **/ -int tscBuildMultiTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - pCmd->msgType = TSDB_MSG_TYPE_TABLES_META; - assert(pCmd->payloadLen + minMsgSize() <= pCmd->allocSize); - - tscDebug("0x%"PRIx64" build load multi-tablemeta msg completed, numOfTables:%d, msg size:%d", pSql->self, pCmd->count, - pCmd->payloadLen); - - return pCmd->payloadLen; -} - -int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char* pMsg = pCmd->payload; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - SSTableVgroupMsg *pStableVgroupMsg = (SSTableVgroupMsg *)pMsg; - pStableVgroupMsg->numOfTables = htonl(pQueryInfo->numOfTables); - pMsg += sizeof(SSTableVgroupMsg); - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, i); - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pMsg); - assert(code == TSDB_CODE_SUCCESS); - - pMsg += TSDB_TABLE_FNAME_LEN; - } - - pCmd->msgType = TSDB_MSG_TYPE_STABLE_VGROUP; - pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildRetrieveFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - - char *pMsg = pCmd->payload; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - int32_t numOfFuncs = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo); - - SRetrieveFuncMsg *pRetrieveFuncMsg = (SRetrieveFuncMsg *)pMsg; - pRetrieveFuncMsg->num = htonl(numOfFuncs); - - pMsg += sizeof(SRetrieveFuncMsg); - for(int32_t i = 0; i < numOfFuncs; ++i) { - SUdfInfo* pUdf = taosArrayGet(pQueryInfo->pUdfInfo, i); - STR_TO_NET_VARSTR(pMsg, pUdf->name); - pMsg += varDataNetTLen(pMsg); - } - - pCmd->msgType = TSDB_MSG_TYPE_RETRIEVE_FUNC; - pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload); - - return TSDB_CODE_SUCCESS; -} - -int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd *pCmd = &pSql->cmd; - STscObj *pObj = pSql->pTscObj; - - pthread_mutex_lock(&pObj->mutex); - - int32_t numOfQueries = 2; - SSqlObj *tpSql = pObj->sqlList; - while (tpSql) { - tpSql = tpSql->next; - numOfQueries++; - } - - int32_t numOfStreams = 2; - SSqlStream *pStream = pObj->streamList; - while (pStream) { - pStream = pStream->next; - numOfStreams++; - } - - int size = numOfQueries * sizeof(SQueryDesc) + numOfStreams * sizeof(SStreamDesc) + sizeof(SHeartBeatMsg) + 100; - if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { - pthread_mutex_unlock(&pObj->mutex); - tscError("0x%"PRIx64" failed to create heartbeat msg", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - // TODO the expired hb and client can not be identified by server till now. - SHeartBeatMsg *pHeartbeat = (SHeartBeatMsg *)pCmd->payload; - tstrncpy(pHeartbeat->clientVer, version, tListLen(pHeartbeat->clientVer)); - - pHeartbeat->numOfQueries = numOfQueries; - pHeartbeat->numOfStreams = numOfStreams; - - pHeartbeat->pid = htonl(taosGetPId()); - taosGetCurrentAPPName(pHeartbeat->appName, NULL); - - int msgLen = tscBuildQueryStreamDesc(pHeartbeat, pObj); - - pthread_mutex_unlock(&pObj->mutex); - - pCmd->payloadLen = msgLen; - pCmd->msgType = TSDB_MSG_TYPE_HEARTBEAT; - - return TSDB_CODE_SUCCESS; -} - -static int32_t tableMetaMsgConvert(STableMetaMsg* pMetaMsg) { - pMetaMsg->tid = htonl(pMetaMsg->tid); - pMetaMsg->sversion = htons(pMetaMsg->sversion); - pMetaMsg->tversion = htons(pMetaMsg->tversion); - pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); - - pMetaMsg->uid = htobe64(pMetaMsg->uid); - pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - - if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && - (pMetaMsg->tid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { - tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, - pMetaMsg->tid, pMetaMsg->tableFname); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (pMetaMsg->numOfTags > TSDB_MAX_TAGS) { - tscError("invalid numOfTags:%d", pMetaMsg->numOfTags); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (pMetaMsg->numOfColumns > TSDB_MAX_COLUMNS || pMetaMsg->numOfColumns <= 0) { - tscError("invalid numOfColumns:%d", pMetaMsg->numOfColumns); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - for (int i = 0; i < pMetaMsg->vgroup.numOfEps; ++i) { - pMetaMsg->vgroup.epAddr[i].port = htons(pMetaMsg->vgroup.epAddr[i].port); - } - - SSchema* pSchema = pMetaMsg->schema; - - int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags; - for (int i = 0; i < numOfTotalCols; ++i) { - pSchema->bytes = htons(pSchema->bytes); - pSchema->colId = htons(pSchema->colId); - - if (pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - assert(i == 0); - } - - pSchema++; - } - - return TSDB_CODE_SUCCESS; -} - -// update the vgroupInfo if needed -static void doUpdateVgroupInfo(int32_t vgId, SVgroupMsg *pVgroupMsg) { - assert(vgId > 0); - - SNewVgroupInfo vgroupInfo = {.inUse = -1}; - taosHashGetClone(tscVgroupMap, &vgId, sizeof(vgId), NULL, &vgroupInfo); - - // vgroup info exists, compare with it - if (((vgroupInfo.inUse >= 0) && !vgroupInfoIdentical(&vgroupInfo, pVgroupMsg)) || (vgroupInfo.inUse < 0)) { - vgroupInfo = createNewVgroupInfo(pVgroupMsg); - taosHashPut(tscVgroupMap, &vgId, sizeof(vgId), &vgroupInfo, sizeof(vgroupInfo)); - tscDebug("add/update new VgroupInfo, vgId:%d, total cached:%d", vgId, (int32_t) taosHashGetSize(tscVgroupMap)); - } -} - -static void doAddTableMetaToLocalBuf(STableMeta* pTableMeta, STableMetaMsg* pMetaMsg, bool updateSTable) { - if (pTableMeta->tableType == TSDB_CHILD_TABLE) { - // add or update the corresponding super table meta data info - int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); - - // The super tableMeta already exists, create it according to tableMeta and add it to hash map - if (updateSTable) { - STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg); - uint32_t size = tscGetTableMetaSize(pSupTableMeta); - int32_t code = taosHashPut(tscTableMetaMap, pTableMeta->sTableName, len, pSupTableMeta, size); - assert(code == TSDB_CODE_SUCCESS); - - tfree(pSupTableMeta); - } - - CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); - taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), cMeta, sizeof(CChildTableMeta)); - tfree(cMeta); - } else { - uint32_t s = tscGetTableMetaSize(pTableMeta); - taosHashPut(tscTableMetaMap, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), pTableMeta, s); - } -} - -int tscProcessTableMetaRsp(SSqlObj *pSql) { - STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; - int32_t code = tableMetaMsgConvert(pMetaMsg); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - assert(pTableMetaInfo->pTableMeta == NULL); - - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - if (pTableMeta == NULL){ - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { - tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, tNameGetTableName(&pTableMetaInfo->name)); - tfree(pTableMeta); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - assert(strncmp(pMetaMsg->tableFname, name, tListLen(pMetaMsg->tableFname)) == 0); - - doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, true); - if (pTableMeta->tableType != TSDB_SUPER_TABLE) { - doUpdateVgroupInfo(pTableMeta->vgId, &pMetaMsg->vgroup); - } - - tscDebug("0x%"PRIx64" recv table meta, uid:%" PRIu64 ", tid:%d, name:%s, numOfCols:%d, numOfTags:%d", pSql->self, - pTableMeta->id.uid, pTableMeta->id.tid, tNameGetTableName(&pTableMetaInfo->name), pTableMeta->tableInfo.numOfColumns, - pTableMeta->tableInfo.numOfTags); - - free(pTableMeta); - return TSDB_CODE_SUCCESS; -} - -static SArray* createVgroupIdListFromMsg(char* pMsg, SHashObj* pSet, char* name, int32_t* size, uint64_t id) { - SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; - - pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); - *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); - - SArray* vgroupIdList = taosArrayInit(pVgroupMsg->numOfVgroups, sizeof(int32_t)); - - if (pVgroupMsg->numOfVgroups <= 0) { - tscDebug("0x%" PRIx64 " empty vgroup id list, no corresponding tables for stable:%s", id, name); - } else { - // just init, no need to lock - for (int32_t j = 0; j < pVgroupMsg->numOfVgroups; ++j) { - SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; - vmsg->vgId = htonl(vmsg->vgId); - for (int32_t k = 0; k < vmsg->numOfEps; ++k) { - vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); - } - - taosArrayPush(vgroupIdList, &vmsg->vgId); - - if (taosHashGet(pSet, &vmsg->vgId, sizeof(vmsg->vgId)) == NULL) { - taosHashPut(pSet, &vmsg->vgId, sizeof(vmsg->vgId), "", 0); - doUpdateVgroupInfo(vmsg->vgId, vmsg); - } - } - } - - return vgroupIdList; -} - -static SVgroupsInfo* createVgroupInfoFromMsg(char* pMsg, int32_t* size, uint64_t id) { - SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)pMsg; - pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); - - *size = (int32_t)(sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg)); - - size_t vgroupsz = sizeof(SVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); - SVgroupsInfo *pVgroupInfo = calloc(1, vgroupsz); - assert(pVgroupInfo != NULL); - - pVgroupInfo->numOfVgroups = pVgroupMsg->numOfVgroups; - if (pVgroupInfo->numOfVgroups <= 0) { - tscDebug("0x%" PRIx64 " empty vgroup info, no corresponding tables for stable", id); - } else { - for (int32_t j = 0; j < pVgroupInfo->numOfVgroups; ++j) { - // just init, no need to lock - SVgroupMsg *pVgroup = &pVgroupInfo->vgroups[j]; - - SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; - vmsg->vgId = htonl(vmsg->vgId); - for (int32_t k = 0; k < vmsg->numOfEps; ++k) { - vmsg->epAddr[k].port = htons(vmsg->epAddr[k].port); - } - - pVgroup->numOfEps = vmsg->numOfEps; - pVgroup->vgId = vmsg->vgId; - for (int32_t k = 0; k < vmsg->numOfEps; ++k) { - pVgroup->epAddr[k].port = vmsg->epAddr[k].port; - tstrncpy(pVgroup->epAddr[k].fqdn, vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); -// pVgroup->epAddr[k].fqdn = strndup(vmsg->epAddr[k].fqdn, TSDB_FQDN_LEN); - } - - doUpdateVgroupInfo(pVgroup->vgId, vmsg); - } - } - - return pVgroupInfo; -} - -int tscProcessRetrieveFuncRsp(SSqlObj* pSql) { - SSqlCmd* pCmd = &pSql->cmd; - SUdfFuncMsg* pFuncMsg = (SUdfFuncMsg *)pSql->res.pRsp; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - pFuncMsg->num = htonl(pFuncMsg->num); - assert(pFuncMsg->num == taosArrayGetSize(pQueryInfo->pUdfInfo)); - - char* pMsg = pFuncMsg->content; - for(int32_t i = 0; i < pFuncMsg->num; ++i) { - SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; - - for(int32_t j = 0; j < pFuncMsg->num; ++j) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); - if (strcmp(pUdfInfo->name, pFunc->name) != 0) { - continue; - } - - if (pUdfInfo->content) { - continue; - } - - pUdfInfo->resBytes = htons(pFunc->resBytes); - pUdfInfo->resType = pFunc->resType; - pUdfInfo->funcType = htonl(pFunc->funcType); - pUdfInfo->contLen = htonl(pFunc->len); - pUdfInfo->bufSize = htonl(pFunc->bufSize); - - pUdfInfo->content = malloc(pUdfInfo->contLen); - memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); - - pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; - } - } - - // master sqlObj locates in param - SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); - if(parent == NULL) { - return pSql->res.code; - } - - SQueryInfo* parQueryInfo = tscGetQueryInfo(&parent->cmd); - - assert(parent->signature == parent && (int64_t)pSql->param == parent->self); - taosArrayDestroy(parQueryInfo->pUdfInfo); - - parQueryInfo->pUdfInfo = pQueryInfo->pUdfInfo; // assigned to parent sql obj. - pQueryInfo->pUdfInfo = NULL; - return TSDB_CODE_SUCCESS; -} - -int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { - char *rsp = pSql->res.pRsp; - - SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp; - pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables); - pMultiMeta->numOfVgroup = htonl(pMultiMeta->numOfVgroup); - pMultiMeta->numOfUdf = htonl(pMultiMeta->numOfUdf); - - rsp += sizeof(SMultiTableMeta); - - SSqlObj* pParentSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); - if(pParentSql == NULL) { - return pSql->res.code; - } - - SSqlCmd *pParentCmd = &pParentSql->cmd; - SHashObj *pSet = taosHashInit(pMultiMeta->numOfVgroup, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - - char* buf = NULL; - char* pMsg = pMultiMeta->meta; - - // decompresss the message payload - if (pMultiMeta->compressed) { - buf = malloc(pMultiMeta->rawLen - sizeof(SMultiTableMeta)); - int32_t len = tsDecompressString(pMultiMeta->meta, pMultiMeta->contLen - sizeof(SMultiTableMeta), 1, - buf, pMultiMeta->rawLen - sizeof(SMultiTableMeta), ONE_STAGE_COMP, NULL, 0); - assert(len == pMultiMeta->rawLen - sizeof(SMultiTableMeta)); - - pMsg = buf; - } - - if (pParentCmd->pTableMetaMap == NULL) { - pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - } - - for (int32_t i = 0; i < pMultiMeta->numOfTables; i++) { - STableMetaMsg *pMetaMsg = (STableMetaMsg *)pMsg; - int32_t code = tableMetaMsgConvert(pMetaMsg); - if (code != TSDB_CODE_SUCCESS) { - taosHashCleanup(pSet); - taosReleaseRef(tscObjRef, pParentSql->self); - - tfree(buf); - return code; - } - - bool freeMeta = false; - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { - tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, pMetaMsg->tableFname); - tfree(pTableMeta); - taosHashCleanup(pSet); - taosReleaseRef(tscObjRef, pParentSql->self); - - tfree(buf); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (pMultiMeta->metaClone == 1 || pTableMeta->tableType == TSDB_SUPER_TABLE) { - STableMetaVgroupInfo p = {.pTableMeta = pTableMeta,}; - size_t keyLen = strnlen(pMetaMsg->tableFname, TSDB_TABLE_FNAME_LEN); - void* t = taosHashGet(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen); - assert(t == NULL); - - taosHashPut(pParentCmd->pTableMetaMap, pMetaMsg->tableFname, keyLen, &p, sizeof(STableMetaVgroupInfo)); - } else { - freeMeta = true; - } - - // for each super table, only update meta information once - bool updateStableMeta = false; - if (pTableMeta->tableType == TSDB_CHILD_TABLE && taosHashGet(pSet, &pMetaMsg->suid, sizeof(pMetaMsg->suid)) == NULL) { - updateStableMeta = true; - taosHashPut(pSet, &pTableMeta->suid, sizeof(pMetaMsg->suid), "", 0); - } - - // create the tableMeta and add it into the TableMeta map - doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, updateStableMeta); - - // for each vgroup, only update the information once. - int64_t vgId = pMetaMsg->vgroup.vgId; - if (pTableMeta->tableType != TSDB_SUPER_TABLE && taosHashGet(pSet, &vgId, sizeof(vgId)) == NULL) { - doUpdateVgroupInfo((int32_t) vgId, &pMetaMsg->vgroup); - taosHashPut(pSet, &vgId, sizeof(vgId), "", 0); - } - - pMsg += pMetaMsg->contLen; - if (freeMeta) { - tfree(pTableMeta); - } - } - - for(int32_t i = 0; i < pMultiMeta->numOfVgroup; ++i) { - char fname[TSDB_TABLE_FNAME_LEN] = {0}; - tstrncpy(fname, pMsg, TSDB_TABLE_FNAME_LEN); - size_t len = strnlen(fname, TSDB_TABLE_FNAME_LEN); - - pMsg += TSDB_TABLE_FNAME_LEN; - - STableMetaVgroupInfo* p = taosHashGet(pParentCmd->pTableMetaMap, fname, len); - assert(p != NULL); - - int32_t size = 0; - if (p->vgroupIdList!= NULL) { - taosArrayDestroy(p->vgroupIdList); - } - - p->vgroupIdList = createVgroupIdListFromMsg(pMsg, pSet, fname, &size, pSql->self); - - int32_t numOfVgId = (int32_t) taosArrayGetSize(p->vgroupIdList); - int32_t s = sizeof(tFilePage) + numOfVgId * sizeof(int32_t); - - tFilePage* idList = calloc(1, s); - idList->num = numOfVgId; - memcpy(idList->data, TARRAY_GET_START(p->vgroupIdList), numOfVgId * sizeof(int32_t)); - - void* idListInst = taosCachePut(tscVgroupListBuf, fname, len, idList, s, 5000); - taosCacheRelease(tscVgroupListBuf, (void*) &idListInst, false); - - tfree(idList); - pMsg += size; - } - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pParentCmd); - if (pMultiMeta->numOfUdf > 0) { - assert(pQueryInfo->pUdfInfo != NULL); - } - - for(int32_t i = 0; i < pMultiMeta->numOfUdf; ++i) { - SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg; - - for(int32_t j = 0; j < pMultiMeta->numOfUdf; ++j) { - SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j); - if (strcmp(pUdfInfo->name, pFunc->name) != 0) { - continue; - } - - if (pUdfInfo->content) { - continue; - } - - pUdfInfo->resBytes = htons(pFunc->resBytes); - pUdfInfo->resType = pFunc->resType; - pUdfInfo->funcType = htonl(pFunc->funcType); - pUdfInfo->contLen = htonl(pFunc->len); - pUdfInfo->bufSize = htonl(pFunc->bufSize); - - pUdfInfo->content = malloc(pUdfInfo->contLen); - memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen); - - pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen; - } - } - - pSql->res.code = TSDB_CODE_SUCCESS; - pSql->res.numOfTotal = pMultiMeta->numOfTables; - tscDebug("0x%"PRIx64" load multi-tableMeta from mnode, numOfTables:%d", pSql->self, pMultiMeta->numOfTables); - - taosHashCleanup(pSet); - taosReleaseRef(tscObjRef, pParentSql->self); - - tfree(buf); - return TSDB_CODE_SUCCESS; -} - -int tscProcessSTableVgroupRsp(SSqlObj *pSql) { - // master sqlObj locates in param - SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param); - if(parent == NULL) { - return pSql->res.code; - } - - assert(parent->signature == parent && (int64_t)pSql->param == parent->self); - - SSqlRes* pRes = &pSql->res; - - // NOTE: the order of several table must be preserved. - SSTableVgroupRspMsg *pStableVgroup = (SSTableVgroupRspMsg *)pRes->pRsp; - pStableVgroup->numOfTables = htonl(pStableVgroup->numOfTables); - char *pMsg = pRes->pRsp + sizeof(SSTableVgroupRspMsg); - - SSqlCmd* pCmd = &parent->cmd; - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - - char fName[TSDB_TABLE_FNAME_LEN] = {0}; - for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) { - char* name = pMsg; - pMsg += TSDB_TABLE_FNAME_LEN; - - STableMetaInfo *pInfo = NULL; - for(int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { - STableMetaInfo *pInfo1 = tscGetTableMetaInfoFromCmd(pCmd, j); - memset(fName, 0, tListLen(fName)); - - tNameExtractFullName(&pInfo1->name, fName); - if (strcmp(name, fName) != 0) { - continue; - } - - pInfo = pInfo1; - break; - } - - if (!pInfo){ - continue; - } - int32_t size = 0; - pInfo->vgroupList = createVgroupInfoFromMsg(pMsg, &size, pSql->self); - pMsg += size; - } - - taosReleaseRef(tscObjRef, parent->self); - return pSql->res.code; -} - -int tscProcessShowRsp(SSqlObj *pSql) { - STableMetaMsg *pMetaMsg; - SShowRsp * pShow; - SSchema * pSchema; - - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - pShow = (SShowRsp *)pRes->pRsp; - pShow->qhandle = htobe64(pShow->qhandle); - pRes->qId = pShow->qhandle; - - tscResetForNextRetrieve(pRes); - pMetaMsg = &(pShow->tableMeta); - - pMetaMsg->numOfColumns = ntohs(pMetaMsg->numOfColumns); - - pSchema = pMetaMsg->schema; - pMetaMsg->tid = ntohs(pMetaMsg->tid); - for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { - pSchema->bytes = htons(pSchema->bytes); - pSchema++; - } - - tfree(pTableMetaInfo->pTableMeta); - pTableMetaInfo->pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); - - SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - if (pQueryInfo->colList == NULL) { - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - } - - SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; - - SColumnIndex index = {0}; - pSchema = pMetaMsg->schema; - - uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; - for (int16_t i = 0; i < pMetaMsg->numOfColumns; ++i, ++pSchema) { - index.columnIndex = i; - tscColumnListInsert(pQueryInfo->colList, i, uid, pSchema); - - TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); - SInternalField* pInfo = tscFieldInfoAppend(pFieldInfo, &f); - - pInfo->pExpr = tscExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, - pTableSchema[i].type, pTableSchema[i].bytes, getNewResColId(pCmd), pTableSchema[i].bytes, false); - } - - pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - tscFieldInfoUpdateOffset(pQueryInfo); - return 0; -} - -static void createHbObj(STscObj* pObj) { - if (pObj->hbrid != 0) { - return; - } - - SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); - if (NULL == pSql) return; - - pSql->fp = tscProcessHeartBeatRsp; - - SQueryInfo *pQueryInfo = tscGetQueryInfoS(&pSql->cmd); - if (pQueryInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tfree(pSql); - return; - } - - pQueryInfo->command = TSDB_SQL_HB; - - pSql->cmd.command = pQueryInfo->command; - if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - tfree(pSql); - return; - } - - pSql->param = pObj; - pSql->pTscObj = pObj; - pSql->signature = pSql; - - registerSqlObj(pSql); - tscDebug("0x%"PRIx64" HB is allocated, pObj:%p", pSql->self, pObj); - - pObj->hbrid = pSql->self; -} - -int tscProcessUseDbRsp(SSqlObj *pSql) { - STscObj * pObj = pSql->pTscObj; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - - pthread_mutex_lock(&pObj->mutex); - int ret = tNameExtractFullName(&pTableMetaInfo->name, pObj->db); - pthread_mutex_unlock(&pObj->mutex); - - return ret; -} - -//todo only invalid the buffered data that belongs to dropped databases -int tscProcessDropDbRsp(SSqlObj *pSql) { - //TODO LOCK DB WHEN MODIFY IT - //pSql->pTscObj->db[0] = 0; - - taosHashClear(tscTableMetaMap); - taosHashClear(tscVgroupMap); - taosCacheEmpty(tscVgroupListBuf); - return 0; -} - -int tscProcessDropTableRsp(SSqlObj *pSql) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - tscRemoveCachedTableMeta(pTableMetaInfo, pSql->self); - tfree(pTableMetaInfo->pTableMeta); - return 0; -} - -int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0); - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - - tscDebug("0x%"PRIx64" remove tableMeta in hashMap after alter-table: %s", pSql->self, name); - - bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - tfree(pTableMetaInfo->pTableMeta); - - if (isSuperTable) { // if it is a super table, iterate the hashTable and remove all the childTableMeta - if (pSql->res.pRsp == NULL) { - tscDebug("0x%"PRIx64" unexpected resp from mnode, super table: %s failed to update super table meta ", pSql->self, name); - return 0; - } - return tscProcessTableMetaRsp(pSql); - } - - return 0; -} - -int tscProcessAlterDbMsgRsp(SSqlObj *pSql) { - UNUSED(pSql); - return 0; -} -int tscProcessCompactRsp(SSqlObj *pSql) { - UNUSED(pSql); - return TSDB_CODE_SUCCESS; -} - -int tscProcessShowCreateRsp(SSqlObj *pSql) { - return tscLocalResultCommonBuilder(pSql, 1); -} - -int tscProcessQueryRsp(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - - SQueryTableRsp *pQueryAttr = (SQueryTableRsp *)pRes->pRsp; - pQueryAttr->qId = htobe64(pQueryAttr->qId); - - pRes->qId = pQueryAttr->qId; - pRes->data = NULL; - - tscResetForNextRetrieve(pRes); - tscDebug("0x%"PRIx64" query rsp received, qId:0x%"PRIx64, pSql->self, pRes->qId); - return 0; -} - -static void decompressQueryColData(SSqlObj *pSql, SSqlRes *pRes, SQueryInfo* pQueryInfo, char **data, int8_t compressed, int32_t compLen) { - int32_t decompLen = 0; - int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - int32_t *compSizes; - char *pData = *data; - compSizes = (int32_t *)(pData + compLen); - - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); - int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); - char *outputBuf = tcalloc(pRes->numOfRows, (pField->bytes + offset)); - - char *p = outputBuf; - int32_t bufOffset; - for (int32_t i = 0; i < numOfCols; ++i) { - SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); - bufOffset = pInfo->field.bytes * pRes->numOfRows; - - int32_t flen = (*(tDataTypes[pInfo->field.type].decompFunc))(pData, htonl(compSizes[i]), pRes->numOfRows, p, bufOffset, - compressed, NULL, 0); - - p += flen; - decompLen +=flen; - pData += htonl(compSizes[i]); - } - - /* Resize rsp as decompressed data will occupy more space */ - pRes->rspLen = pRes->rspLen - (compLen + numOfCols * sizeof(int32_t)) + decompLen; - char *new_rsp = (char *)realloc(pRes->pRsp, pRes->rspLen); - if (new_rsp == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return; - } else { - pRes->pRsp = new_rsp; - *data = ((SRetrieveTableRsp *)pRes->pRsp)->data; - pData = *data + compLen + numOfCols * sizeof(int32_t); - } - - tscDebug("0x%"PRIx64" decompress col data, compressed size:%d, decompressed size:%d", - pSql->self, (int32_t)(compLen + numOfCols * sizeof(int32_t)), decompLen); - - int32_t tailLen = pRes->rspLen - sizeof(SRetrieveTableRsp) - decompLen; - memmove(*data + decompLen, pData, tailLen); - memmove(*data, outputBuf, decompLen); - - tfree(outputBuf); -} - -int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - assert(pRes->rspLen >= sizeof(SRetrieveTableRsp)); - - SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *)pRes->pRsp; - if (pRetrieve == NULL) { - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; - } - - pRes->numOfRows = htonl(pRetrieve->numOfRows); - pRes->precision = htons(pRetrieve->precision); - pRes->offset = htobe64(pRetrieve->offset); - pRes->useconds = htobe64(pRetrieve->useconds); - pRes->completed = (pRetrieve->completed == 1); - pRes->data = pRetrieve->data; - - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - if (tscCreateResPointerInfo(pRes, pQueryInfo) != TSDB_CODE_SUCCESS) { - return pRes->code; - } - - //Decompress col data if compressed from server - if (pRetrieve->compressed) { - int32_t compLen = htonl(pRetrieve->compLen); - decompressQueryColData(pSql, pRes, pQueryInfo, &pRes->data, pRetrieve->compressed, compLen); - } - - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if ((pCmd->command == TSDB_SQL_RETRIEVE_MNODE) || - ((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) || - (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY) && - !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE))) { - tscSetResRawPtr(pRes, pQueryInfo); - } - - if (pSql->pSubscription != NULL) { - int32_t numOfCols = pQueryInfo->fieldsInfo.numOfOutput; - - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, numOfCols - 1); - int16_t offset = tscFieldInfoGetOffset(pQueryInfo, numOfCols - 1); - - char* p = pRes->data + (pField->bytes + offset) * pRes->numOfRows; - - int32_t numOfTables = htonl(*(int32_t*)p); - p += sizeof(int32_t); - for (int i = 0; i < numOfTables; i++) { - int64_t uid = htobe64(*(int64_t*)p); - p += sizeof(int64_t); - p += sizeof(int32_t); // skip tid - TSKEY key = htobe64(*(TSKEY*)p); - p += sizeof(TSKEY); - tscUpdateSubscriptionProgress(pSql->pSubscription, uid, key); - } - } - - pRes->row = 0; - tscDebug("0x%"PRIx64" numOfRows:%d, offset:%" PRId64 ", complete:%d, qId:0x%"PRIx64, pSql->self, pRes->numOfRows, pRes->offset, - pRes->completed, pRes->qId); - - return 0; -} - -void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); - -static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool autocreate) { - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - if (NULL == pNew) { - tscError("0x%"PRIx64" malloc failed for new sqlobj to get table meta", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_META; - - tscAddQueryInfo(&pNew->cmd); - - SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { - tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); - - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - STableMetaInfo *pNewTableMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); - assert(pNewQueryInfo->numOfTables == 1); - - tNameAssign(&pNewTableMetaInfo->name, &pTableMetaInfo->name); - - registerSqlObj(pNew); - - pNew->fp = tscTableMetaCallBack; - pNew->param = (void *)pSql->self; - - tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get tableMeta, auto create:%d, metaRid from %"PRId64" to %"PRId64, - pSql->self, pNew->self, autocreate, pSql->metaRid, pNew->self); - pSql->metaRid = pNew->self; - - { - STableInfoMsg *pInfoMsg = (STableInfoMsg *)pNew->cmd.payload; - int32_t code = tNameExtractFullName(&pNewTableMetaInfo->name, pInfoMsg->tableFname); - if (code != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - pInfoMsg->createFlag = htons(autocreate? 1 : 0); - char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); - - // tag data exists - if (autocreate && pSql->cmd.insertParam.tagData.dataLen != 0) { - pMsg = serializeTagData(&pSql->cmd.insertParam.tagData, pMsg); - } - - pNew->cmd.payloadLen = (int32_t)(pMsg - (char*)pInfoMsg); - pNew->cmd.msgType = TSDB_MSG_TYPE_TABLE_META; - } - - int32_t code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated - } - - return code; -} - -int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone) { - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - if (NULL == pNew) { - tscError("0x%"PRIx64" failed to allocate sqlobj to get multiple table meta", pSql->self); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_MULTI_META; - - int32_t numOfTable = (int32_t) taosArrayGetSize(pNameList); - int32_t numOfVgroupList = (int32_t) taosArrayGetSize(pVgroupNameList); - int32_t numOfUdf = pUdfList ? (int32_t)taosArrayGetSize(pUdfList) : 0; - - int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + TSDB_FUNC_NAME_LEN * numOfUdf + sizeof(SMultiTableInfoMsg); - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, size)) { - tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self); - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SMultiTableInfoMsg* pInfo = (SMultiTableInfoMsg*) pNew->cmd.payload; - pInfo->metaClone = metaClone? 1:0; - pInfo->numOfTables = htonl((uint32_t) taosArrayGetSize(pNameList)); - pInfo->numOfVgroups = htonl((uint32_t) taosArrayGetSize(pVgroupNameList)); - pInfo->numOfUdfs = htonl(numOfUdf); - - char* start = pInfo->tableNames; - int32_t len = 0; - for(int32_t i = 0; i < numOfTable; ++i) { - char* name = taosArrayGetP(pNameList, i); - if (i < numOfTable - 1 || numOfVgroupList > 0 || numOfUdf > 0) { - len = sprintf(start, "%s,", name); - } else { - len = sprintf(start, "%s", name); - } - - start += len; - } - - for(int32_t i = 0; i < numOfVgroupList; ++i) { - char* name = taosArrayGetP(pVgroupNameList, i); - if (i < numOfVgroupList - 1 || numOfUdf > 0) { - len = sprintf(start, "%s,", name); - } else { - len = sprintf(start, "%s", name); - } - - start += len; - } - - for(int32_t i = 0; i < numOfUdf; ++i) { - SUdfInfo * u = taosArrayGet(pUdfList, i); - if (i < numOfUdf - 1) { - len = sprintf(start, "%s,", u->name); - } else { - len = sprintf(start, "%s", u->name); - } - - start += len; - } - - pNew->cmd.payloadLen = (int32_t) ((start - pInfo->tableNames) + sizeof(SMultiTableInfoMsg)); - pNew->cmd.msgType = TSDB_MSG_TYPE_TABLES_META; - - registerSqlObj(pNew); - tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, udf:%d, msg size:%d", pSql->self, - pNew->self, numOfTable, numOfVgroupList, numOfUdf, pNew->cmd.payloadLen); - - pNew->fp = fp; - pNew->param = (void *)pSql->self; - - tscDebug("0x%"PRIx64" metaRid from 0x%" PRIx64 " to 0x%" PRIx64 , pSql->self, pSql->metaRid, pNew->self); - - pSql->metaRid = pNew->self; - int32_t code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated - } - - return code; -} - -int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) { - assert(tNameIsValid(&pTableMetaInfo->name)); - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - - size_t len = strlen(name); - // just make runtime happy - if (pTableMetaInfo->tableMetaCapacity != 0 && pTableMetaInfo->pTableMeta != NULL) { - memset(pTableMetaInfo->pTableMeta, 0, pTableMetaInfo->tableMetaCapacity); - } - - if (NULL == taosHashGetCloneExt(tscTableMetaMap, name, len, NULL, (void **)&(pTableMetaInfo->pTableMeta), &pTableMetaInfo->tableMetaCapacity)) { - tfree(pTableMetaInfo->pTableMeta); - } - - STableMeta* pMeta = pTableMetaInfo->pTableMeta; - STableMeta* pSTMeta = (STableMeta *)(pSql->pBuf); - - if (pMeta && pMeta->id.uid > 0) { - // in case of child table, here only get the - if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromSTableMeta(&pTableMetaInfo->pTableMeta, name, &pTableMetaInfo->tableMetaCapacity, (STableMeta **)(&pSTMeta)); - pSql->pBuf = (void *)(pSTMeta); - if (code != TSDB_CODE_SUCCESS) { - return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); - } - } - - tscDebug("0x%"PRIx64 " %s retrieve tableMeta from cache, numOfCols:%d, numOfTags:%d", pSql->self, name, pMeta->tableInfo.numOfColumns, pMeta->tableInfo.numOfTags); - return TSDB_CODE_SUCCESS; - } - - if (onlyLocal) { - return TSDB_CODE_TSC_NO_META_CACHED; - } - - return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate); -} - -int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { - return tscGetTableMetaImpl(pSql, pTableMetaInfo, false, false); -} - -int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool createIfNotExists, bool onlyLocal) { - return tscGetTableMetaImpl(pSql, pTableMetaInfo, createIfNotExists, onlyLocal); -} - -int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) { - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - if (NULL == pNew) { - tscError("%p malloc failed for new sqlobj to get user-defined functions", pSql); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - pNew->cmd.command = TSDB_SQL_RETRIEVE_FUNC; - - if (tscAddQueryInfo(&pNew->cmd) != TSDB_CODE_SUCCESS) { - tscError("%p malloc failed for new queryinfo", pSql); - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd); - - pNewQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(SUdfInfo)); - for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) { - SUdfInfo info = {0}; - SUdfInfo* p1 = taosArrayGet(pQueryInfo->pUdfInfo, i); - info = *p1; - info.name = strdup(p1->name); - taosArrayPush(pNewQueryInfo->pUdfInfo, &info); - } - - pNew->cmd.active = pNewQueryInfo; - - if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { - tscError("%p malloc failed for payload to get table meta", pSql); - tscFreeSqlObj(pNew); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - tscDebug("%p new pSqlObj:%p to retrieve udf", pSql, pNew); - registerSqlObj(pNew); - - pNew->fp = tscTableMetaCallBack; - pNew->param = (void *)pSql->self; - - tscDebug("%p metaRid from %" PRId64 " to %" PRId64 , pSql, pSql->metaRid, pNew->self); - - pSql->metaRid = pNew->self; - - int32_t code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated - } - - return code; -} - -static void freeElem(void* p) { - tfree(*(char**)p); -} - -/** - * retrieve table meta from mnode, and then update the local table meta hashmap. - * @param pSql sql object - * @param tableIndex table index - * @return status code - */ -int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { - SSqlCmd* pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - - char name[TSDB_TABLE_FNAME_LEN] = {0}; - int32_t code = tNameExtractFullName(&pTableMetaInfo->name, name); - if (code != TSDB_CODE_SUCCESS) { - tscError("0x%"PRIx64" failed to generate the table full name", pSql->self); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - if (pTableMeta) { - tscDebug("0x%"PRIx64" update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRIu64, pSql->self, name, - tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); - } - - - // remove stored tableMeta info in hash table - tscResetSqlCmd(pCmd, true, pSql->self); - - SArray* pNameList = taosArrayInit(1, POINTER_BYTES); - SArray* vgroupList = taosArrayInit(1, POINTER_BYTES); - - char* n = strdup(name); - taosArrayPush(pNameList, &n); - code = getMultiTableMetaFromMnode(pSql, pNameList, vgroupList, NULL, tscTableMetaCallBack, true); - taosArrayDestroyEx(pNameList, freeElem); - taosArrayDestroyEx(vgroupList, freeElem); - - return code; -} - -static bool allVgroupInfoRetrieved(SQueryInfo* pQueryInfo) { - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - if (pTableMetaInfo->vgroupList == NULL) { - return false; - } - } - - // all super tables vgroupinfo are retrieved, no need to retrieve vgroup info anymore - return true; -} - -int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) { - int32_t code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - if (allVgroupInfoRetrieved(pQueryInfo)) { - return TSDB_CODE_SUCCESS; - } - SSqlObj *pNew = calloc(1, sizeof(SSqlObj)); - pNew->pTscObj = pSql->pTscObj; - pNew->signature = pNew; - - pNew->cmd.command = TSDB_SQL_STABLEVGROUP; - - // TODO TEST IT - SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd); - if (pNewQueryInfo == NULL) { - tscFreeSqlObj(pNew); - return code; - } - - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); - STableMeta* pTableMeta = tscTableMetaDup(pMInfo->pTableMeta); - tscAddTableMetaInfo(pNewQueryInfo, &pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); - } - - if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pNew); - return code; - } - - pNewQueryInfo->numOfTables = pQueryInfo->numOfTables; - registerSqlObj(pNew); - - tscDebug("0x%"PRIx64" svgroupRid from %" PRId64 " to %" PRId64 , pSql->self, pSql->svgroupRid, pNew->self); - - pSql->svgroupRid = pNew->self; - tscDebug("0x%"PRIx64" new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql->self, pNew, pNewQueryInfo->numOfTables); - - pNew->fp = tscTableMetaCallBack; - pNew->param = (void *)pSql->self; - code = tscBuildAndSendRequest(pNew, NULL); - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; - } - - return code; -} +#include "os.h" +#include "tdef.h" +#include "tname.h" +#include "clientInt.h" +#include "clientLog.h" +#include "tmsgtype.h" +#include "trpc.h" -#endif +int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody) = {0}; +int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { pMsgBody->msgType = TSDB_MSG_TYPE_CONNECT; - pMsgBody->msgLen = sizeof(SConnectMsg); + pMsgBody->msgInfo.len = sizeof(SConnectMsg); pMsgBody->requestObjRefId = pRequest->self; SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg)); @@ -3119,7 +50,7 @@ int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { pConnect->startTime = htobe64(appInfo.startTime); tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app)); - pMsgBody->pData = pConnect; + pMsgBody->msgInfo.pMsg = pConnect; return 0; } @@ -3160,26 +91,32 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { pTscObj->pAppInfo->clusterId = pConnect->clusterId; atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); - pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo)); - pRequest->body.pResInfo->pMsg = pMsg; - + pRequest->body.resInfo.pRspMsg = pMsg; tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns); return 0; } int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { pMsgBody->requestObjRefId = pRequest->self; - pMsgBody->msgLen = pRequest->body.paramLen; - pMsgBody->pData = pRequest->body.param; + pMsgBody->msgInfo = pRequest->body.requestMsg; switch(pRequest->type) { case TSDB_SQL_CREATE_USER: pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER; break; + case TSDB_SQL_DROP_USER: + pMsgBody->msgType = TSDB_MSG_TYPE_DROP_USER; + break; + case TSDB_SQL_CREATE_ACCT: + pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_ACCT; + break; + case TSDB_SQL_DROP_ACCT: + pMsgBody->msgType = TSDB_MSG_TYPE_DROP_ACCT; + break; case TSDB_SQL_CREATE_DB: { pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_DB; - SCreateDbMsg* pCreateMsg = pRequest->body.param; + SCreateDbMsg* pCreateMsg = pRequest->body.requestMsg.pMsg; SName name = {0}; int32_t ret = tNameSetDbName(&name, pRequest->pTscObj->acctId, pCreateMsg->db, strnlen(pCreateMsg->db, tListLen(pCreateMsg->db))); if (ret != TSDB_CODE_SUCCESS) { @@ -3187,7 +124,14 @@ int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { } tNameGetFullDbName(&name, pCreateMsg->db); - + break; + } + case TSDB_SQL_USE_DB: { + pMsgBody->msgType = TSDB_MSG_TYPE_USE_DB; + break; + } + case TSDB_SQL_CREATE_TABLE: { + pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_STB; break; } case TSDB_SQL_SHOW: @@ -3196,36 +140,6 @@ int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { } } -STableMeta* createTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { - assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2); - - size_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema); - STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize); - - pTableMeta->tableType = pTableMetaMsg->tableType; - pTableMeta->vgId = pTableMetaMsg->vgId; - pTableMeta->suid = pTableMetaMsg->suid; - pTableMeta->uid = pTableMetaMsg->tuid; - - pTableMeta->tableInfo = (STableComInfo) { - .numOfTags = pTableMetaMsg->numOfTags, - .precision = pTableMetaMsg->precision, - .numOfColumns = pTableMetaMsg->numOfColumns, - }; - - pTableMeta->sversion = pTableMetaMsg->sversion; - pTableMeta->tversion = pTableMetaMsg->tversion; - - memcpy(pTableMeta->schema, pTableMetaMsg->pSchema, schemaSize); - - int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns; - for(int32_t i = 0; i < numOfTotalCols; ++i) { - pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; - } - - return pTableMeta; -} - int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { SShowRsp* pShow = (SShowRsp *)pMsg; pShow->showId = htonl(pShow->showId); @@ -3248,12 +162,8 @@ int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) pFields[i].bytes = pSchema[i].bytes; } - if (pRequest->body.pResInfo == NULL) { - pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo)); - } - - pRequest->body.pResInfo->pMsg = pMsg; - SClientResultInfo* pResInfo = pRequest->body.pResInfo; + pRequest->body.resInfo.pRspMsg = pMsg; + SReqResultInfo* pResInfo = &pRequest->body.resInfo; pResInfo->fields = pFields; pResInfo->numOfCols = pMetaMsg->numOfColumns; @@ -3267,27 +177,27 @@ int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) int buildRetrieveMnodeMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { pMsgBody->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - pMsgBody->msgLen = sizeof(SRetrieveTableMsg); + pMsgBody->msgInfo.len = sizeof(SRetrieveTableMsg); pMsgBody->requestObjRefId = pRequest->self; SRetrieveTableMsg *pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg)); pRetrieveMsg->showId = htonl(pRequest->body.execId); - pMsgBody->pData = pRetrieveMsg; + pMsgBody->msgInfo.pMsg = pRetrieveMsg; return TSDB_CODE_SUCCESS; } int32_t processRetrieveMnodeRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { assert(msgLen >= sizeof(SRetrieveTableRsp)); - tfree(pRequest->body.pResInfo->pMsg); - pRequest->body.pResInfo->pMsg = pMsg; + tfree(pRequest->body.resInfo.pRspMsg); + pRequest->body.resInfo.pRspMsg = pMsg; SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *) pMsg; pRetrieve->numOfRows = htonl(pRetrieve->numOfRows); pRetrieve->precision = htons(pRetrieve->precision); - SClientResultInfo* pResInfo = pRequest->body.pResInfo; + SReqResultInfo* pResInfo = &pRequest->body.resInfo; pResInfo->numOfRows = pRetrieve->numOfRows; pResInfo->pData = pRetrieve->data; // todo fix this in async model @@ -3303,6 +213,20 @@ int32_t processCreateDbRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgL // todo rsp with the vnode id list } +int32_t processUseDbRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + SUseDbRsp* pUseDbRsp = (SUseDbRsp*) pMsg; + SName name = {0}; + tNameFromString(&name, pUseDbRsp->db, T_NAME_ACCT|T_NAME_DB); + + char db[TSDB_DB_NAME_LEN] = {0}; + tNameGetDbName(&name, db); + setConnectionDB(pRequest->pTscObj, db); +} + +int32_t processCreateTableRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + assert(pMsg != NULL); +} + void initMsgHandleFp() { #if 0 tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; @@ -3328,7 +252,7 @@ void initMsgHandleFp() { tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; - tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAG_VAL] = tscBuildUpdateTagMsg; tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; @@ -3383,6 +307,10 @@ void initMsgHandleFp() { handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp; buildRequestMsgFp[TSDB_SQL_CREATE_USER] = doBuildMsgSupp; + buildRequestMsgFp[TSDB_SQL_DROP_USER] = doBuildMsgSupp; + + buildRequestMsgFp[TSDB_SQL_CREATE_ACCT] = doBuildMsgSupp; + buildRequestMsgFp[TSDB_SQL_DROP_ACCT] = doBuildMsgSupp; buildRequestMsgFp[TSDB_SQL_SHOW] = doBuildMsgSupp; handleRequestRspFp[TSDB_SQL_SHOW] = processShowRsp; @@ -3392,4 +320,10 @@ void initMsgHandleFp() { buildRequestMsgFp[TSDB_SQL_CREATE_DB] = doBuildMsgSupp; handleRequestRspFp[TSDB_SQL_CREATE_DB] = processCreateDbRsp; + + buildRequestMsgFp[TSDB_SQL_USE_DB] = doBuildMsgSupp; + handleRequestRspFp[TSDB_SQL_USE_DB] = processUseDbRsp; + + buildRequestMsgFp[TSDB_SQL_CREATE_TABLE] = doBuildMsgSupp; + handleRequestRspFp[TSDB_SQL_CREATE_TABLE] = processCreateTableRsp; } \ No newline at end of file diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 944247e88e57901d57bdaf18a1025c53b08ad107..0ef305d3b610aa3c7e3b39d48e0b94492bb821e8 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -16,15 +16,15 @@ #include #include #include -#include "tglobal.h" #pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -#include "../inc/clientInt.h" #include "taos.h" +#include "tglobal.h" +#include "../inc/clientInt.h" namespace { } // namespace @@ -40,13 +40,13 @@ TEST(testCase, driverInit_Test) { TEST(testCase, connect_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); - assert(pConn != NULL); +// assert(pConn != NULL); taos_close(pConn); } TEST(testCase, create_user_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); - assert(pConn != NULL); +// assert(pConn != NULL); TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { @@ -57,23 +57,36 @@ TEST(testCase, create_user_Test) { taos_close(pConn); } -//TEST(testCase, drop_user_Test) { -// TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} +TEST(testCase, create_account_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); -TEST(testCase, show_user_Test) { + TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_account_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_user_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "show users"); TAOS_ROW pRow = NULL; @@ -89,10 +102,23 @@ TEST(testCase, show_user_Test) { taos_close(pConn); } -TEST(testCase, show_db_Test) { +TEST(testCase, drop_user_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "drop user abc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_db_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "show databases"); TAOS_ROW pRow = NULL; @@ -112,7 +138,40 @@ TEST(testCase, create_db_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); - TAOS_RES* pRes = taos_query(pConn, "create database abc"); + TAOS_RES* pRes = taos_query(pConn, "create database abc1"); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_close(pConn); +} + +TEST(testCase, use_db_test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_close(pConn); +} + +TEST(testCase, create_stable_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); TAOS_FIELD* pFields = taos_fetch_fields(pRes); ASSERT_TRUE(pFields == NULL); diff --git a/source/common/src/tname.c b/source/common/src/tname.c index fa303fe4e923317c276ccc025499593ca3a8b66a..96610164836dfe32d6219cba0f068abccbe3c7b3 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -6,21 +6,6 @@ #define VALID_NAME_TYPE(x) ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T) -char* extractDBName(const char* tableId, char* name) { - size_t offset1 = strcspn(tableId, &TS_PATH_DELIMITER[0]); - size_t len = strcspn(&tableId[offset1 + 1], &TS_PATH_DELIMITER[0]); - - return strncpy(name, &tableId[offset1 + 1], len); -} - -// todo remove it -size_t tableIdPrefix(const char* name, char* prefix, int32_t len) { - tstrncpy(prefix, name, len); - strcat(prefix, TS_PATH_DELIMITER); - - return strlen(prefix); -} - bool tscValidateTableNameLength(size_t len) { return len < TSDB_TABLE_NAME_LEN; } @@ -125,7 +110,7 @@ int32_t tNameExtractFullName(const SName* name, char* dst) { return -1; } - int32_t len = snprintf(dst, TSDB_FULL_DB_NAME_LEN, "%s.%s", name->acctId, name->dbname); + int32_t len = snprintf(dst, TSDB_FULL_DB_NAME_LEN, "%d.%s", name->acctId, name->dbname); size_t tnameLen = strlen(name->tname); if (tnameLen > 0) { @@ -141,7 +126,9 @@ int32_t tNameExtractFullName(const SName* name, char* dst) { int32_t tNameLen(const SName* name) { assert(name != NULL); - int32_t len = (int32_t) strlen(name->acctId); + + char tmp[12] = {0}; + int32_t len = sprintf(tmp, "%d", name->acctId); int32_t len1 = (int32_t) strlen(name->dbname); int32_t len2 = (int32_t) strlen(name->tname); @@ -161,10 +148,6 @@ bool tNameIsValid(const SName* name) { return false; } - if (strlen(name->acctId) <= 0) { - return false; - } - if (name->type == TSDB_DB_NAME_T) { return strlen(name->dbname) > 0; } else { @@ -237,13 +220,6 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { return -1; } - int32_t len = (int32_t)(p - str); - - // too long account id or too long db name -// if ((len >= tListLen(dst->acctId)) || (len <= 0)) { -// return -1; -// } -// memcpy (dst->acctId, str, len); dst->acctId = strtoll(str, NULL, 10); } @@ -272,9 +248,8 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { dst->type = TSDB_TABLE_NAME_T; char* start = (char*) ((p == NULL)? str: (p+1)); - int32_t len = (int32_t) strlen(start); - // too long account id or too long db name + int32_t len = (int32_t) strlen(start); if ((len >= tListLen(dst->tname)) || (len <= 0)) { return -1; } diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index b9a1e4cecbfe333591844c3ebbaebe5a96a0c741..083935c706e06abee8d07a74e9b4515350671e39 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -31,7 +31,7 @@ static struct { } global = {0}; void dmnSigintHandle(int signum, void *info, void *ctx) { - uError("singal:%d is received", signum); + uInfo("singal:%d is received", signum); global.stop = true; } diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 50b1a1cf20fa2db10c401911aff6b8178f374d64..be60d2fcae26fc0a054b83c24a72f9cd7272d600 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -73,7 +73,6 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_VGROUP_LIST] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_KILL_STREAM] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_SHOW] = dndProcessMnodeReadMsg; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index d3f1b06a4a88fd5d96f760ed1ae7e2f9a968a3bf..598d6e47bed1987587dbc7fcc9dff25df68457dc 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -17,11 +17,23 @@ #include "dndVnodes.h" #include "dndTransport.h" +typedef struct { + int32_t vgId; + int32_t vgVersion; + int8_t dropped; + uint64_t dbUid; + char db[TSDB_FULL_DB_NAME_LEN]; + char path[PATH_MAX + 20]; +} SWrapperCfg; + typedef struct { int32_t vgId; int32_t refCount; + int32_t vgVersion; int8_t dropped; int8_t accessState; + uint64_t dbUid; + char *db; char *path; SVnode *pImpl; taos_queue pWriteQ; @@ -32,13 +44,13 @@ typedef struct { } SVnodeObj; typedef struct { - int32_t vnodeNum; - int32_t opened; - int32_t failed; - int32_t threadIndex; - pthread_t *pThreadId; - SVnodeObj *pVnodes; - SDnode * pDnode; + int32_t vnodeNum; + int32_t opened; + int32_t failed; + int32_t threadIndex; + pthread_t thread; + SDnode *pDnode; + SWrapperCfg *pCfgs; } SVnodeThread; static int32_t dndInitVnodeReadWorker(SDnode *pDnode); @@ -60,8 +72,8 @@ static void dndFreeVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode); static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode); static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode); -static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg); -static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg); +static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); +static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg); static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs); @@ -71,18 +83,16 @@ void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pE void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); void dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); -static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg); +static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg); -static SVnodeObj * dndAcquireVnode(SDnode *pDnode, int32_t vgId); +static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId); static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode); -static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, SVnode *pImpl); -static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode); +static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl); +static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode); static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes); -static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes); +static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); static int32_t dndWriteVnodesToFile(SDnode *pDnode); -static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg); -static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode); static int32_t dndOpenVnodes(SDnode *pDnode); static void dndCloseVnodes(SDnode *pDnode); @@ -126,22 +136,25 @@ static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) { dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); } -static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, SVnode *pImpl) { +static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodeObj * pVnode = calloc(1, sizeof(SVnodeObj)); + SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj)); if (pVnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pVnode->vgId = vgId; - pVnode->refCount = 0; + pVnode->vgId = pCfg->vgId; + pVnode->refCount = 1; pVnode->dropped = 0; pVnode->accessState = TSDB_VN_ALL_ACCCESS; pVnode->pImpl = pImpl; + pVnode->vgVersion = pCfg->vgVersion; + pVnode->dbUid = pCfg->dbUid; + pVnode->db = tstrdup(pCfg->db); + pVnode->path = tstrdup(pCfg->path); - pVnode->path = tstrdup(path); - if (pVnode->path == NULL) { + if (pVnode->path == NULL || pVnode->db == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -167,7 +180,7 @@ static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, S } taosWLockLatch(&pMgmt->latch); - int32_t code = taosHashPut(pMgmt->hash, &vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); + int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); taosWUnLockLatch(&pMgmt->latch); if (code != 0) { @@ -176,7 +189,7 @@ static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, S return code; } -static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode) { +static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; taosWLockLatch(&pMgmt->latch); taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); @@ -195,6 +208,9 @@ static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode) { dndFreeVnodeWriteQueue(pDnode, pVnode); dndFreeVnodeApplyQueue(pDnode, pVnode); dndFreeVnodeSyncQueue(pDnode, pVnode); + free(pVnode->path); + free(pVnode->db); + free(pVnode); } static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { @@ -208,16 +224,16 @@ static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { void *pIter = taosHashIterate(pMgmt->hash, NULL); while (pIter) { SVnodeObj **ppVnode = pIter; - SVnodeObj * pVnode = *ppVnode; - if (pVnode) { + SVnodeObj *pVnode = *ppVnode; + if (pVnode && num < size) { + int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); + pVnodes[num] = (*ppVnode); num++; - if (num < size) { - int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); - pVnodes[num] = (*ppVnode); - } + pIter = taosHashIterate(pMgmt->hash, pIter); + } else { + taosHashCancelIterate(pMgmt->hash, pIter); } - pIter = taosHashIterate(pMgmt->hash, pIter); } taosRUnLockLatch(&pMgmt->latch); @@ -226,15 +242,15 @@ static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { return pVnodes; } -static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes) { - int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 30000; - char * content = calloc(1, maxLen + 1); - cJSON * root = NULL; - FILE * fp = NULL; - char file[PATH_MAX + 20] = {0}; - SVnodeObj *pVnodes = NULL; +static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { + int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 30000; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + FILE *fp = NULL; + char file[PATH_MAX + 20] = {0}; + SWrapperCfg *pCfgs = NULL; snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); @@ -270,31 +286,55 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_ goto PRASE_VNODE_OVER; } - pVnodes = calloc(vnodesNum, sizeof(SVnodeObj)); - if (pVnodes == NULL) { + pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg)); + if (pCfgs == NULL) { dError("failed to read %s since out of memory", file); goto PRASE_VNODE_OVER; } for (int32_t i = 0; i < vnodesNum; ++i) { - cJSON * vnode = cJSON_GetArrayItem(vnodes, i); - SVnodeObj *pVnode = &pVnodes[i]; + cJSON *vnode = cJSON_GetArrayItem(vnodes, i); + SWrapperCfg *pCfg = &pCfgs[i]; cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); - if (!vgId || vgId->type != cJSON_String) { + if (!vgId || vgId->type != cJSON_Number) { dError("failed to read %s since vgId not found", file); goto PRASE_VNODE_OVER; } - pVnode->vgId = atoi(vgId->valuestring); + pCfg->vgId = vgId->valueint; + snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCfg->vgId); cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); - if (!dropped || dropped->type != cJSON_String) { + if (!dropped || dropped->type != cJSON_Number) { dError("failed to read %s since dropped not found", file); goto PRASE_VNODE_OVER; } - pVnode->dropped = atoi(vnode->valuestring); + pCfg->dropped = dropped->valueint; + + cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion"); + if (!vgVersion || vgVersion->type != cJSON_Number) { + dError("failed to read %s since vgVersion not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->vgVersion = vgVersion->valueint; + + cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid"); + if (!dbUid || dbUid->type != cJSON_String) { + dError("failed to read %s since dbUid not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->dbUid = atoll(dbUid->valuestring); + + cJSON *db = cJSON_GetObjectItem(vnode, "db"); + if (!db || db->type != cJSON_String) { + dError("failed to read %s since db not found", file); + goto PRASE_VNODE_OVER; + } + tstrncpy(pCfg->db, db->valuestring, TSDB_FULL_DB_NAME_LEN); } + *ppCfgs = pCfgs; + *numOfVnodes = vnodesNum; code = 0; dInfo("succcessed to read file %s", file); @@ -313,30 +353,35 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); FILE *fp = fopen(file, "w"); - if (fp != NULL) { + if (fp == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write %s since %s", file, terrstr()); return -1; } - - int32_t len = 0; - int32_t maxLen = 30000; - char * content = calloc(1, maxLen + 1); int32_t numOfVnodes = 0; SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); + int32_t len = 0; + int32_t maxLen = 65536; + char *content = calloc(1, maxLen + 1); + len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"vnodes\": [{\n"); + len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); for (int32_t i = 0; i < numOfVnodes; ++i) { SVnodeObj *pVnode = pVnodes[i]; - len += snprintf(content + len, maxLen - len, " \"vgId\": \"%d\",\n", pVnode->vgId); - len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\"\n", pVnode->dropped); + len += snprintf(content + len, maxLen - len, " {\n"); + len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped); + len += snprintf(content + len, maxLen - len, " \"vgVersion\": %d,\n", pVnode->vgVersion); + len += snprintf(content + len, maxLen - len, " \"dbUid\": \"%" PRIu64 "\",\n", pVnode->dbUid); + len += snprintf(content + len, maxLen - len, " \"db\": \"%s\"\n", pVnode->db); if (i < numOfVnodes - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); + len += snprintf(content + len, maxLen - len, " },\n"); } else { - len += snprintf(content + len, maxLen - len, " }]\n"); + len += snprintf(content + len, maxLen - len, " }\n"); } } + len += snprintf(content + len, maxLen - len, " ]\n"); len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); @@ -358,74 +403,29 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { return taosRenameFile(file, realfile); } -static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg) { - char path[PATH_MAX + 20] = {0}; - snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, vgId); - // SVnode *pImpl = vnodeCreate(vgId, path, pCfg); - - SVnode *pImpl = vnodeOpen(path, NULL); - if (pImpl == NULL) { - return -1; - } - - int32_t code = dndCreateVnodeWrapper(pDnode, vgId, path, pImpl); - if (code != 0) { - vnodeClose(pImpl); - vnodeDestroy(path); - terrno = code; - return code; - } - - code = dndWriteVnodesToFile(pDnode); - if (code != 0) { - vnodeClose(pImpl); - vnodeDestroy(path); - terrno = code; - return code; - } - - return 0; -} - -static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode) { - pVnode->dropped = 1; - if (dndWriteVnodesToFile(pDnode) != 0) { - pVnode->dropped = 0; - return -1; - } - - dndDropVnodeWrapper(pDnode, pVnode); - vnodeClose(pVnode->pImpl); - vnodeDestroy(pVnode->path); - dndWriteVnodesToFile(pDnode); - return 0; -} - static void *dnodeOpenVnodeFunc(void *param) { SVnodeThread *pThread = param; - SDnode * pDnode = pThread->pDnode; - SVnodesMgmt * pMgmt = &pDnode->vmgmt; + SDnode *pDnode = pThread->pDnode; + SVnodesMgmt *pMgmt = &pDnode->vmgmt; dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); setThreadName("open-vnodes"); for (int32_t v = 0; v < pThread->vnodeNum; ++v) { - SVnodeObj *pVnode = &pThread->pVnodes[v]; + SWrapperCfg *pCfg = &pThread->pCfgs[v]; char stepDesc[TSDB_STEP_DESC_LEN] = {0}; - snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pVnode->vgId, + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId, pMgmt->openVnodes, pMgmt->totalVnodes); dndReportStartup(pDnode, "open-vnodes", stepDesc); - char path[PATH_MAX + 20] = {0}; - snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, pVnode->vgId); - SVnode *pImpl = vnodeOpen(path, NULL); + SVnode *pImpl = vnodeOpen(pCfg->path, NULL); if (pImpl == NULL) { - dError("vgId:%d, failed to open vnode by thread:%d", pVnode->vgId, pThread->threadIndex); + dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->failed++; } else { - dndCreateVnodeWrapper(pDnode, pVnode->vgId, path, pImpl); - dDebug("vgId:%d, is opened by thread:%d", pVnode->vgId, pThread->threadIndex); + dndOpenVnode(pDnode, pCfg, pImpl); + dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->opened++; } @@ -448,9 +448,9 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { return -1; } - SVnodeObj *pVnodes = NULL; - int32_t numOfVnodes = 0; - if (dndGetVnodesFromFile(pDnode, &pVnodes, &numOfVnodes) != 0) { + SWrapperCfg *pCfgs = NULL; + int32_t numOfVnodes = 0; + if (dndGetVnodesFromFile(pDnode, &pCfgs, &numOfVnodes) != 0) { dInfo("failed to get vnode list from disk since %s", terrstr()); return -1; } @@ -463,13 +463,14 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); for (int32_t t = 0; t < threadNum; ++t) { threads[t].threadIndex = t; - threads[t].pVnodes = calloc(vnodesPerThread, sizeof(SVnodeObj)); + threads[t].pDnode = pDnode; + threads[t].pCfgs = calloc(vnodesPerThread, sizeof(SWrapperCfg)); } for (int32_t v = 0; v < numOfVnodes; ++v) { int32_t t = v % threadNum; SVnodeThread *pThread = &threads[t]; - pThread->pVnodes[pThread->vnodeNum++] = pVnodes[v]; + pThread->pCfgs[pThread->vnodeNum++] = pCfgs[v]; } dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); @@ -478,19 +479,25 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { SVnodeThread *pThread = &threads[t]; if (pThread->vnodeNum == 0) continue; - pThread->pThreadId = taosCreateThread(dnodeOpenVnodeFunc, pThread); - if (pThread->pThreadId == NULL) { + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnodeFunc, pThread) != 0) { dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); } + + pthread_attr_destroy(&thAttr); } for (int32_t t = 0; t < threadNum; ++t) { SVnodeThread *pThread = &threads[t]; - taosDestoryThread(pThread->pThreadId); - pThread->pThreadId = NULL; - free(pThread->pVnodes); + if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { + pthread_join(pThread->thread, NULL); + } + free(pThread->pCfgs); } free(threads); + free(pCfgs); if (pMgmt->openVnodes != pMgmt->totalVnodes) { dError("there are total vnodes:%d, opened:%d", pMgmt->totalVnodes, pMgmt->openVnodes); @@ -508,7 +515,8 @@ static void dndCloseVnodes(SDnode *pDnode) { SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); for (int32_t i = 0; i < numOfVnodes; ++i) { - dndDropVnodeWrapper(pDnode, pVnodes[i]); + dndReleaseVnode(pDnode, pVnodes[i]); + dndCloseVnode(pDnode, pVnodes[i]); } if (pVnodes != NULL) { @@ -523,11 +531,12 @@ static void dndCloseVnodes(SDnode *pDnode) { dInfo("total vnodes:%d are all closed", numOfVnodes); } -static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg *pCfg) { +static SCreateVnodeMsg *dndParseCreateVnodeReq(SRpcMsg *rpcMsg) { SCreateVnodeMsg *pCreate = rpcMsg->pCont; pCreate->vgId = htonl(pCreate->vgId); pCreate->dnodeId = htonl(pCreate->dnodeId); pCreate->dbUid = htobe64(pCreate->dbUid); + pCreate->vgVersion = htonl(pCreate->vgVersion); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); pCreate->totalBlocks = htonl(pCreate->totalBlocks); pCreate->daysPerFile = htonl(pCreate->daysPerFile); @@ -544,9 +553,10 @@ static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg pReplica->port = htons(pReplica->port); } - *vgId = pCreate->vgId; + return pCreate; +} -#if 0 +static void dndGenerateVnodeCfg(SCreateVnodeMsg *pCreate, SVnodeCfg *pCfg) { pCfg->wsize = pCreate->cacheBlockSize; pCfg->ssize = pCreate->cacheBlockSize; pCfg->wsize = pCreate->cacheBlockSize; @@ -567,8 +577,15 @@ static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg pCfg->walCfg.rollPeriod = 128; pCfg->walCfg.segSize = 128; pCfg->walCfg.vgId = pCreate->vgId; -#endif - return 0; +} + +static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeMsg *pCreate, SWrapperCfg *pCfg) { + memcpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN); + pCfg->dbUid = pCreate->dbUid; + pCfg->dropped = 0; + snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCreate->vgId); + pCfg->vgId = pCreate->vgId; + pCfg->vgVersion = pCreate->vgVersion; } static SDropVnodeMsg *vnodeParseDropVnodeReq(SRpcMsg *rpcMsg) { @@ -584,48 +601,83 @@ static SAuthVnodeMsg *vnodeParseAuthVnodeReq(SRpcMsg *rpcMsg) { } static int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { + SCreateVnodeMsg *pCreate = dndParseCreateVnodeReq(rpcMsg); + dDebug("vgId:%d, create vnode req is received", pCreate->vgId); + SVnodeCfg vnodeCfg = {0}; - int32_t vgId = 0; + dndGenerateVnodeCfg(pCreate, &vnodeCfg); - dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg); - dDebug("vgId:%d, create vnode req is received", vgId); + SWrapperCfg wrapperCfg = {0}; + dndGenerateWrapperCfg(pDnode, pCreate, &wrapperCfg); - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); + SVnodeObj *pVnode = dndAcquireVnode(pDnode, pCreate->vgId); if (pVnode != NULL) { - dDebug("vgId:%d, already exist, return success", vgId); + dDebug("vgId:%d, already exist, return success", pCreate->vgId); dndReleaseVnode(pDnode, pVnode); return 0; } - if (dndCreateVnode(pDnode, vgId, &vnodeCfg) != 0) { - dError("vgId:%d, failed to create vnode since %s", vgId, terrstr()); - return terrno; + SVnode *pImpl = vnodeOpen(wrapperCfg.path, NULL /*pCfg*/); + if (pImpl == NULL) { + return -1; + } + + int32_t code = dndOpenVnode(pDnode, &wrapperCfg, pImpl); + if (code != 0) { + vnodeClose(pImpl); + vnodeDestroy(wrapperCfg.path); + terrno = code; + return code; + } + + code = dndWriteVnodesToFile(pDnode); + if (code != 0) { + vnodeClose(pImpl); + vnodeDestroy(wrapperCfg.path); + terrno = code; + return code; } return 0; } static int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { + SAlterVnodeMsg *pAlter = (SAlterVnodeMsg *)dndParseCreateVnodeReq(rpcMsg); + dDebug("vgId:%d, alter vnode req is received", pAlter->vgId); + SVnodeCfg vnodeCfg = {0}; - int32_t vgId = 0; + dndGenerateVnodeCfg(pAlter, &vnodeCfg); - dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg); - dDebug("vgId:%d, alter vnode req is received", vgId); + SWrapperCfg wrapperCfg = {0}; + dndGenerateWrapperCfg(pDnode, pAlter, &wrapperCfg); - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); + SVnodeObj *pVnode = dndAcquireVnode(pDnode, pAlter->vgId); if (pVnode == NULL) { - dDebug("vgId:%d, failed to alter vnode since %s", vgId, terrstr()); + dDebug("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr()); return terrno; } + if (wrapperCfg.vgVersion == pVnode->vgVersion) { + dndReleaseVnode(pDnode, pVnode); + dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", pAlter->vgId); + return 0; + } + if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { - dError("vgId:%d, failed to alter vnode since %s", vgId, terrstr()); + dError("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr()); dndReleaseVnode(pDnode, pVnode); return terrno; } + int32_t oldVersion = pVnode->vgVersion; + pVnode->vgVersion = wrapperCfg.vgVersion; + int32_t code = dndWriteVnodesToFile(pDnode); + if (code != 0) { + pVnode->vgVersion = oldVersion; + } + dndReleaseVnode(pDnode, pVnode); - return 0; + return code; } static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { @@ -637,15 +689,21 @@ static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); if (pVnode == NULL) { dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); - return terrno; + return 0; } - if (dndDropVnode(pDnode, pVnode) != 0) { - dError("vgId:%d, failed to drop vnode since %s", vgId, terrstr()); - dndReleaseVnode(pDnode, pVnode); + pVnode->dropped = 1; + if (dndWriteVnodesToFile(pDnode) != 0) { + pVnode->dropped = 0; return terrno; } + dndReleaseVnode(pDnode, pVnode); + dndCloseVnode(pDnode, pVnode); + vnodeClose(pVnode->pImpl); + vnodeDestroy(pVnode->path); + dndWriteVnodesToFile(pDnode); + return 0; } @@ -738,48 +796,76 @@ static void dndProcessVnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { break; } - if (code != 0) { - SRpcMsg rsp = {.code = code, .handle = pMsg->handle}; - rpcSendResponse(&rsp); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } + SRpcMsg rsp = {.code = code, .handle = pMsg->handle, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rsp); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); } -static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) { - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_QUERY); +static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { + SRpcMsg *pRsp = NULL; + vnodeProcessQueryReq(pVnode->pImpl, pMsg, &pRsp); } -static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) { - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_FETCH); +static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { + SRpcMsg *pRsp = NULL; + vnodeProcessFetchReq(pVnode->pImpl, pMsg, &pRsp); } static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { - SVnodeMsg *pMsg = vnodeInitMsg(numOfMsgs); - SRpcMsg * pRpcMsg = NULL; + SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); for (int32_t i = 0; i < numOfMsgs; ++i) { - taosGetQitem(qall, (void **)&pRpcMsg); - vnodeAppendMsg(pMsg, pRpcMsg); - taosFreeQitem(pRpcMsg); + SRpcMsg *pMsg = NULL; + taosGetQitem(qall, (void **)&pMsg); + void *ptr = taosArrayPush(pArray, &pMsg); + assert(ptr != NULL); } - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_WRITE); + vnodeProcessWMsgs(pVnode->pImpl, pArray); + + for (size_t i = 0; i < numOfMsgs; i++) { + SRpcMsg *pRsp = NULL; + SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); + int32_t code = vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp); + if (pRsp != NULL) { + rpcSendResponse(pRsp); + free(pRsp); + } else { + if (code != 0) code = terrno; + SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code}; + rpcSendResponse(&rpcRsp); + } + } + + for (size_t i = 0; i < numOfMsgs; i++) { + SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + } + + taosArrayDestroy(pArray); } static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { - SVnodeMsg *pMsg = NULL; + SRpcMsg *pMsg = NULL; + for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_APPLY); + + SRpcMsg *pRsp = NULL; + (void)vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp); } } static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { - SVnodeMsg *pMsg = NULL; + SRpcMsg *pMsg = NULL; + for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); - vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_SYNC); + + SRpcMsg *pRsp = NULL; + (void)vnodeProcessSyncReq(pVnode->pImpl, pMsg, &pRsp); } } @@ -807,40 +893,14 @@ static int32_t dndWriteRpcMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) { } } -static int32_t dndWriteVnodeMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) { - int32_t code = 0; - - if (pQueue == NULL) { - code = TSDB_CODE_MSG_NOT_PROCESSED; - } else { - SVnodeMsg *pMsg = vnodeInitMsg(1); - if (pMsg == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - if (vnodeAppendMsg(pMsg, pRpcMsg) != 0) { - code = terrno; - } else { - if (taosWriteQitem(pQueue, pMsg) != 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - } - } - } - } - - if (code != TSDB_CODE_SUCCESS) { - SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; - rpcSendResponse(&rsp); - rpcFreeCont(pRpcMsg->pCont); - } -} - static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) { SMsgHead *pHead = (SMsgHead *)pMsg->pCont; + pHead->contLen = htonl(pHead->contLen); pHead->vgId = htonl(pHead->vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); if (pVnode == NULL) { - SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; + SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; rpcSendResponse(&rsp); rpcFreeCont(pMsg->pCont); } @@ -864,7 +924,7 @@ void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); if (pVnode != NULL) { - dndWriteVnodeMsgToVnodeQueue(pVnode->pSyncQ, pMsg); + dndWriteRpcMsgToVnodeQueue(pVnode->pSyncQ, pMsg); dndReleaseVnode(pDnode, pVnode); } } @@ -872,7 +932,7 @@ void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); if (pVnode != NULL) { - dndWriteVnodeMsgToVnodeQueue(pVnode->pQueryQ, pMsg); + dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg); dndReleaseVnode(pDnode, pVnode); } } @@ -880,12 +940,12 @@ void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); if (pVnode != NULL) { - dndWriteVnodeMsgToVnodeQueue(pVnode->pFetchQ, pMsg); + dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg); dndReleaseVnode(pDnode, pVnode); } } -static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg) { +static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); if (pVnode == NULL) { return -1; @@ -1029,7 +1089,7 @@ static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) { } static int32_t dndInitVnodeWriteWorker(SDnode *pDnode) { - SVnodesMgmt * pMgmt = &pDnode->vmgmt; + SVnodesMgmt *pMgmt = &pDnode->vmgmt; SMWorkerPool *pPool = &pMgmt->writePool; pPool->name = "vnode-write"; pPool->max = pDnode->opt.numOfCores; @@ -1050,7 +1110,7 @@ static void dndCleanupVnodeWriteWorker(SDnode *pDnode) { static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue); + pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue); if (pVnode->pSyncQ == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -1061,7 +1121,7 @@ static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) { static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pSyncQ); + tMWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); pVnode->pSyncQ = NULL; } @@ -1137,12 +1197,12 @@ void dndGetVnodeLoads(SDnode *pDnode, SVnodeLoads *pLoads) { pLoads->num = taosHashGetSize(pMgmt->hash); int32_t v = 0; - void * pIter = taosHashIterate(pMgmt->hash, NULL); + void *pIter = taosHashIterate(pMgmt->hash, NULL); while (pIter) { SVnodeObj **ppVnode = pIter; if (ppVnode == NULL || *ppVnode == NULL) continue; - SVnodeObj * pVnode = *ppVnode; + SVnodeObj *pVnode = *ppVnode; SVnodeLoad *pLoad = &pLoads->data[v++]; vnodeGetLoad(pVnode->pImpl, pLoad); diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index 0dbee0d3379daa55e69ba6ed5e0b2beaa400dec7..a4996ecb3bfc0792add96700c51c4b83dcf6d80e 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -176,6 +176,12 @@ SDnode *dndInit(SDnodeOpt *pOption) { return NULL; } + if (vnodeInit(1) != 0) { + dError("failed to init vnode env"); + dndCleanup(pDnode); + return NULL; + } + if (dndInitDnode(pDnode) != 0) { dError("failed to init dnode"); dndCleanup(pDnode); @@ -222,8 +228,10 @@ void dndCleanup(SDnode *pDnode) { dndCleanupMnode(pDnode); dndCleanupVnodes(pDnode); dndCleanupDnode(pDnode); + vnodeClear(); walCleanUp(); rpcCleanup(); + dndCleanupEnv(pDnode); free(pDnode); dInfo("TDengine is cleaned up successfully"); diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index a5ece72f425f8d00bd6d4328350cb48dab186cc5..a29926c8026950028690bf3d07fa224ee1dd5084 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -8,8 +8,8 @@ add_subdirectory(db) add_subdirectory(dnode) # add_subdirectory(func) # add_subdirectory(mnode) -# add_subdirectory(profile) -# add_subdirectory(show) +add_subdirectory(profile) +add_subdirectory(show) add_subdirectory(stb) # add_subdirectory(sync) # add_subdirectory(telem) @@ -17,4 +17,4 @@ add_subdirectory(stb) add_subdirectory(user) add_subdirectory(vgroup) -# add_subdirectory(common) +add_subdirectory(sut) diff --git a/source/dnode/mgmt/impl/test/acct/CMakeLists.txt b/source/dnode/mgmt/impl/test/acct/CMakeLists.txt index 8b160fc446b2f501e8f2ece4e76e1dbc27a77a74..a06becd1279b2f66f73cb3d8fbf347c9e4848e2b 100644 --- a/source/dnode/mgmt/impl/test/acct/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/acct/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_acct "") - -target_sources(dnode_test_acct - PRIVATE - "acct.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. ACCT_SRC) +add_executable(dnode_test_acct ${ACCT_SRC}) target_link_libraries( dnode_test_acct - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_acct - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/acct/acct.cpp b/source/dnode/mgmt/impl/test/acct/acct.cpp index e2b819da2828dc4f7d4d0a915edf76c1bc29a06c..69ddae675d12e353a610769592565b18d0078ada 100644 --- a/source/dnode/mgmt/impl/test/acct/acct.cpp +++ b/source/dnode/mgmt/impl/test/acct/acct.cpp @@ -9,103 +9,59 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestAcct : public ::testing::Test { protected: - static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SServer* pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - return pServer; - } - - static void SetUpTestSuite() { - initLog("/tmp/tdlog"); - - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9012"; - pServer = CreateServer("/tmp/dnode_test_user", fqdn, 9012, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9012); - taosMsleep(300); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - pServer = NULL; - pClient = NULL; - } - - static SServer* pServer; - static SClient* pClient; - static int32_t connId; + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_acct", 9012); } + static void TearDownTestSuite() { test.Cleanup(); } + + static Testbase test; + + public: + void SetUp() override {} + void TearDown() override {} }; -SServer* DndTestAcct::pServer; -SClient* DndTestAcct::pClient; -int32_t DndTestAcct::connId; +Testbase DndTestAcct::test; TEST_F(DndTestAcct, 01_CreateAcct) { - ASSERT_NE(pClient, nullptr); - - SCreateAcctMsg* pReq = (SCreateAcctMsg*)rpcMallocCont(sizeof(SCreateAcctMsg)); + int32_t contLen = sizeof(SCreateAcctMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateAcctMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_ACCT; + SCreateAcctMsg* pReq = (SCreateAcctMsg*)rpcMallocCont(contLen); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_ACCT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } TEST_F(DndTestAcct, 02_AlterAcct) { - ASSERT_NE(pClient, nullptr); + int32_t contLen = sizeof(SCreateAcctMsg); - SAlterAcctMsg* pReq = (SAlterAcctMsg*)rpcMallocCont(sizeof(SAlterAcctMsg)); + SAlterAcctMsg* pReq = (SAlterAcctMsg*)rpcMallocCont(contLen); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SAlterAcctMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_ACCT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_ALTER_ACCT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } TEST_F(DndTestAcct, 03_DropAcct) { - ASSERT_NE(pClient, nullptr); - - SDropAcctMsg* pReq = (SDropAcctMsg*)rpcMallocCont(sizeof(SDropAcctMsg)); + int32_t contLen = sizeof(SDropAcctMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropAcctMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_ACCT; + SDropAcctMsg* pReq = (SDropAcctMsg*)rpcMallocCont(contLen); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_ACCT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } TEST_F(DndTestAcct, 04_ShowAcct) { - ASSERT_NE(pClient, nullptr); + int32_t contLen = sizeof(SShowMsg); - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + SShowMsg* pReq = (SShowMsg*)rpcMallocCont(contLen); pReq->type = TSDB_MGMT_TABLE_ACCT; - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_SHOW, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt b/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt index e97ef7688140bc4d85a7d1a734b21a6b586c1297..a63f3106e6299299adf963427604c7384931f35c 100644 --- a/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_cluster "") - -target_sources(dnode_test_cluster - PRIVATE - "cluster.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. CLUSTER_SRC) +add_executable(dnode_test_cluster ${CLUSTER_SRC}) target_link_libraries( dnode_test_cluster - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_cluster - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/cluster/cluster.cpp b/source/dnode/mgmt/impl/test/cluster/cluster.cpp index 0aca2fde4badaa17b6fa70186ed9ad157c588678..7230c3eb74ca2e0cb08454f1f170b046918c4cc7 100644 --- a/source/dnode/mgmt/impl/test/cluster/cluster.cpp +++ b/source/dnode/mgmt/impl/test/cluster/cluster.cpp @@ -9,162 +9,33 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestCluster : public ::testing::Test { protected: - static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SServer* pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - return pServer; - } + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_cluster", 9030); } + static void TearDownTestSuite() { test.Cleanup(); } - static void SetUpTestSuite() { - initLog("/tmp/tdlog"); - - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9030"; - pServer = CreateServer("/tmp/dnode_test_cluster", fqdn, 9030, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9030); - taosMsleep(1100); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - pServer = NULL; - pClient = NULL; - } - - static SServer* pServer; - static SClient* pClient; - static int32_t connId; + static Testbase test; public: void SetUp() override {} void TearDown() override {} - - void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = showType; - if (db != NULL) { - strcpy(pShow->db, db); - } - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htonl(pMeta->numOfTags); - pMeta->numOfColumns = htonl(pMeta->numOfColumns); - pMeta->sversion = htonl(pMeta->sversion); - pMeta->tversion = htonl(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_STREQ(pMeta->tbFname, showName); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->numOfColumns, columns); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->update, 0); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tuid, 0); - EXPECT_EQ(pMeta->suid, 0); - } - - void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { - SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, type); - EXPECT_EQ(pSchema->bytes, bytes); - EXPECT_STREQ(pSchema->name, name); - } - - void SendThenCheckShowRetrieveMsg(int32_t rows) { - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - // EXPECT_EQ(pRetrieveRsp->completed, completed); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - pData = pRetrieveRsp->data; - pos = 0; - } - - void CheckInt32() { - int32_t data = *((int32_t*)(pData + pos)); - pos += sizeof(int32_t); - EXPECT_GT(data, 0); - } - - void CheckTimestamp() { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_GT(data, 0); - } - - void CheckBinary(int32_t len) { - pos += sizeof(VarDataLenT); - char* data = (char*)(pData + pos); - pos += len; - } - - int32_t showId; - STableMetaMsg* pMeta; - SRetrieveTableRsp* pRetrieveRsp; - char* pData; - int32_t pos; }; -SServer* DndTestCluster::pServer; -SClient* DndTestCluster::pClient; -int32_t DndTestCluster::connId; +Testbase DndTestCluster::test; TEST_F(DndTestCluster, 01_ShowCluster) { - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_CLUSTER, "show cluster", 3, NULL); - CheckSchema(0, TSDB_DATA_TYPE_INT, 4, "id"); - CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, "name"); - CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_CLUSTER, ""); + CHECK_META( "show cluster", 3); + CHECK_SCHEMA(0, TSDB_DATA_TYPE_INT, 4, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, "name"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); - SendThenCheckShowRetrieveMsg(1); - CheckInt32(); - CheckBinary(TSDB_CLUSTER_ID_LEN); + IgnoreInt32(); + IgnoreBinary(TSDB_CLUSTER_ID_LEN); CheckTimestamp(); } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/db/CMakeLists.txt b/source/dnode/mgmt/impl/test/db/CMakeLists.txt index db96e2bad32eff4e99e6a3c68421608f655f18ec..cb9f1600fc2a76951a5cb81cf94c5d1ef48d5eb1 100644 --- a/source/dnode/mgmt/impl/test/db/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/db/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_db "") - -target_sources(dnode_test_db - PRIVATE - "db.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. DB_SRC) +add_executable(dnode_test_db ${DB_SRC}) target_link_libraries( dnode_test_db - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_db - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp index d465a62f2d5269632c4f5022e0bafca3a1e45670..378f46aa4defe07fa739b170b3cf04f442d99d60 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -9,199 +9,52 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestDb : public ::testing::Test { protected: - static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SServer* pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - return pServer; - } - - static void SetUpTestSuite() { - initLog("/tmp/tdlog"); - - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9040"; - pServer = CreateServer("/tmp/dnode_test_db", fqdn, 9040, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9040); - taosMsleep(1100); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - pServer = NULL; - pClient = NULL; - } + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_db", 9040); } + static void TearDownTestSuite() { test.Cleanup(); } - static SServer* pServer; - static SClient* pClient; - static int32_t connId; + static Testbase test; public: void SetUp() override {} void TearDown() override {} - - void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = showType; - if (db != NULL) { - strcpy(pShow->db, db); - } - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htonl(pMeta->numOfTags); - pMeta->numOfColumns = htonl(pMeta->numOfColumns); - pMeta->sversion = htonl(pMeta->sversion); - pMeta->tversion = htonl(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_STREQ(pMeta->tbFname, showName); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->numOfColumns, columns); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->update, 0); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tuid, 0); - EXPECT_EQ(pMeta->suid, 0); - } - - void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { - SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, type); - EXPECT_EQ(pSchema->bytes, bytes); - EXPECT_STREQ(pSchema->name, name); - } - - void SendThenCheckShowRetrieveMsg(int32_t rows) { - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - // EXPECT_EQ(pRetrieveRsp->completed, completed); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - pData = pRetrieveRsp->data; - pos = 0; - } - - void CheckInt8(int8_t val) { - int8_t data = *((int8_t*)(pData + pos)); - pos += sizeof(int8_t); - EXPECT_EQ(data, val); - } - - void CheckInt16(int16_t val) { - int16_t data = *((int16_t*)(pData + pos)); - pos += sizeof(int16_t); - EXPECT_EQ(data, val); - } - - void CheckInt32(int32_t val) { - int32_t data = *((int32_t*)(pData + pos)); - pos += sizeof(int32_t); - EXPECT_EQ(data, val); - } - - void CheckInt64(int64_t val) { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_EQ(data, val); - } - - void CheckTimestamp() { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_GT(data, 0); - } - - void CheckBinary(const char* val, int32_t len) { - pos += sizeof(VarDataLenT); - char* data = (char*)(pData + pos); - pos += len; - EXPECT_STREQ(data, val); - } - - int32_t showId; - STableMetaMsg* pMeta; - SRetrieveTableRsp* pRetrieveRsp; - char* pData; - int32_t pos; }; -SServer* DndTestDb::pServer; -SClient* DndTestDb::pClient; -int32_t DndTestDb::connId; +Testbase DndTestDb::test; TEST_F(DndTestDb, 01_ShowDb) { - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); - CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); - CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "vgroups"); - CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "replica"); - CheckSchema(4, TSDB_DATA_TYPE_SMALLINT, 2, "quorum"); - CheckSchema(5, TSDB_DATA_TYPE_SMALLINT, 2, "days"); - CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2"); - CheckSchema(7, TSDB_DATA_TYPE_INT, 4, "cache"); - CheckSchema(8, TSDB_DATA_TYPE_INT, 4, "blocks"); - CheckSchema(9, TSDB_DATA_TYPE_INT, 4, "minrows"); - CheckSchema(10, TSDB_DATA_TYPE_INT, 4, "maxrows"); - CheckSchema(11, TSDB_DATA_TYPE_TINYINT, 1, "wallevel"); - CheckSchema(12, TSDB_DATA_TYPE_INT, 4, "fsync"); - CheckSchema(13, TSDB_DATA_TYPE_TINYINT, 1, "comp"); - CheckSchema(14, TSDB_DATA_TYPE_TINYINT, 1, "cachelast"); - CheckSchema(15, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision"); - CheckSchema(16, TSDB_DATA_TYPE_TINYINT, 1, "update"); - - SendThenCheckShowRetrieveMsg(0); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + CHECK_META("show databases", 17); + CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_SMALLINT, 2, "vgroups"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_SMALLINT, 2, "replica"); + CHECK_SCHEMA(4, TSDB_DATA_TYPE_SMALLINT, 2, "quorum"); + CHECK_SCHEMA(5, TSDB_DATA_TYPE_SMALLINT, 2, "days"); + CHECK_SCHEMA(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2"); + CHECK_SCHEMA(7, TSDB_DATA_TYPE_INT, 4, "cache"); + CHECK_SCHEMA(8, TSDB_DATA_TYPE_INT, 4, "blocks"); + CHECK_SCHEMA(9, TSDB_DATA_TYPE_INT, 4, "minrows"); + CHECK_SCHEMA(10, TSDB_DATA_TYPE_INT, 4, "maxrows"); + CHECK_SCHEMA(11, TSDB_DATA_TYPE_TINYINT, 1, "wallevel"); + CHECK_SCHEMA(12, TSDB_DATA_TYPE_INT, 4, "fsync"); + CHECK_SCHEMA(13, TSDB_DATA_TYPE_TINYINT, 1, "comp"); + CHECK_SCHEMA(14, TSDB_DATA_TYPE_TINYINT, 1, "cachelast"); + CHECK_SCHEMA(15, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision"); + CHECK_SCHEMA(16, TSDB_DATA_TYPE_TINYINT, 1, "update"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); } TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { { - SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + int32_t contLen = sizeof(SCreateDbMsg); + + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d1"); pReq->numOfVgroups = htonl(2); pReq->cacheBlockSize = htonl(16); @@ -210,8 +63,8 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep2 = htonl(3650); - pReq->minRowsPerFileBlock = htonl(100); - pReq->maxRowsPerFileBlock = htonl(4096); + pReq->minRows = htonl(100); + pReq->maxRows = htonl(4096); pReq->commitTime = htonl(3600); pReq->fsyncPeriod = htonl(3000); pReq->walLevel = 1; @@ -223,20 +76,16 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { pReq->cacheLastRow = 0; pReq->ignoreExist = 1; - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - // taosMsleep(1000000); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); - SendThenCheckShowRetrieveMsg(1); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + CHECK_META("show databases", 17); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); CheckInt16(2); // vgroups @@ -255,12 +104,15 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { CheckBinary("ms", 3); // precision CheckInt8(0); // update - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_VGROUP, "show vgroups", 4, "1.d1"); - CheckSchema(0, TSDB_DATA_TYPE_INT, 4, "vgId"); - CheckSchema(1, TSDB_DATA_TYPE_INT, 4, "tables"); - CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "v1_dnode"); - CheckSchema(3, TSDB_DATA_TYPE_BINARY, 9 + VARSTR_HEADER_SIZE, "v1_status"); - SendThenCheckShowRetrieveMsg(2); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_VGROUP, "1.d1"); + CHECK_META("show vgroups", 4); + CHECK_SCHEMA(0, TSDB_DATA_TYPE_INT, 4, "vgId"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_INT, 4, "tables"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_SMALLINT, 2, "v1_dnode"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_BINARY, 9 + VARSTR_HEADER_SIZE, "v1_status"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); CheckInt32(1); CheckInt32(2); CheckInt32(0); @@ -271,7 +123,9 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { CheckBinary("master", 9); { - SAlterDbMsg* pReq = (SAlterDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg)); + int32_t contLen = sizeof(SAlterDbMsg); + + SAlterDbMsg* pReq = (SAlterDbMsg*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d1"); pReq->totalBlocks = htonl(12); pReq->daysToKeep0 = htonl(300); @@ -282,19 +136,14 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { pReq->quorum = 2; pReq->cacheLastRow = 1; - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SAlterDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_DB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_ALTER_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); - SendThenCheckShowRetrieveMsg(1); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); CheckInt16(2); // vgroups @@ -314,19 +163,14 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { CheckInt8(0); // update // restart - stopServer(pServer); - pServer = NULL; - - uInfo("start all server"); + test.Restart(); - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9040"; - pServer = startServer("/tmp/dnode_test_db", fqdn, 9040, firstEp); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + CHECK_META("show databases", 17); - uInfo("all server is running"); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); - SendThenCheckShowRetrieveMsg(1); CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); CheckInt16(2); // vgroups @@ -346,27 +190,28 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { CheckInt8(0); // update { - SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(sizeof(SDropDbMsg)); - strcpy(pReq->db, "1.d1"); + int32_t contLen = sizeof(SDropDbMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DB; + SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(contLen); + strcpy(pReq->db, "1.d1"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); - SendThenCheckShowRetrieveMsg(0); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + CHECK_META("show databases", 17); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); } TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { { - SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + int32_t contLen = sizeof(SCreateDbMsg); + + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d2"); pReq->numOfVgroups = htonl(2); pReq->cacheBlockSize = htonl(16); @@ -375,8 +220,8 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep2 = htonl(3650); - pReq->minRowsPerFileBlock = htonl(100); - pReq->maxRowsPerFileBlock = htonl(4096); + pReq->minRows = htonl(100); + pReq->maxRows = htonl(4096); pReq->commitTime = htonl(3600); pReq->fsyncPeriod = htonl(3000); pReq->walLevel = 1; @@ -388,33 +233,26 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { pReq->cacheLastRow = 0; pReq->ignoreExist = 1; - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); - SendThenCheckShowRetrieveMsg(1); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DB, ""); + CHECK_META("show databases", 17); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("d2", TSDB_DB_NAME_LEN - 1); { - SUseDbMsg* pReq = (SUseDbMsg*)rpcMallocCont(sizeof(SUseDbMsg)); + int32_t contLen = sizeof(SUseDbMsg); + + SUseDbMsg* pReq = (SUseDbMsg*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d2"); pReq->vgVersion = htonl(-1); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SUseDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_USE_DB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_USE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); diff --git a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt index 48b74a4eff8c0e07f115c8cb4cfcd04448413319..5796590865a0b4d5ec3e66b38441ea67465c79df 100644 --- a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_dnode "") - -target_sources(dnode_test_dnode - PRIVATE - "dnode.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. DTEST_SRC) +add_executable(dnode_test_dnode ${DTEST_SRC}) target_link_libraries( dnode_test_dnode - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_dnode - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index 1fa91f26cb6edb13aa9588ef26d0f4136d0621c9..c9708ec1cfcb8fdec5ddf10cd63918100187c010 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -9,189 +9,62 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestDnode : public ::testing::Test { public: - static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SServer* pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - return pServer; - } + void SetUp() override {} + void TearDown() override {} + public: static void SetUpTestSuite() { - initLog("/tmp/tdlog"); - + test.Init("/tmp/dnode_test_dnode1", 9041); const char* fqdn = "localhost"; const char* firstEp = "localhost:9041"; - pServer1 = CreateServer("/tmp/dnode_test_dnode1", fqdn, 9041, firstEp); - pServer2 = CreateServer("/tmp/dnode_test_dnode2", fqdn, 9042, firstEp); - pServer3 = CreateServer("/tmp/dnode_test_dnode3", fqdn, 9043, firstEp); - pServer4 = CreateServer("/tmp/dnode_test_dnode4", fqdn, 9044, firstEp); - pServer5 = CreateServer("/tmp/dnode_test_dnode5", fqdn, 9045, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9041); + + server2.Start("/tmp/dnode_test_dnode2", fqdn, 9042, firstEp); + server3.Start("/tmp/dnode_test_dnode3", fqdn, 9043, firstEp); + server4.Start("/tmp/dnode_test_dnode4", fqdn, 9044, firstEp); + server5.Start("/tmp/dnode_test_dnode5", fqdn, 9045, firstEp); taosMsleep(300); } static void TearDownTestSuite() { - stopServer(pServer1); - stopServer(pServer2); - stopServer(pServer3); - stopServer(pServer4); - stopServer(pServer5); - dropClient(pClient); - pServer1 = NULL; - pServer2 = NULL; - pServer3 = NULL; - pServer4 = NULL; - pServer5 = NULL; - pClient = NULL; - } - - static SServer* pServer1; - static SServer* pServer2; - static SServer* pServer3; - static SServer* pServer4; - static SServer* pServer5; - static SClient* pClient; - - public: - void SetUp() override {} - void TearDown() override {} - - void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) { - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = showType; - strcpy(pShow->db, ""); - - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htonl(pMeta->numOfTags); - pMeta->numOfColumns = htonl(pMeta->numOfColumns); - pMeta->sversion = htonl(pMeta->sversion); - pMeta->tversion = htonl(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_STREQ(pMeta->tbFname, showName); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->numOfColumns, columns); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->update, 0); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tuid, 0); - EXPECT_EQ(pMeta->suid, 0); - } - - void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { - SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, type); - EXPECT_EQ(pSchema->bytes, bytes); - EXPECT_STREQ(pSchema->name, name); - } - - void SendThenCheckShowRetrieveMsg(int32_t rows) { - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - // EXPECT_EQ(pRetrieveRsp->completed, completed); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - pData = pRetrieveRsp->data; - pos = 0; + server2.Stop(); + server3.Stop(); + server4.Stop(); + server5.Stop(); + test.Cleanup(); } - void CheckInt16(int16_t val) { - int16_t data = *((int16_t*)(pData + pos)); - pos += sizeof(int16_t); - EXPECT_EQ(data, val); - } - - void CheckInt64(int64_t val) { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_EQ(data, val); - } + static Testbase test; + static TestServer server2; + static TestServer server3; + static TestServer server4; + static TestServer server5; +}; - void CheckTimestamp() { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_GT(data, 0); - } +Testbase DndTestDnode::test; +TestServer DndTestDnode::server2; +TestServer DndTestDnode::server3; +TestServer DndTestDnode::server4; +TestServer DndTestDnode::server5; - void CheckBinary(const char* val, int32_t len) { - pos += sizeof(VarDataLenT); - char* data = (char*)(pData + pos); - pos += len; - EXPECT_STREQ(data, val); - } +TEST_F(DndTestDnode, 01_ShowDnode) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + CHECK_META("show dnodes", 7); - int32_t showId; - STableMetaMsg* pMeta; - SRetrieveTableRsp* pRetrieveRsp; - char* pData; - int32_t pos; -}; + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_SMALLINT, 2, "vnodes"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_SMALLINT, 2, "max_vnodes"); + CHECK_SCHEMA(4, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "status"); + CHECK_SCHEMA(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CHECK_SCHEMA(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline_reason"); -SServer* DndTestDnode::pServer1; -SServer* DndTestDnode::pServer2; -SServer* DndTestDnode::pServer3; -SServer* DndTestDnode::pServer4; -SServer* DndTestDnode::pServer5; -SClient* DndTestDnode::pClient; + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); -TEST_F(DndTestDnode, 01_ShowDnode) { - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - CheckSchema(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); - CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); - CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "vnodes"); - CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "max_vnodes"); - CheckSchema(4, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "status"); - CheckSchema(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline_reason"); - - SendThenCheckShowRetrieveMsg(1); CheckInt16(1); CheckBinary("localhost:9041", TSDB_EP_LEN); CheckInt16(0); @@ -202,40 +75,36 @@ TEST_F(DndTestDnode, 01_ShowDnode) { } TEST_F(DndTestDnode, 02_ConfigDnode) { - SCfgDnodeMsg* pReq = (SCfgDnodeMsg*)rpcMallocCont(sizeof(SCfgDnodeMsg)); + int32_t contLen = sizeof(SCfgDnodeMsg); + + SCfgDnodeMsg* pReq = (SCfgDnodeMsg*)rpcMallocCont(contLen); pReq->dnodeId = htonl(1); strcpy(pReq->config, "ddebugflag 131"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCfgDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CONFIG_DNODE; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONFIG_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { { - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9042"); + int32_t contLen = sizeof(SCreateDnodeMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->ep, "localhost:9042"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } taosMsleep(1300); - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - SendThenCheckShowRetrieveMsg(2); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + CHECK_META("show dnodes", 7); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + CheckInt16(1); CheckInt16(2); CheckBinary("localhost:9041", TSDB_EP_LEN); @@ -252,22 +121,21 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckBinary("", 24); { - SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg)); - pReq->dnodeId = htonl(2); + int32_t contLen = sizeof(SDropDnodeMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DNODE; + SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - SendThenCheckShowRetrieveMsg(1); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + CHECK_META("show dnodes", 7); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + CheckInt16(1); CheckBinary("localhost:9041", TSDB_EP_LEN); CheckInt16(0); @@ -277,53 +145,44 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { CheckBinary("", 24); { - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9043"); + int32_t contLen = sizeof(SCreateDnodeMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->ep, "localhost:9043"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } { - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9044"); + int32_t contLen = sizeof(SCreateDnodeMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->ep, "localhost:9044"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } { - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9045"); + int32_t contLen = sizeof(SCreateDnodeMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->ep, "localhost:9045"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } taosMsleep(1300); - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - SendThenCheckShowRetrieveMsg(4); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + CHECK_META("show dnodes", 7); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 4); + CheckInt16(1); CheckInt16(3); CheckInt16(4); @@ -355,31 +214,18 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { // restart uInfo("stop all server"); - stopServer(pServer1); - stopServer(pServer2); - stopServer(pServer3); - stopServer(pServer4); - stopServer(pServer5); - pServer1 = NULL; - pServer2 = NULL; - pServer3 = NULL; - pServer4 = NULL; - pServer5 = NULL; - - uInfo("start all server"); - - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9041"; - pServer1 = startServer("/tmp/dnode_test_dnode1", fqdn, 9041, firstEp); - pServer3 = startServer("/tmp/dnode_test_dnode3", fqdn, 9043, firstEp); - pServer4 = startServer("/tmp/dnode_test_dnode4", fqdn, 9044, firstEp); - pServer5 = startServer("/tmp/dnode_test_dnode5", fqdn, 9045, firstEp); - - uInfo("all server is running"); + test.Restart(); + server2.Restart(); + server3.Restart(); + server4.Restart(); + server5.Restart(); taosMsleep(1300); - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); - SendThenCheckShowRetrieveMsg(4); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + CHECK_META("show dnodes", 7); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 4); + CheckInt16(1); CheckInt16(3); CheckInt16(4); diff --git a/source/dnode/mgmt/impl/test/profile/CMakeLists.txt b/source/dnode/mgmt/impl/test/profile/CMakeLists.txt index d8e4eae0426037ecf20a7888a18360ccdd023391..0edd63172034b364e448f5164a4334d1a06f34d4 100644 --- a/source/dnode/mgmt/impl/test/profile/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/profile/CMakeLists.txt @@ -1,27 +1,11 @@ -add_executable(dndTestProfile "") - -target_sources(dndTestProfile - PRIVATE - "profile.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. PROFILE_SRC) +add_executable(dnode_test_profile ${PROFILE_SRC}) target_link_libraries( - dndTestProfile - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dndTestProfile - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + dnode_test_profile + PUBLIC sut ) add_test( - NAME dndTestProfile - COMMAND dndTestProfile + NAME dnode_test_profile + COMMAND dnode_test_profile ) diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index f1e4704777b1794ce8ff3bc183eeb34551b3db53..2e82d4d1aa6d1c5007ef7453cb4fe956118823c5 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -9,55 +9,35 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestProfile : public ::testing::Test { protected: - void SetUp() override {} - void TearDown() override {} + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_profile", 9080); } + static void TearDownTestSuite() { test.Cleanup(); } - static void SetUpTestSuite() { - const char* user = "root"; - const char* pass = "taosdata"; - const char* path = "/tmp/dndTestProfile"; - const char* fqdn = "localhost"; - uint16_t port = 9522; + static Testbase test; - pServer = createServer(path, fqdn, port); - ASSERT(pServer); - pClient = createClient(user, pass, fqdn, port); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - } + public: + void SetUp() override {} + void TearDown() override {} - static SServer* pServer; - static SClient* pClient; - static int32_t connId; + int32_t connId; }; -SServer* DndTestProfile::pServer; -SClient* DndTestProfile::pClient; -int32_t DndTestProfile::connId; +Testbase DndTestProfile::test; -TEST_F(DndTestProfile, SConnectMsg_01) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestProfile, 01_ConnectMsg) { + int32_t contLen = sizeof(SConnectMsg); - SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(contLen); pReq->pid = htonl(1234); - strcpy(pReq->app, "dndTestProfile"); + strcpy(pReq->app, "dnode_test_profile"); strcpy(pReq->db, ""); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SConnectMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); SConnectRsp* pRsp = (SConnectRsp*)pMsg->pCont; ASSERT_NE(pRsp, nullptr); @@ -69,183 +49,65 @@ TEST_F(DndTestProfile, SConnectMsg_01) { EXPECT_EQ(pRsp->acctId, 1); EXPECT_GT(pRsp->clusterId, 0); EXPECT_EQ(pRsp->connId, 1); - EXPECT_EQ(pRsp->superAuth, 1); - EXPECT_EQ(pRsp->readAuth, 1); - EXPECT_EQ(pRsp->writeAuth, 1); + EXPECT_EQ(pRsp->superUser, 1); EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9522); + EXPECT_EQ(pRsp->epSet.port[0], 9080); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); connId = pRsp->connId; } -TEST_F(DndTestProfile, SConnectMsg_02) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestProfile, 02_ConnectMsg_InvalidDB) { + int32_t contLen = sizeof(SConnectMsg); - SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(contLen); pReq->pid = htonl(1234); - strcpy(pReq->app, "dndTestProfile"); + strcpy(pReq->app, "dnode_test_profile"); strcpy(pReq->db, "invalid_db"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SConnectMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_DB); ASSERT_EQ(pMsg->contLen, 0); } -TEST_F(DndTestProfile, SConnectMsg_03) { - ASSERT_NE(pClient, nullptr); - int32_t showId = 0; - - { - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pReq->type = TSDB_MGMT_TABLE_CONNS; - strcpy(pReq->db, ""); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - - SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->showId = htonl(pRsp->showId); - STableMetaMsg* pMeta = &pRsp->tableMeta; - pMeta->contLen = htonl(pMeta->contLen); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); - pMeta->tid = htonl(pMeta->tid); - pMeta->uid = htobe64(pMeta->uid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pRsp->showId; - - EXPECT_NE(pRsp->showId, 0); - EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tbFname, ""); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->numOfColumns, 7); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tid, 0); - EXPECT_EQ(pMeta->uid, 0); - EXPECT_STREQ(pMeta->sTableName, ""); - EXPECT_EQ(pMeta->suid, 0); - - SSchema* pSchema = NULL; - pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "connId"); - - pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "user"); - - pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "program"); - - pSchema = &pMeta->pSchema[3]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "pid"); - - pSchema = &pMeta->pSchema[4]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "ip:port"); - - pSchema = &pMeta->pSchema[5]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); - EXPECT_EQ(pSchema->bytes, 8); - EXPECT_STREQ(pSchema->name, "login_time"); - - pSchema = &pMeta->pSchema[6]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); - EXPECT_EQ(pSchema->bytes, 8); - EXPECT_STREQ(pSchema->name, "last_access"); - } - - { - SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pReq->showId = htonl(showId); - pReq->free = 0; - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SRetrieveTableMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - - SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->compLen = htonl(pRsp->compLen); - - EXPECT_EQ(pRsp->numOfRows, 1); - EXPECT_EQ(pRsp->useconds, 0); - EXPECT_EQ(pRsp->completed, 1); - EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRsp->compressed, 0); - EXPECT_EQ(pRsp->compLen, 0); - } +TEST_F(DndTestProfile, 03_ConnectMsg_Show) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_CONNS, ""); + CHECK_META("show connections", 7); + CHECK_SCHEMA(0, TSDB_DATA_TYPE_INT, 4, "connId"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "user"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_BINARY, TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, "program"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_INT, 4, "pid"); + CHECK_SCHEMA(4, TSDB_DATA_TYPE_BINARY, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, "ip:port"); + CHECK_SCHEMA(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "login_time"); + CHECK_SCHEMA(6, TSDB_DATA_TYPE_TIMESTAMP, 8, "last_access"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + CheckInt32(1); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("dnode_test_profile", TSDB_APP_NAME_LEN); + CheckInt32(1234); + IgnoreBinary(TSDB_IPv4ADDR_LEN + 6); + CheckTimestamp(); + CheckTimestamp(); } -TEST_F(DndTestProfile, SHeartBeatMsg_01) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestProfile, 04_HeartBeatMsg) { + int32_t contLen = sizeof(SHeartBeatMsg); - SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); + SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(contLen); pReq->connId = htonl(connId); pReq->pid = htonl(1234); pReq->numOfQueries = htonl(0); pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "dndTestProfile"); + strcpy(pReq->app, "dnode_test_profile"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SHeartBeatMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_HEARTBEAT, pReq, contLen); ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pMsg->pCont; ASSERT_NE(pRsp, nullptr); @@ -265,62 +127,49 @@ TEST_F(DndTestProfile, SHeartBeatMsg_01) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9522); + EXPECT_EQ(pRsp->epSet.port[0], 9080); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } -TEST_F(DndTestProfile, SKillConnMsg_01) { - ASSERT_NE(pClient, nullptr); - +TEST_F(DndTestProfile, 05_KillConnMsg) { { - SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(sizeof(SKillConnMsg)); - pReq->connId = htonl(connId); + int32_t contLen = sizeof(SKillConnMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SKillConnMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_KILL_CONN; + SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(contLen); + pReq->connId = htonl(connId); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_KILL_CONN, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } { - SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); + int32_t contLen = sizeof(SHeartBeatMsg); + + SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(contLen); pReq->connId = htonl(connId); pReq->pid = htonl(1234); pReq->numOfQueries = htonl(0); pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "dndTestProfile"); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SHeartBeatMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; + strcpy(pReq->app, "dnode_test_profile"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_HEARTBEAT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONNECTION); ASSERT_EQ(pMsg->contLen, 0); } { - SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); + int32_t contLen = sizeof(SConnectMsg); + + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(contLen); pReq->pid = htonl(1234); - strcpy(pReq->app, "dndTestProfile"); + strcpy(pReq->app, "dnode_test_profile"); strcpy(pReq->db, ""); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SConnectMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); SConnectRsp* pRsp = (SConnectRsp*)pMsg->pCont; ASSERT_NE(pRsp, nullptr); @@ -332,71 +181,55 @@ TEST_F(DndTestProfile, SKillConnMsg_01) { EXPECT_EQ(pRsp->acctId, 1); EXPECT_GT(pRsp->clusterId, 0); EXPECT_GT(pRsp->connId, connId); - EXPECT_EQ(pRsp->readAuth, 1); - EXPECT_EQ(pRsp->writeAuth, 1); + EXPECT_EQ(pRsp->superUser, 1); EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9522); + EXPECT_EQ(pRsp->epSet.port[0], 9080); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); connId = pRsp->connId; } } -TEST_F(DndTestProfile, SKillConnMsg_02) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestProfile, 06_KillConnMsg_InvalidConn) { + int32_t contLen = sizeof(SKillConnMsg); - SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(sizeof(SKillConnMsg)); + SKillConnMsg* pReq = (SKillConnMsg*)rpcMallocCont(contLen); pReq->connId = htonl(2345); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SKillConnMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_KILL_CONN; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_KILL_CONN, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); } -TEST_F(DndTestProfile, SKillQueryMsg_01) { - ASSERT_NE(pClient, nullptr); - +TEST_F(DndTestProfile, 07_KillQueryMsg) { { - SKillQueryMsg* pReq = (SKillQueryMsg*)rpcMallocCont(sizeof(SKillQueryMsg)); + int32_t contLen = sizeof(SKillQueryMsg); + + SKillQueryMsg* pReq = (SKillQueryMsg*)rpcMallocCont(contLen); pReq->connId = htonl(connId); pReq->queryId = htonl(1234); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SKillQueryMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_KILL_QUERY; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_KILL_QUERY, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); ASSERT_EQ(pMsg->contLen, 0); } { - SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); + int32_t contLen = sizeof(SHeartBeatMsg); + + SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(contLen); pReq->connId = htonl(connId); pReq->pid = htonl(1234); pReq->numOfQueries = htonl(0); pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "dndTestProfile"); + strcpy(pReq->app, "dnode_test_profile"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SHeartBeatMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_HEARTBEAT, pReq, contLen); ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pMsg->pCont; ASSERT_NE(pRsp, nullptr); @@ -416,314 +249,42 @@ TEST_F(DndTestProfile, SKillQueryMsg_01) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9522); + EXPECT_EQ(pRsp->epSet.port[0], 9080); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } } -TEST_F(DndTestProfile, SKillQueryMsg_02) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestProfile, 08_KillQueryMsg_InvalidConn) { + int32_t contLen = sizeof(SKillQueryMsg); - SKillQueryMsg* pReq = (SKillQueryMsg*)rpcMallocCont(sizeof(SKillQueryMsg)); + SKillQueryMsg* pReq = (SKillQueryMsg*)rpcMallocCont(contLen); pReq->connId = htonl(2345); pReq->queryId = htonl(1234); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SKillQueryMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_KILL_QUERY; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_KILL_QUERY, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); } -TEST_F(DndTestProfile, SKillQueryMsg_03) { - ASSERT_NE(pClient, nullptr); - int32_t showId = 0; - - { - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pReq->type = TSDB_MGMT_TABLE_QUERIES; - strcpy(pReq->db, ""); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - - SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->showId = htonl(pRsp->showId); - STableMetaMsg* pMeta = &pRsp->tableMeta; - pMeta->contLen = htonl(pMeta->contLen); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); - pMeta->tid = htonl(pMeta->tid); - pMeta->uid = htobe64(pMeta->uid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pRsp->showId; - - EXPECT_NE(pRsp->showId, 0); - EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tbFname, ""); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->numOfColumns, 14); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tid, 0); - EXPECT_EQ(pMeta->uid, 0); - EXPECT_STREQ(pMeta->sTableName, ""); - EXPECT_EQ(pMeta->suid, 0); - - SSchema* pSchema = NULL; - pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "queryId"); - - pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "connId"); - - pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "user"); - - pSchema = &pMeta->pSchema[3]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "ip:port"); - } - - { - SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pReq->showId = htonl(showId); - pReq->free = 0; - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SRetrieveTableMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - - SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->offset = htobe64(pRsp->offset); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->compLen = htonl(pRsp->compLen); - - EXPECT_EQ(pRsp->numOfRows, 0); - EXPECT_EQ(pRsp->offset, 0); - EXPECT_EQ(pRsp->useconds, 0); - EXPECT_EQ(pRsp->completed, 1); - EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRsp->compressed, 0); - EXPECT_EQ(pRsp->reserved, 0); - EXPECT_EQ(pRsp->compLen, 0); - } -} - -TEST_F(DndTestProfile, SKillStreamMsg_01) { - ASSERT_NE(pClient, nullptr); - - { - SKillStreamMsg* pReq = (SKillStreamMsg*)rpcMallocCont(sizeof(SKillStreamMsg)); - pReq->connId = htonl(connId); - pReq->streamId = htonl(3579); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SKillStreamMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_KILL_STREAM; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - ASSERT_EQ(pMsg->contLen, 0); - } - - { - SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); - pReq->connId = htonl(connId); - pReq->pid = htonl(1234); - pReq->numOfQueries = htonl(0); - pReq->numOfStreams = htonl(0); - strcpy(pReq->app, "dndTestProfile"); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SHeartBeatMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - - SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->connId = htonl(pRsp->connId); - pRsp->queryId = htonl(pRsp->queryId); - pRsp->streamId = htonl(pRsp->streamId); - pRsp->totalDnodes = htonl(pRsp->totalDnodes); - pRsp->onlineDnodes = htonl(pRsp->onlineDnodes); - pRsp->epSet.port[0] = htons(pRsp->epSet.port[0]); - - EXPECT_EQ(pRsp->connId, connId); - EXPECT_EQ(pRsp->queryId, 0); - EXPECT_EQ(pRsp->streamId, 3579); - EXPECT_EQ(pRsp->totalDnodes, 1); - EXPECT_EQ(pRsp->onlineDnodes, 1); - EXPECT_EQ(pRsp->killConnection, 0); - - EXPECT_EQ(pRsp->epSet.inUse, 0); - EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9522); - EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); - } -} - -TEST_F(DndTestProfile, SKillStreamMsg_02) { - ASSERT_NE(pClient, nullptr); - - SKillStreamMsg* pReq = (SKillStreamMsg*)rpcMallocCont(sizeof(SKillStreamMsg)); - pReq->connId = htonl(2345); - pReq->streamId = htonl(1234); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SKillStreamMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_KILL_QUERY; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_CONN_ID); -} - -TEST_F(DndTestProfile, SKillStreamMsg_03) { - ASSERT_NE(pClient, nullptr); - int32_t showId = 0; - - { - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pReq->type = TSDB_MGMT_TABLE_STREAMS; - strcpy(pReq->db, ""); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - - SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->showId = htonl(pRsp->showId); - STableMetaMsg* pMeta = &pRsp->tableMeta; - pMeta->contLen = htonl(pMeta->contLen); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); - pMeta->tid = htonl(pMeta->tid); - pMeta->uid = htobe64(pMeta->uid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pRsp->showId; - - EXPECT_NE(pRsp->showId, 0); - EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tbFname, ""); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->numOfColumns, 10); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tid, 0); - EXPECT_EQ(pMeta->uid, 0); - EXPECT_STREQ(pMeta->sTableName, ""); - EXPECT_EQ(pMeta->suid, 0); - - SSchema* pSchema = NULL; - pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "streamId"); - - pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "connId"); - - pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "user"); - } - - { - SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pReq->showId = htonl(showId); - pReq->free = 0; - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SRetrieveTableMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - - SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->offset = htobe64(pRsp->offset); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->compLen = htonl(pRsp->compLen); - - EXPECT_EQ(pRsp->numOfRows, 0); - EXPECT_EQ(pRsp->offset, 0); - EXPECT_EQ(pRsp->useconds, 0); - EXPECT_EQ(pRsp->completed, 1); - EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRsp->compressed, 0); - EXPECT_EQ(pRsp->reserved, 0); - EXPECT_EQ(pRsp->compLen, 0); - } +TEST_F(DndTestProfile, 09_KillQueryMsg) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_QUERIES, ""); + CHECK_META("show queries", 14); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_INT, 4, "queryId"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_INT, 4, "connId"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "user"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_BINARY, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, "ip:port"); + CHECK_SCHEMA(4, TSDB_DATA_TYPE_BINARY, 22 + VARSTR_HEADER_SIZE, "qid"); + CHECK_SCHEMA(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "created_time"); + CHECK_SCHEMA(6, TSDB_DATA_TYPE_BIGINT, 8, "time"); + CHECK_SCHEMA(7, TSDB_DATA_TYPE_BINARY, 18 + VARSTR_HEADER_SIZE, "sql_obj_id"); + CHECK_SCHEMA(8, TSDB_DATA_TYPE_INT, 4, "pid"); + CHECK_SCHEMA(9, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "ep"); + CHECK_SCHEMA(10, TSDB_DATA_TYPE_BOOL, 1, "stable_query"); + CHECK_SCHEMA(11, TSDB_DATA_TYPE_INT, 4, "sub_queries"); + CHECK_SCHEMA(12, TSDB_DATA_TYPE_BINARY, TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, "sub_query_info"); + CHECK_SCHEMA(13, TSDB_DATA_TYPE_BINARY, TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, "sql"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); } diff --git a/source/dnode/mgmt/impl/test/show/CMakeLists.txt b/source/dnode/mgmt/impl/test/show/CMakeLists.txt index b3dc3297a8a8926707fc3d409555e0cdefff9c70..9786f65605ca25686b8d88e236d8834ce24274d9 100644 --- a/source/dnode/mgmt/impl/test/show/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/show/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_show "") - -target_sources(dnode_test_show - PRIVATE - "show.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. SHOW_SRC) +add_executable(dnode_test_show ${SHOW_SRC}) target_link_libraries( dnode_test_show - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_show - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mgmt/impl/test/show/show.cpp index f0c20f71e572640a190c724bab5cf9cf343dab7f..bec681cb8fed2115f70ebbb147a3c21926060a36 100644 --- a/source/dnode/mgmt/impl/test/show/show.cpp +++ b/source/dnode/mgmt/impl/test/show/show.cpp @@ -9,226 +9,79 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestShow : public ::testing::Test { protected: + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_show", 9091); } + static void TearDownTestSuite() { test.Cleanup(); } + + static Testbase test; + + public: void SetUp() override {} void TearDown() override {} - - static void SetUpTestSuite() { - const char* user = "root"; - const char* pass = "taosdata"; - const char* path = "/tmp/dndTestShow"; - const char* fqdn = "localhost"; - uint16_t port = 9523; - - pServer = createServer(path, fqdn, port); - ASSERT(pServer); - pClient = createClient(user, pass, fqdn, port); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - } - - static SServer* pServer; - static SClient* pClient; - static int32_t connId; }; -SServer* DndTestShow::pServer; -SClient* DndTestShow::pClient; -int32_t DndTestShow::connId; +Testbase DndTestShow::test; -TEST_F(DndTestShow, SShowMsg_01) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestShow, 01_ShowMsg_InvalidMsgMax) { + int32_t contLen = sizeof(SShowMsg); - SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); - pReq->pid = htonl(1234); - strcpy(pReq->app, "dndTestShow"); + SShowMsg* pReq = (SShowMsg*)rpcMallocCont(contLen); + pReq->type = TSDB_MGMT_TABLE_MAX; strcpy(pReq->db, ""); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SConnectMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_SHOW, pReq, contLen); ASSERT_NE(pMsg, nullptr); - - SConnectRsp* pRsp = (SConnectRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->connId = htonl(pRsp->connId); - - EXPECT_EQ(pRsp->connId, 1); - connId = pRsp->connId; + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); } -TEST_F(DndTestShow, SShowMsg_02) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestShow, 02_ShowMsg_InvalidMsgStart) { + int32_t contLen = sizeof(SShowMsg); SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pReq->type = TSDB_MGMT_TABLE_MAX; + pReq->type = TSDB_MGMT_TABLE_START; strcpy(pReq->db, ""); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_SHOW, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); } -TEST_F(DndTestShow, SShowMsg_03) { - ASSERT_NE(pClient, nullptr); +TEST_F(DndTestShow, 02_ShowMsg_Conn) { + int32_t contLen = sizeof(SConnectMsg); - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pReq->type = TSDB_MGMT_TABLE_START; + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(contLen); + pReq->pid = htonl(1234); + strcpy(pReq->app, "dnode_test_show"); strcpy(pReq->db, ""); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, TSDB_CODE_MND_INVALID_MSG_TYPE); -} - -TEST_F(DndTestShow, SShowMsg_04) { - ASSERT_NE(pClient, nullptr); - int32_t showId = 0; - - { - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pReq->type = TSDB_MGMT_TABLE_CONNS; - strcpy(pReq->db, ""); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - - SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->showId = htonl(pRsp->showId); - STableMetaMsg* pMeta = &pRsp->tableMeta; - pMeta->contLen = htonl(pMeta->contLen); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); - pMeta->tid = htonl(pMeta->tid); - pMeta->uid = htobe64(pMeta->uid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pRsp->showId; - - EXPECT_NE(pRsp->showId, 0); - EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tbFname, ""); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->numOfColumns, 7); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tid, 0); - EXPECT_EQ(pMeta->uid, 0); - EXPECT_STREQ(pMeta->sTableName, ""); - EXPECT_EQ(pMeta->suid, 0); - - SSchema* pSchema = NULL; - pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "connId"); - - pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "user"); - - pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "program"); - - pSchema = &pMeta->pSchema[3]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "pid"); - - pSchema = &pMeta->pSchema[4]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "ip:port"); - - pSchema = &pMeta->pSchema[5]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); - EXPECT_EQ(pSchema->bytes, 8); - EXPECT_STREQ(pSchema->name, "login_time"); - - pSchema = &pMeta->pSchema[6]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); - EXPECT_EQ(pSchema->bytes, 8); - EXPECT_STREQ(pSchema->name, "last_access"); - } - - { - SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pReq->showId = htonl(showId); - pReq->free = 0; - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SRetrieveTableMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - - SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->offset = htobe64(pRsp->offset); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->compLen = htonl(pRsp->compLen); - - EXPECT_EQ(pRsp->numOfRows, 1); - EXPECT_EQ(pRsp->offset, 0); - EXPECT_EQ(pRsp->useconds, 0); - EXPECT_EQ(pRsp->completed, 1); - EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRsp->compressed, 0); - EXPECT_EQ(pRsp->reserved, 0); - EXPECT_EQ(pRsp->compLen, 0); - } + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_CONNS, ""); + + STableMetaMsg* pMeta = test.GetShowMeta(); + EXPECT_STREQ(pMeta->tbFname, "show connections"); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->numOfColumns, 7); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->update, 0); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tuid, 0); + EXPECT_EQ(pMeta->suid, 0); + + test.SendShowRetrieveMsg(); + + SRetrieveTableRsp* pRetrieveRsp = test.GetRetrieveRsp(); + EXPECT_EQ(pRetrieveRsp->numOfRows, 1); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + EXPECT_EQ(pRetrieveRsp->completed, 1); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); } diff --git a/source/dnode/mgmt/impl/test/stb/CMakeLists.txt b/source/dnode/mgmt/impl/test/stb/CMakeLists.txt index 524e3e5c7ee5fdb65830c7cdcb32e5cfa985606b..68e4dfb07f114e7644638938130445549aa023ec 100644 --- a/source/dnode/mgmt/impl/test/stb/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/stb/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_stb "") - -target_sources(dnode_test_stb - PRIVATE - "stb.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. STB_SRC) +add_executable(dnode_test_stb ${STB_SRC}) target_link_libraries( dnode_test_stb - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_stb - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/stb/stb.cpp b/source/dnode/mgmt/impl/test/stb/stb.cpp index ee8c86bcea3b5764ca3158deabbced027b70d4ee..992ebaf902668c8d646d71969891c9d16831a601 100644 --- a/source/dnode/mgmt/impl/test/stb/stb.cpp +++ b/source/dnode/mgmt/impl/test/stb/stb.cpp @@ -9,176 +9,27 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestStb : public ::testing::Test { protected: - static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SServer* pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - return pServer; - } - - static void SetUpTestSuite() { - initLog("/tmp/tdlog"); + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_stb", 9101); } + static void TearDownTestSuite() { test.Cleanup(); } - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9101"; - pServer = CreateServer("/tmp/dnode_test_stb", fqdn, 9101, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9101); - taosMsleep(1100); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - pServer = NULL; - pClient = NULL; - } - - static SServer* pServer; - static SClient* pClient; - static int32_t connId; + static Testbase test; public: void SetUp() override {} void TearDown() override {} - - void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = showType; - if (db != NULL) { - strcpy(pShow->db, db); - } - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htonl(pMeta->numOfTags); - pMeta->numOfColumns = htonl(pMeta->numOfColumns); - pMeta->sversion = htonl(pMeta->sversion); - pMeta->tversion = htonl(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_STREQ(pMeta->tbFname, showName); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->numOfColumns, columns); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->update, 0); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tuid, 0); - EXPECT_EQ(pMeta->suid, 0); - } - - void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { - SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, type); - EXPECT_EQ(pSchema->bytes, bytes); - EXPECT_STREQ(pSchema->name, name); - } - - void SendThenCheckShowRetrieveMsg(int32_t rows) { - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - // EXPECT_EQ(pRetrieveRsp->completed, completed); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - pData = pRetrieveRsp->data; - pos = 0; - } - - void CheckInt8(int8_t val) { - int8_t data = *((int8_t*)(pData + pos)); - pos += sizeof(int8_t); - EXPECT_EQ(data, val); - } - - void CheckInt16(int16_t val) { - int16_t data = *((int16_t*)(pData + pos)); - pos += sizeof(int16_t); - EXPECT_EQ(data, val); - } - - void CheckInt32(int32_t val) { - int32_t data = *((int32_t*)(pData + pos)); - pos += sizeof(int32_t); - EXPECT_EQ(data, val); - } - - void CheckInt64(int64_t val) { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_EQ(data, val); - } - - void CheckTimestamp() { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_GT(data, 0); - } - - void CheckBinary(const char* val, int32_t len) { - pos += sizeof(VarDataLenT); - char* data = (char*)(pData + pos); - pos += len; - EXPECT_STREQ(data, val); - } - - int32_t showId; - STableMetaMsg* pMeta; - SRetrieveTableRsp* pRetrieveRsp; - char* pData; - int32_t pos; }; -SServer* DndTestStb::pServer; -SClient* DndTestStb::pClient; -int32_t DndTestStb::connId; +Testbase DndTestStb::test; TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { { - SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + int32_t contLen = sizeof(SCreateDbMsg); + + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(contLen); strcpy(pReq->db, "1.d1"); pReq->numOfVgroups = htonl(2); pReq->cacheBlockSize = htonl(16); @@ -187,8 +38,8 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep2 = htonl(3650); - pReq->minRowsPerFileBlock = htonl(100); - pReq->maxRowsPerFileBlock = htonl(4096); + pReq->minRows = htonl(100); + pReq->maxRows = htonl(4096); pReq->commitTime = htonl(3600); pReq->fsyncPeriod = htonl(3000); pReq->walLevel = 1; @@ -200,13 +51,7 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { pReq->cacheLastRow = 0; pReq->ignoreExist = 1; - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } @@ -214,9 +59,9 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { { int32_t cols = 2; int32_t tags = 3; - int32_t size = (tags + cols) * sizeof(SSchema) + sizeof(SCreateStbMsg); + int32_t contLen = (tags + cols) * sizeof(SSchema) + sizeof(SCreateStbMsg); - SCreateStbMsg* pReq = (SCreateStbMsg*)rpcMallocCont(size); + SCreateStbMsg* pReq = (SCreateStbMsg*)rpcMallocCont(contLen); strcpy(pReq->name, "1.d1.stb"); pReq->numOfTags = htonl(tags); pReq->numOfColumns = htonl(cols); @@ -261,24 +106,21 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { strcpy(pSchema->name, "tag3"); } - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = size; - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_STB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_STB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_STB, "show stables", 4, "1.d1"); - CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, "name"); - CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - CheckSchema(2, TSDB_DATA_TYPE_INT, 4, "columns"); - CheckSchema(3, TSDB_DATA_TYPE_INT, 4, "tags"); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_STB, "1.d1"); + CHECK_META("show stables", 4); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, "name"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_INT, 4, "columns"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_INT, 4, "tags"); - SendThenCheckShowRetrieveMsg(1); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("stb", TSDB_TABLE_NAME_LEN); CheckTimestamp(); CheckInt32(2); @@ -286,16 +128,12 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { // ----- meta ------ { - STableInfoMsg* pReq = (STableInfoMsg*)rpcMallocCont(sizeof(STableInfoMsg)); - strcpy(pReq->tableFname, "1.d1.stb"); + int32_t contLen = sizeof(STableInfoMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(STableInfoMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_TABLE_META; + STableInfoMsg* pReq = (STableInfoMsg*)rpcMallocCont(contLen); + strcpy(pReq->tableFname, "1.d1.stb"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_TABLE_META, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -336,39 +174,31 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { } // restart - stopServer(pServer); - pServer = NULL; + test.Restart(); - uInfo("start all server"); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_STB, "1.d1"); + CHECK_META("show stables", 4); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9101"; - pServer = startServer("/tmp/dnode_test_stb", fqdn, 9101, firstEp); - - uInfo("all server is running"); - - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_STB, "show stables", 4, "1.d1"); - SendThenCheckShowRetrieveMsg(1); CheckBinary("stb", TSDB_TABLE_NAME_LEN); CheckTimestamp(); CheckInt32(2); CheckInt32(3); { - SDropStbMsg* pReq = (SDropStbMsg*)rpcMallocCont(sizeof(SDropStbMsg)); - strcpy(pReq->name, "1.d1.stb"); + int32_t contLen = sizeof(SDropStbMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropStbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_STB; + SDropStbMsg* pReq = (SDropStbMsg*)rpcMallocCont(contLen); + strcpy(pReq->name, "1.d1.stb"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_STB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_STB, "show stables", 4, "1.d1"); - SendThenCheckShowRetrieveMsg(0); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_STB, "1.d1"); + CHECK_META("show stables", 4); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); } diff --git a/source/dnode/mgmt/impl/test/sut/CMakeLists.txt b/source/dnode/mgmt/impl/test/sut/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..d20c680fad3066e03a0ba6d93ab577b48e959755 --- /dev/null +++ b/source/dnode/mgmt/impl/test/sut/CMakeLists.txt @@ -0,0 +1,14 @@ +aux_source_directory(src SUT_SRC) +add_library(sut STATIC ${SUT_SRC}) +target_link_libraries( + sut + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories( + sut + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp deleted file mode 100644 index f2010b581388f24359ff2c15f4517a3fded22047..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "deploy.h" - -void initLog(const char* path) { - dDebugFlag = 143; - vDebugFlag = 0; - mDebugFlag = 207; - cDebugFlag = 0; - jniDebugFlag = 0; - tmrDebugFlag = 0; - uDebugFlag = 143; - rpcDebugFlag = 0; - odbcDebugFlag = 0; - qDebugFlag = 0; - wDebugFlag = 0; - sDebugFlag = 0; - tsdbDebugFlag = 0; - cqDebugFlag = 0; - tscEmbeddedInUtil = 1; - - taosRemoveDir(path); - taosMkDir(path); - - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", path); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { - printf("failed to init log file\n"); - } -} - -void* runServer(void* param) { - SServer* pServer = (SServer*)param; - while (1) { - taosMsleep(100); - pthread_testcancel(); - } -} - -void initOption(SDnodeOpt* pOption, const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - pOption->sver = 1; - pOption->numOfCores = 1; - pOption->numOfSupportMnodes = 1; - pOption->numOfSupportVnodes = 1; - pOption->numOfSupportQnodes = 1; - pOption->statusInterval = 1; - pOption->numOfThreadsPerCore = 1; - pOption->ratioOfQueryCores = 1; - pOption->maxShellConns = 1000; - pOption->shellActivityTimer = 30; - pOption->serverPort = port; - strcpy(pOption->dataDir, path); - snprintf(pOption->localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); - snprintf(pOption->localFqdn, TSDB_FQDN_LEN, "%s", fqdn); - snprintf(pOption->firstEp, TSDB_EP_LEN, "%s", firstEp); -} - -SServer* startServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - taosMkDir(path); - - SDnodeOpt option = {0}; - initOption(&option, path, fqdn, port, firstEp); - - SDnode* pDnode = dndInit(&option); - ASSERT(pDnode); - - SServer* pServer = (SServer*)calloc(1, sizeof(SServer)); - ASSERT(pServer); - - pServer->pDnode = pDnode; - pServer->threadId = taosCreateThread(runServer, pServer); - ASSERT(pServer->threadId); - - return pServer; -} - -SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - taosRemoveDir(path); - return startServer(path, fqdn, port, firstEp); -} - -void stopServer(SServer* pServer) { - if (pServer == NULL) return; - if (pServer->threadId != NULL) { - taosDestoryThread(pServer->threadId); - } - - if (pServer->pDnode != NULL) { - dndCleanup(pServer->pDnode); - pServer->pDnode = NULL; - } -} - -void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { - SClient* pClient = (SClient*)parent; - pClient->pRsp = pMsg; - uInfo("response:%s from dnode, pCont:%p contLen:%d code:0x%X", taosMsg[pMsg->msgType], pMsg->pCont, pMsg->contLen, - pMsg->code); - tsem_post(&pClient->sem); -} - -SClient* createClient(const char* user, const char* pass, const char* fqdn, uint16_t port) { - SClient* pClient = (SClient*)calloc(1, sizeof(SClient)); - ASSERT(pClient); - - char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; - taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); - - SRpcInit rpcInit; - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.label = (char*)"DND-C"; - rpcInit.numOfThreads = 1; - rpcInit.cfp = processClientRsp; - rpcInit.sessions = 1024; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = 30 * 1000; - rpcInit.user = (char*)user; - rpcInit.ckey = (char*)"key"; - rpcInit.parent = pClient; - rpcInit.secret = (char*)secretEncrypt; - rpcInit.parent = pClient; - // rpcInit.spi = 1; - - pClient->clientRpc = rpcOpen(&rpcInit); - ASSERT(pClient->clientRpc); - - tsem_init(&pClient->sem, 0, 0); - strcpy(pClient->fqdn, fqdn); - pClient->port = port; - - return pClient; -} - -void dropClient(SClient* pClient) { - tsem_destroy(&pClient->sem); - rpcClose(pClient->clientRpc); -} - -void sendMsg(SClient* pClient, SRpcMsg* pMsg) { - SEpSet epSet = {0}; - epSet.inUse = 0; - epSet.numOfEps = 1; - epSet.port[0] = pClient->port; - memcpy(epSet.fqdn[0], pClient->fqdn, TSDB_FQDN_LEN); - - rpcSendRequest(pClient->clientRpc, &epSet, pMsg, NULL); - tsem_wait(&pClient->sem); -} diff --git a/source/dnode/mgmt/impl/test/sut/deploy.h b/source/dnode/mgmt/impl/test/sut/deploy.h deleted file mode 100644 index 4c082df5f3b1592b474e1d62fa0068f28bde1233..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/impl/test/sut/deploy.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include "os.h" - -#include "dnode.h" -#include "taosmsg.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tnote.h" -#include "trpc.h" -#include "tthread.h" -#include "ulog.h" -#include "tdataformat.h" - -typedef struct { - SDnode* pDnode; - pthread_t* threadId; -} SServer; - -typedef struct { - char fqdn[TSDB_FQDN_LEN]; - uint16_t port; - void* clientRpc; - SRpcMsg* pRsp; - tsem_t sem; -} SClient; - -void initLog(const char* path); -SServer* createServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp); -SServer* startServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp); -void stopServer(SServer* pServer); -SClient* createClient(const char* user, const char* pass, const char* fqdn, uint16_t port); -void dropClient(SClient* pClient); -void sendMsg(SClient* pClient, SRpcMsg* pMsg); diff --git a/source/dnode/mgmt/impl/test/sut/inc/base.h b/source/dnode/mgmt/impl/test/sut/inc/base.h new file mode 100644 index 0000000000000000000000000000000000000000..b0de6f5c0c2b878bf57b7eede971126642959156 --- /dev/null +++ b/source/dnode/mgmt/impl/test/sut/inc/base.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TEST_BASE_H_ +#define _TD_TEST_BASE_H_ + +#include +#include "os.h" + +#include "dnode.h" +#include "taosmsg.h" +#include "tconfig.h" +#include "tdataformat.h" +#include "tglobal.h" +#include "tnote.h" +#include "trpc.h" +#include "tthread.h" +#include "ulog.h" + +#include "client.h" +#include "server.h" + +class Testbase { + public: + void Init(const char* path, int16_t port); + void Cleanup(); + void Restart(); + SRpcMsg* SendMsg(int8_t msgType, void* pCont, int32_t contLen); + + private: + void InitLog(const char* path); + + private: + TestServer server; + TestClient client; + int32_t connId; + + public: + void SendShowMetaMsg(int8_t showType, const char* db); + void SendShowRetrieveMsg(); + + STableMetaMsg* GetShowMeta(); + SRetrieveTableRsp* GetRetrieveRsp(); + + int32_t GetMetaNum(); + const char* GetMetaTbName(); + int32_t GetMetaColId(int32_t index); + int8_t GetMetaType(int32_t index); + int32_t GetMetaBytes(int32_t index); + const char* GetMetaName(int32_t index); + + const char* GetShowName(); + int32_t GetShowRows(); + int8_t GetShowInt8(); + int16_t GetShowInt16(); + int32_t GetShowInt32(); + int64_t GetShowInt64(); + int64_t GetShowTimestamp(); + const char* GetShowBinary(int32_t len); + + private: + int32_t showId; + STableMetaMsg* pMeta; + SRetrieveTableRsp* pRetrieveRsp; + char* pData; + int32_t pos; +}; + +#define CHECK_META(tbName, numOfColumns) \ + { \ + EXPECT_EQ(test.GetMetaNum(), numOfColumns); \ + EXPECT_STREQ(test.GetMetaTbName(), tbName); \ + } + +#define CHECK_SCHEMA(colId, type, bytes, colName) \ + { \ + EXPECT_EQ(test.GetMetaType(colId), type); \ + EXPECT_EQ(test.GetMetaBytes(colId), bytes); \ + EXPECT_STREQ(test.GetMetaName(colId), colName); \ + } + +#define CheckBinary(val, len) \ + { EXPECT_STREQ(test.GetShowBinary(len), val); } + +#define CheckInt8(val) \ + { EXPECT_EQ(test.GetShowInt8(), val); } + +#define CheckInt16(val) \ + { EXPECT_EQ(test.GetShowInt16(), val); } + +#define CheckInt32(val) \ + { EXPECT_EQ(test.GetShowInt32(), val); } + +#define CheckInt64(val) \ + { EXPECT_EQ(test.GetShowInt64(), val); } + +#define CheckTimestamp() \ + { EXPECT_GT(test.GetShowTimestamp(), 0); } + +#define IgnoreBinary(len) \ + { test.GetShowBinary(len); } + +#define IgnoreInt8() \ + { test.GetShowInt8(); } + +#define IgnoreInt16() \ + { test.GetShowInt16(); } + +#define IgnoreInt32() \ + { test.GetShowInt32(); } + +#define IgnoreInt64() \ + { test.GetShowInt64(); } + +#define IgnoreTimestamp() \ + { test.GetShowTimestamp(); } + +#endif /* _TD_TEST_BASE_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/inc/tsdbSMA.h b/source/dnode/mgmt/impl/test/sut/inc/client.h similarity index 59% rename from source/dnode/vnode/tsdb/inc/tsdbSMA.h rename to source/dnode/mgmt/impl/test/sut/inc/client.h index 800a276bc85c1cc97cc84b9deb214bb66c154a0b..e1e430318e891d2cf8c91740ebbc9d8d5de183cb 100644 --- a/source/dnode/vnode/tsdb/inc/tsdbSMA.h +++ b/source/dnode/mgmt/impl/test/sut/inc/client.h @@ -13,15 +13,24 @@ * along with this program. If not, see . */ -#ifndef _TD_TSDB_SMA_H_ -#define _TD_TSDB_SMA_H_ +#ifndef _TD_TEST_CLIENT_H_ +#define _TD_TEST_CLIENT_H_ -#ifdef __cplusplus -extern "C" { -#endif +class TestClient { + public: + bool Init(const char* user, const char* pass, const char* fqdn, uint16_t port); + void Cleanup(); -#ifdef __cplusplus -} -#endif + SRpcMsg* SendMsg(SRpcMsg* pMsg); + void SetRpcRsp(SRpcMsg* pRsp); + tsem_t* GetSem(); -#endif /*_TD_TSDB_SMA_H_*/ \ No newline at end of file + private: + char fqdn[TSDB_FQDN_LEN]; + uint16_t port; + void* clientRpc; + SRpcMsg* pRsp; + tsem_t sem; +}; + +#endif /* _TD_TEST_CLIENT_H_ */ \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/inc/tsdbIdx.h b/source/dnode/mgmt/impl/test/sut/inc/server.h similarity index 54% rename from source/dnode/vnode/tsdb/inc/tsdbIdx.h rename to source/dnode/mgmt/impl/test/sut/inc/server.h index 73b2c5e6c5eaa02db5961d70e3ad0ec89b10a2c8..aa7b37f22118ca0ca4081235624903dc010f85af 100644 --- a/source/dnode/vnode/tsdb/inc/tsdbIdx.h +++ b/source/dnode/mgmt/impl/test/sut/inc/server.h @@ -13,15 +13,26 @@ * along with this program. If not, see . */ -#ifndef _TD_TSDB_IDX_H_ -#define _TD_TSDB_IDX_H_ +#ifndef _TD_TEST_SERVER_H_ +#define _TD_TEST_SERVER_H_ -#ifdef __cplusplus -extern "C" { -#endif +class TestServer { + public: + bool Start(const char* path, const char* fqdn, uint16_t port, const char* firstEp); + void Stop(); + void Restart(); -#ifdef __cplusplus -} -#endif + private: + SDnodeOpt BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp); + bool DoStart(); -#endif /*_TD_TSDB_IDX_H_*/ \ No newline at end of file + private: + SDnode* pDnode; + pthread_t* threadId; + char path[PATH_MAX]; + char fqdn[TSDB_FQDN_LEN]; + char firstEp[TSDB_EP_LEN]; + uint16_t port; +}; + +#endif /* _TD_TEST_SERVER_H_ */ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/sut/src/base.cpp b/source/dnode/mgmt/impl/test/sut/src/base.cpp new file mode 100644 index 0000000000000000000000000000000000000000..59e99b1e1d9b740ac78079d2c899dd00e36b141b --- /dev/null +++ b/source/dnode/mgmt/impl/test/sut/src/base.cpp @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "base.h" + +void Testbase::InitLog(const char* path) { + dDebugFlag = 207; + vDebugFlag = 0; + mDebugFlag = 207; + cDebugFlag = 0; + jniDebugFlag = 0; + tmrDebugFlag = 0; + uDebugFlag = 143; + rpcDebugFlag = 0; + odbcDebugFlag = 0; + qDebugFlag = 0; + wDebugFlag = 0; + sDebugFlag = 0; + tsdbDebugFlag = 0; + cqDebugFlag = 0; + tscEmbeddedInUtil = 1; + + taosRemoveDir(path); + taosMkDir(path); + + char temp[PATH_MAX]; + snprintf(temp, PATH_MAX, "%s/taosdlog", path); + if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + printf("failed to init log file\n"); + } +} + +void Testbase::Init(const char* path, int16_t port) { + char fqdn[] = "localhost"; + char firstEp[TSDB_EP_LEN] = {0}; + snprintf(firstEp, TSDB_EP_LEN, "%s:%u", fqdn, port); + + InitLog("/tmp/tdlog"); + server.Start(path, fqdn, port, firstEp); + client.Init("root", "taosdata", fqdn, port); + taosMsleep(1100); +} + +void Testbase::Cleanup() { + server.Stop(); + client.Cleanup(); +} + +void Testbase::Restart() { server.Restart(); } + +SRpcMsg* Testbase::SendMsg(int8_t msgType, void* pCont, int32_t contLen) { + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pCont; + rpcMsg.contLen = contLen; + rpcMsg.msgType = msgType; + + return client.SendMsg(&rpcMsg); +} + +void Testbase::SendShowMetaMsg(int8_t showType, const char* db) { + int32_t contLen = sizeof(SShowMsg); + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(contLen); + pShow->type = showType; + strcpy(pShow->db, db); + + SRpcMsg* pMsg = SendMsg(TSDB_MSG_TYPE_SHOW, pShow, contLen); + SShowRsp* pShowRsp = (SShowRsp*)pMsg->pCont; + + ASSERT(pShowRsp != nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + pMeta = &pShowRsp->tableMeta; + pMeta->numOfTags = htonl(pMeta->numOfTags); + pMeta->numOfColumns = htonl(pMeta->numOfColumns); + pMeta->sversion = htonl(pMeta->sversion); + pMeta->tversion = htonl(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; +} + +int32_t Testbase::GetMetaColId(int32_t index) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->colId = htonl(pSchema->colId); + return pSchema->colId; +} + +int8_t Testbase::GetMetaType(int32_t index) { + SSchema* pSchema = &pMeta->pSchema[index]; + return pSchema->type; +} + +int32_t Testbase::GetMetaBytes(int32_t index) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->bytes = htonl(pSchema->bytes); + return pSchema->bytes; +} + +const char* Testbase::GetMetaName(int32_t index) { + SSchema* pSchema = &pMeta->pSchema[index]; + return pSchema->name; +} + +int32_t Testbase::GetMetaNum() { return pMeta->numOfColumns; } + +const char* Testbase::GetMetaTbName() { return pMeta->tbFname; } + +void Testbase::SendShowRetrieveMsg() { + int32_t contLen = sizeof(SRetrieveTableMsg); + + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(contLen); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg* pMsg = SendMsg(TSDB_MSG_TYPE_SHOW_RETRIEVE, pRetrieve, contLen); + pRetrieveRsp = (SRetrieveTableRsp*)pMsg->pCont; + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + pData = pRetrieveRsp->data; + pos = 0; +} + +const char* Testbase::GetShowName() { return pMeta->tbFname; } + +int8_t Testbase::GetShowInt8() { + int8_t data = *((int8_t*)(pData + pos)); + pos += sizeof(int8_t); + return data; +} + +int16_t Testbase::GetShowInt16() { + int16_t data = *((int16_t*)(pData + pos)); + pos += sizeof(int16_t); + return data; +} + +int32_t Testbase::GetShowInt32() { + int32_t data = *((int32_t*)(pData + pos)); + pos += sizeof(int32_t); + return data; +} + +int64_t Testbase::GetShowInt64() { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + return data; +} + +int64_t Testbase::GetShowTimestamp() { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + return data; +} + +const char* Testbase::GetShowBinary(int32_t len) { + pos += sizeof(VarDataLenT); + char* data = (char*)(pData + pos); + pos += len; + return data; +} + +int32_t Testbase::GetShowRows() { return pRetrieveRsp->numOfRows; } + +STableMetaMsg* Testbase::GetShowMeta() { return pMeta; } + +SRetrieveTableRsp* Testbase::GetRetrieveRsp() { return pRetrieveRsp; } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fce3da15073e830908390fe95d9b98f0fcc5bada --- /dev/null +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "base.h" + +static void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { + TestClient* client = (TestClient*)parent; + client->SetRpcRsp(pMsg); + uInfo("response:%s from dnode, code:0x%x", taosMsg[pMsg->msgType], pMsg->code); + tsem_post(client->GetSem()); +} + +void TestClient::SetRpcRsp(SRpcMsg* pRsp) { this->pRsp = pRsp; }; + +tsem_t* TestClient::GetSem() { return &sem; } + +bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) { + char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; + taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.label = (char*)"DND-C"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processClientRsp; + rpcInit.sessions = 1024; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = 30 * 1000; + rpcInit.user = (char*)user; + rpcInit.ckey = (char*)"key"; + rpcInit.parent = this; + rpcInit.secret = (char*)secretEncrypt; + // rpcInit.spi = 1; + + clientRpc = rpcOpen(&rpcInit); + ASSERT(clientRpc); + + tsem_init(&sem, 0, 0); + strcpy(this->fqdn, fqdn); + this->port = port; + + return true; +} + +void TestClient::Cleanup() { + tsem_destroy(&sem); + rpcClose(clientRpc); +} + +SRpcMsg* TestClient::SendMsg(SRpcMsg* pMsg) { + SEpSet epSet = {0}; + epSet.inUse = 0; + epSet.numOfEps = 1; + epSet.port[0] = port; + memcpy(epSet.fqdn[0], fqdn, TSDB_FQDN_LEN); + + rpcSendRequest(clientRpc, &epSet, pMsg, NULL); + tsem_wait(&sem); + + return pRsp; +} diff --git a/source/dnode/mgmt/impl/test/sut/src/server.cpp b/source/dnode/mgmt/impl/test/sut/src/server.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e96f7deaf5e1c9466a5b94a29cc8d1ad25c8f6d4 --- /dev/null +++ b/source/dnode/mgmt/impl/test/sut/src/server.cpp @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "base.h" + +void* serverLoop(void* param) { + while (1) { + taosMsleep(100); + pthread_testcancel(); + } +} + +SDnodeOpt TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SDnodeOpt option = {0}; + option.sver = 1; + option.numOfCores = 1; + option.numOfSupportMnodes = 1; + option.numOfSupportVnodes = 1; + option.numOfSupportQnodes = 1; + option.statusInterval = 1; + option.numOfThreadsPerCore = 1; + option.ratioOfQueryCores = 1; + option.maxShellConns = 1000; + option.shellActivityTimer = 30; + option.serverPort = port; + strcpy(option.dataDir, path); + snprintf(option.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); + snprintf(option.localFqdn, TSDB_FQDN_LEN, "%s", fqdn); + snprintf(option.firstEp, TSDB_EP_LEN, "%s", firstEp); + return option; +} + +bool TestServer::DoStart() { + SDnodeOpt option = BuildOption(path, fqdn, port, firstEp); + taosMkDir(path); + + pDnode = dndInit(&option); + if (pDnode != NULL) { + return false; + } + + threadId = taosCreateThread(serverLoop, NULL); + if (threadId != NULL) { + return false; + } + return true; +} + +void TestServer::Restart() { + uInfo("start all server"); + Stop(); + DoStart(); + uInfo("all server is running"); +} + +bool TestServer::Start(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + strcpy(this->path, path); + strcpy(this->fqdn, fqdn); + this->port = port; + strcpy(this->firstEp, firstEp); + + taosRemoveDir(path); + return DoStart(); +} + +void TestServer::Stop() { + if (threadId != NULL) { + taosDestoryThread(threadId); + threadId = NULL; + } + + if (pDnode != NULL) { + dndCleanup(pDnode); + pDnode = NULL; + } +} diff --git a/source/dnode/mgmt/impl/test/user/CMakeLists.txt b/source/dnode/mgmt/impl/test/user/CMakeLists.txt index 0d3215a4d0445ab99c8180b31716b12841e21c34..5068660eba1fea5964664819b9a51686cbde2adf 100644 --- a/source/dnode/mgmt/impl/test/user/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/user/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_user "") - -target_sources(dnode_test_user - PRIVATE - "user.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. USER_SRC) +add_executable(dnode_test_user ${USER_SRC}) target_link_libraries( dnode_test_user - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_user - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 32309b106e8f6aef905fb8fd8fb5c72e6d28ecda..a1810e7adaaa7011775dd2e9fbcc3892cce720e6 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -9,168 +9,34 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestUser : public ::testing::Test { protected: - static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SServer* pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - return pServer; - } - - static void SetUpTestSuite() { - initLog("/tmp/tdlog"); - - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9140"; - pServer = CreateServer("/tmp/dnode_test_user", fqdn, 9140, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9140); - taosMsleep(300); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - pServer = NULL; - pClient = NULL; - } + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_user", 9140); } + static void TearDownTestSuite() { test.Cleanup(); } - static SServer* pServer; - static SClient* pClient; - static int32_t connId; + static Testbase test; public: void SetUp() override {} void TearDown() override {} - - void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) { - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = showType; - strcpy(pShow->db, ""); - - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htonl(pMeta->numOfTags); - pMeta->numOfColumns = htonl(pMeta->numOfColumns); - pMeta->sversion = htonl(pMeta->sversion); - pMeta->tversion = htonl(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_STREQ(pMeta->tbFname, showName); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->numOfColumns, columns); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->update, 0); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tuid, 0); - EXPECT_EQ(pMeta->suid, 0); - } - - void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { - SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, type); - EXPECT_EQ(pSchema->bytes, bytes); - EXPECT_STREQ(pSchema->name, name); - } - - void SendThenCheckShowRetrieveMsg(int32_t rows) { - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - // EXPECT_EQ(pRetrieveRsp->completed, completed); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - pData = pRetrieveRsp->data; - pos = 0; - } - - void CheckInt16(int16_t val) { - int16_t data = *((int16_t*)(pData + pos)); - pos += sizeof(int16_t); - EXPECT_EQ(data, val); - } - - void CheckInt64(int64_t val) { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_EQ(data, val); - } - - void CheckTimestamp() { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_GT(data, 0); - } - - void CheckBinary(const char* val, int32_t len) { - pos += sizeof(VarDataLenT); - char* data = (char*)(pData + pos); - pos += len; - EXPECT_STREQ(data, val); - } - - int32_t showId; - STableMetaMsg* pMeta; - SRetrieveTableRsp* pRetrieveRsp; - char* pData; - int32_t pos; }; -SServer* DndTestUser::pServer; -SClient* DndTestUser::pClient; -int32_t DndTestUser::connId; +Testbase DndTestUser::test; TEST_F(DndTestUser, 01_ShowUser) { - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name"); - CheckSchema(1, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "privilege"); - CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); - CheckSchema(3, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "account"); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + CHECK_META("show users", 4); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "privilege"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "account"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); - SendThenCheckShowRetrieveMsg(1); CheckBinary("root", TSDB_USER_LEN); CheckBinary("super", 10); CheckTimestamp(); @@ -179,39 +45,35 @@ TEST_F(DndTestUser, 01_ShowUser) { TEST_F(DndTestUser, 02_Create_Drop_Alter_User) { { - SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); + int32_t contLen = sizeof(SCreateUserMsg); + + SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(contLen); strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p1"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_USER, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } { - SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); + int32_t contLen = sizeof(SCreateUserMsg); + + SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(contLen); strcpy(pReq->user, "u2"); strcpy(pReq->pass, "p2"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_USER, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - SendThenCheckShowRetrieveMsg(3); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + CHECK_META("show users", 4); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 3); + CheckBinary("u1", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); CheckBinary("u2", TSDB_USER_LEN); @@ -226,22 +88,23 @@ TEST_F(DndTestUser, 02_Create_Drop_Alter_User) { CheckBinary("root", TSDB_USER_LEN); { - SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); + int32_t contLen = sizeof(SAlterUserMsg); + + SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(contLen); strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p2"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SAlterUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_ALTER_USER, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - SendThenCheckShowRetrieveMsg(3); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + CHECK_META("show users", 4); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 3); + CheckBinary("u1", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); CheckBinary("u2", TSDB_USER_LEN); @@ -256,21 +119,22 @@ TEST_F(DndTestUser, 02_Create_Drop_Alter_User) { CheckBinary("root", TSDB_USER_LEN); { - SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); - strcpy(pReq->user, "u1"); + int32_t contLen = sizeof(SDropUserMsg); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_USER; + SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(contLen); + strcpy(pReq->user, "u1"); - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_USER, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - SendThenCheckShowRetrieveMsg(2); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + CHECK_META("show users", 4); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + CheckBinary("root", TSDB_USER_LEN); CheckBinary("u2", TSDB_USER_LEN); CheckBinary("super", 10); @@ -281,19 +145,14 @@ TEST_F(DndTestUser, 02_Create_Drop_Alter_User) { CheckBinary("root", TSDB_USER_LEN); // restart - stopServer(pServer); - pServer = NULL; - - uInfo("start all server"); + test.Restart(); - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9140"; - pServer = startServer("/tmp/dnode_test_user", fqdn, 9140, firstEp); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_USER, ""); + CHECK_META("show users", 4); - uInfo("all server is running"); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - SendThenCheckShowRetrieveMsg(2); CheckBinary("root", TSDB_USER_LEN); CheckBinary("u2", TSDB_USER_LEN); CheckBinary("super", 10); diff --git a/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt b/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt index 5670f9dbf2c4ac2c3609494f50bd4c82bd77fef1..b864b0593c26d15c4468c381f95c1ac337e1174f 100644 --- a/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt @@ -1,24 +1,8 @@ -add_executable(dnode_test_vgroup "") - -target_sources(dnode_test_vgroup - PRIVATE - "vgroup.cpp" - "../sut/deploy.cpp" -) - +aux_source_directory(. VGROUP_SRC) +add_executable(dnode_test_vgroup ${VGROUP_SRC}) target_link_libraries( dnode_test_vgroup - PUBLIC dnode - PUBLIC util - PUBLIC os - PUBLIC gtest_main -) - -target_include_directories(dnode_test_vgroup - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" - "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" - "${CMAKE_CURRENT_SOURCE_DIR}/../sut" + PUBLIC sut ) add_test( diff --git a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp b/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp index 3f16cd87d889902b3e7d34fbbf6a2cb2d83e6771..c534fcbabe96b28af1a917bbca718ad6210070b5 100644 --- a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp +++ b/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp @@ -9,216 +9,121 @@ * */ -#include "deploy.h" +#include "base.h" class DndTestVgroup : public ::testing::Test { protected: - static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SServer* pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - return pServer; - } - - static void SetUpTestSuite() { - initLog("/tmp/tdlog"); - - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9150"; - pServer = CreateServer("/tmp/dnode_test_vgroup", fqdn, 9150, firstEp); - pClient = createClient("root", "taosdata", fqdn, 9150); - taosMsleep(1100); - } - - static void TearDownTestSuite() { - stopServer(pServer); - dropClient(pClient); - pServer = NULL; - pClient = NULL; - } + static void SetUpTestSuite() { test.Init("/tmp/dnode_test_vgroup", 9150); } + static void TearDownTestSuite() { test.Cleanup(); } - static SServer* pServer; - static SClient* pClient; - static int32_t connId; + static Testbase test; public: void SetUp() override {} void TearDown() override {} - - void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = showType; - if (db != NULL) { - strcpy(pShow->db, db); - } - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htonl(pMeta->numOfTags); - pMeta->numOfColumns = htonl(pMeta->numOfColumns); - pMeta->sversion = htonl(pMeta->sversion); - pMeta->tversion = htonl(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_STREQ(pMeta->tbFname, showName); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->numOfColumns, columns); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->update, 0); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tuid, 0); - EXPECT_EQ(pMeta->suid, 0); - } - - void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { - SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htonl(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, type); - EXPECT_EQ(pSchema->bytes, bytes); - EXPECT_STREQ(pSchema->name, name); - } - - void SendThenCheckShowRetrieveMsg(int32_t rows) { - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - ASSERT_NE(pClient->pRsp->pCont, nullptr); - - pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - // EXPECT_EQ(pRetrieveRsp->completed, completed); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - pData = pRetrieveRsp->data; - pos = 0; - } - - void CheckInt8(int8_t val) { - int8_t data = *((int8_t*)(pData + pos)); - pos += sizeof(int8_t); - EXPECT_EQ(data, val); - } - - void CheckInt16(int16_t val) { - int16_t data = *((int16_t*)(pData + pos)); - pos += sizeof(int16_t); - EXPECT_EQ(data, val); - } - - void CheckInt32(int32_t val) { - int32_t data = *((int32_t*)(pData + pos)); - pos += sizeof(int32_t); - EXPECT_EQ(data, val); - } - - void CheckInt64(int64_t val) { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_EQ(data, val); - } - - void CheckTimestamp() { - int64_t data = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_GT(data, 0); - } - - void CheckBinary(const char* val, int32_t len) { - pos += sizeof(VarDataLenT); - char* data = (char*)(pData + pos); - pos += len; - EXPECT_STREQ(data, val); - } - - int32_t showId; - STableMetaMsg* pMeta; - SRetrieveTableRsp* pRetrieveRsp; - char* pData; - int32_t pos; }; -SServer* DndTestVgroup::pServer; -SClient* DndTestVgroup::pClient; -int32_t DndTestVgroup::connId; - +Testbase DndTestVgroup::test; TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { { - SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(sizeof(SCreateVnodeMsg)); - pReq->vgId = htonl(2); - pReq->dnodeId = htonl(1); - strcpy(pReq->db, "1.d1"); - pReq->dbUid = htobe64(9527); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->minRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replica = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->selfIndex = 0; - for (int r = 0; r < pReq->replica; ++r) { - SReplica* pReplica = &pReq->replicas[r]; - pReplica->id = htonl(1); - pReplica->port = htons(9150); + for (int i = 0; i < 3; ++i) { + int32_t contLen = sizeof(SCreateVnodeMsg); + + SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(contLen); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(1); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + pReq->vgVersion = htonl(1); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRows = htonl(100); + pReq->minRows = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replica = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->selfIndex = 0; + for (int r = 0; r < pReq->replica; ++r) { + SReplica* pReplica = &pReq->replicas[r]; + pReplica->id = htonl(1); + pReplica->port = htons(9150); + } + + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_VNODE_IN, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); } + } - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateVnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - taosMsleep(1000000); + { + for (int i = 0; i < 3; ++i) { + int32_t contLen = sizeof(SAlterVnodeMsg); + + SAlterVnodeMsg* pReq = (SAlterVnodeMsg*)rpcMallocCont(contLen); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(1); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + pReq->vgVersion = htonl(2); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRows = htonl(100); + pReq->minRows = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replica = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->selfIndex = 0; + for (int r = 0; r < pReq->replica; ++r) { + SReplica* pReplica = &pReq->replicas[r]; + pReplica->id = htonl(1); + pReplica->port = htons(9150); + } + + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_ALTER_VNODE_IN, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } } + { + for (int i = 0; i < 3; ++i) { + int32_t contLen = sizeof(SDropVnodeMsg); + + SDropVnodeMsg* pReq = (SDropVnodeMsg*)rpcMallocCont(contLen); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(1); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropVnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_VNODE_IN; + + SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_VNODE_IN, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + } } - diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 36b6725737bb3b4cd6a070da92fe01060f80d17e..da5492057458b09aed332168919ce1b38a46e959 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -61,15 +61,14 @@ typedef enum { } EAuthOp; typedef enum { - TRN_STAGE_PREPARE = 1, - TRN_STAGE_EXECUTE = 2, + TRN_STAGE_PREPARE = 0, + TRN_STAGE_EXECUTE = 1, + TRN_STAGE_ROLLBACK = 2, TRN_STAGE_COMMIT = 3, - TRN_STAGE_ROLLBACK = 4, - TRN_STAGE_RETRY = 5, - TRN_STAGE_OVER = 6, + TRN_STAGE_OVER = 4, } ETrnStage; -typedef enum { TRN_POLICY_ROLLBACK = 1, TRN_POLICY_RETRY = 2 } ETrnPolicy; +typedef enum { TRN_POLICY_ROLLBACK = 0, TRN_POLICY_RETRY = 1 } ETrnPolicy; typedef enum { DND_STATUS_OFFLINE = 0, diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index e9913803bd7f2fe88db87c2ea2b44aafb062f554..ba72d9553721b9220aa520b8bb11b46c1b7baeb5 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -70,6 +70,7 @@ typedef struct SMnode { tmr_h timer; char *path; SMnodeCfg cfg; + int64_t checkTime; SSdb *pSdb; SDnode *pDnode; SArray *pSteps; diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 5c15e2f987d89ad00c26201bfcdc85352d2a71b0..a8d37ba655ef52c6f4fdf4b77938cab4bb0a6046 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -22,6 +22,16 @@ extern "C" { #endif +typedef struct { + SEpSet epSet; + int8_t msgType; + int8_t msgSent; + int8_t msgReceived; + int32_t errCode; + int32_t contLen; + void *pCont; +} STransAction; + int32_t mndInitTrans(SMnode *pMnode); void mndCleanupTrans(SMnode *pMnode); @@ -30,12 +40,15 @@ void mndTransDrop(STrans *pTrans); int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); -int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); -int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); +int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction); +int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); + int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code); -char *mndTransStageStr(ETrnStage stage); -char *mndTransPolicyStr(ETrnPolicy policy); +void mndTransHandleActionRsp(SMnodeMsg *pMsg); + +char *mndTransStageStr(ETrnStage stage); +char *mndTransPolicyStr(ETrnPolicy policy); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index e9cdedd332bd2f9d29a3e3d606dc856df14b9eec..8a3a2c798af479ca3a419cf2be2560735b100d81 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -26,8 +26,9 @@ int32_t mndInitVgroup(SMnode *pMnode); void mndCleanupVgroup(SMnode *pMnode); SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); -int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); +int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); +SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup); SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 75b0acbd0a6df8e98b1a3304edb15701284805dc..4c2706f4819a8477a99353a3a05409894aad20c0 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -17,7 +17,8 @@ #include "mndAcct.h" #include "mndShow.h" -#define SDB_ACCT_VER 1 +#define TSDB_ACCT_VER_NUMBER 1 +#define TSDB_ACCT_RESERVE_SIZE 64 static int32_t mndCreateDefaultAcct(SMnode *pMnode); static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct); @@ -70,7 +71,7 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) { } static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, SDB_ACCT_VER, sizeof(SAcctObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, TSDB_ACCT_VER_NUMBER, sizeof(SAcctObj) + TSDB_ACCT_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -85,6 +86,7 @@ static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) { SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.maxStreams) SDB_SET_INT64(pRaw, dataPos, pAcct->cfg.maxStorage) SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.accessState) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_ACCT_RESERVE_SIZE) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; @@ -94,7 +96,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != SDB_ACCT_VER) { + if (sver != TSDB_ACCT_VER_NUMBER) { mError("failed to decode acct since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -116,13 +118,13 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, pRow, dataPos, &pAcct->cfg.maxStreams) SDB_GET_INT64(pRaw, pRow, dataPos, &pAcct->cfg.maxStorage) SDB_GET_INT32(pRaw, pRow, dataPos, &pAcct->cfg.accessState) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_ACCT_RESERVE_SIZE) return pRow; } static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct) { mTrace("acct:%s, perform insert action", pAcct->acct); - memset(&pAcct->info, 0, sizeof(SAcctInfo)); return 0; } @@ -134,12 +136,9 @@ static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) { static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct) { mTrace("acct:%s, perform update action", pOldAcct->acct); - memcpy(pOldAcct->acct, pNewAcct->acct, TSDB_USER_LEN); - pOldAcct->createdTime = pNewAcct->createdTime; pOldAcct->updateTime = pNewAcct->updateTime; - pOldAcct->acctId = pNewAcct->acctId; pOldAcct->status = pNewAcct->status; - pOldAcct->cfg = pNewAcct->cfg; + memcpy(&pOldAcct->cfg, &pNewAcct->cfg, sizeof(SAcctInfo)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 7c53a4ebd0516c13856e9d36d506c8babfc880c8..cf8511c0541cc4642e2e18c104496a4acbb259ee 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -18,7 +18,8 @@ #include "mndShow.h" #include "mndTrans.h" -#define SDB_CLUSTER_VER 1 +#define TSDB_CLUSTER_VER_NUMBE 1 +#define TSDB_CLUSTER_RESERVE_SIZE 64 static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw); @@ -62,7 +63,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { } static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, SDB_CLUSTER_VER, sizeof(SClusterObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, TSDB_CLUSTER_VER_NUMBE, sizeof(SClusterObj) + TSDB_CLUSTER_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -70,6 +71,7 @@ static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { SDB_SET_INT64(pRaw, dataPos, pCluster->createdTime) SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime) SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_CLUSTER_RESERVE_SIZE) return pRaw; } @@ -78,7 +80,7 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != SDB_CLUSTER_VER) { + if (sver != TSDB_CLUSTER_VER_NUMBE) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to decode cluster since %s", terrstr()); return NULL; @@ -93,6 +95,7 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pCluster->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pCluster->updateTime) SDB_GET_BINARY(pRaw, pRow, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_CLUSTER_RESERVE_SIZE) return pRow; } @@ -169,6 +172,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->numOfRows = 1; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e85edf66da85011fe9b1c32e18a3568fb3ecc477..108896c121eeb1d3d93ffd3a1a9c93cc6f413235 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -16,13 +16,12 @@ #define _DEFAULT_SOURCE #include "mndDb.h" #include "mndDnode.h" -#include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" -#define TSDB_DB_VER_NUM 1 +#define TSDB_DB_VER_NUMBER 1 #define TSDB_DB_RESERVE_SIZE 64 static SSdbRaw *mndDbActionEncode(SDbObj *pDb); @@ -66,7 +65,7 @@ int32_t mndInitDb(SMnode *pMnode) { void mndCleanupDb(SMnode *pMnode) {} static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUM, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -106,7 +105,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_DB_VER_NUM) { + if (sver != TSDB_DB_VER_NUMBER) { mError("failed to decode db since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -160,7 +159,7 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { mTrace("db:%s, perform update action", pOldDb->name); - pOldDb->updateTime = pNewDb->createdTime; + pOldDb->updateTime = pNewDb->updateTime; pOldDb->cfgVersion = pNewDb->cfgVersion; pOldDb->vgVersion = pNewDb->vgVersion; memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg)); @@ -168,8 +167,12 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { } SDbObj *mndAcquireDb(SMnode *pMnode, char *db) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_DB, db); + SSdb *pSdb = pMnode->pSdb; + SDbObj *pDb = sdbAcquire(pSdb, SDB_DB, db); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + } + return pDb; } void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { @@ -242,68 +245,74 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; } -static int32_t mndSetRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { +static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { SSdbRaw *pDbRaw = mndDbActionEncode(pDb); - if (pDbRaw == NULL || mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1; - sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING); + if (pDbRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1; + if (sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING) != 0) return -1; - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { + for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) { SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); - if (pVgRaw == NULL || mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; - sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING); + if (pVgRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1; } return 0; } -static int32_t mndSetUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { +static int32_t mndSetCreateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { SSdbRaw *pDbRaw = mndDbActionEncode(pDb); - if (pDbRaw == NULL || mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1; - sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED); + if (pDbRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1; + if (sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED) != 0) return -1; - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { + for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) { SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); - if (pVgRaw == NULL || mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1; - sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED); + if (pVgRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED) != 0) return -1; } return 0; } -static int32_t mndSetCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { +static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { SSdbRaw *pDbRaw = mndDbActionEncode(pDb); - if (pDbRaw == NULL || mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1; - sdbSetRawStatus(pDbRaw, SDB_STATUS_READY); + if (pDbRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1; + if (sdbSetRawStatus(pDbRaw, SDB_STATUS_READY) != 0) return -1; - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { + for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) { SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); - if (pVgRaw == NULL || mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; - sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; } return 0; } -static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { - SVgObj *pVgroup = pVgroups + v; +static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + for (int32_t vg = 0; vg < pDb->cfg.numOfVgroups; ++vg) { + SVgObj *pVgroup = pVgroups + vg; for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) { - return -1; - } + STransAction action = {0}; + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SEpSet epset = mndGetDnodeEpset(pDnode); + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); SCreateVnodeMsg *pMsg = mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup); - if (pMsg == NULL) { - return -1; - } + if (pMsg == NULL) return -1; - if (mndTransAppendRedoAction(pTrans, &epset, TSDB_MSG_TYPE_ALTER_VNODE_IN, sizeof(SCreateVnodeMsg), pMsg) != 0) { + action.pCont = pMsg; + action.contLen = sizeof(SCreateVnodeMsg); + action.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); return -1; } @@ -313,26 +322,26 @@ static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SV return 0; } -static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { - SVgObj *pVgroup = pVgroups + v; +static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + for (int32_t vg = 0; vg < pDb->cfg.numOfVgroups; ++vg) { + SVgObj *pVgroup = pVgroups + vg; for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) { - return -1; - } + STransAction action = {0}; + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SEpSet epset = mndGetDnodeEpset(pDnode); + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); SDropVnodeMsg *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup); - if (pMsg == NULL) { - return -1; - } + if (pMsg == NULL) return -1; - if (mndTransAppendUndoAction(pTrans, &epset, TSDB_MSG_TYPE_DROP_VNODE_IN, sizeof(SDropVnodeMsg), pMsg) != 0) { + action.pCont = pMsg; + action.contLen = sizeof(SDropVnodeMsg); + action.msgType = TSDB_MSG_TYPE_DROP_VNODE_IN; + if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pMsg); return -1; } @@ -344,14 +353,14 @@ static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SV static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, SUserObj *pUser) { SDbObj dbObj = {0}; - tstrncpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN); - tstrncpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); + memcpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN); + memcpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); dbObj.createdTime = taosGetTimestampMs(); dbObj.updateTime = dbObj.createdTime; dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN); - dbObj.hashMethod = 1; dbObj.cfgVersion = 1; dbObj.vgVersion = 1; + dbObj.hashMethod = 1; dbObj.cfg = (SDbCfg){.numOfVgroups = pCreate->numOfVgroups, .cacheBlockSize = pCreate->cacheBlockSize, .totalBlocks = pCreate->totalBlocks, @@ -359,8 +368,8 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat .daysToKeep0 = pCreate->daysToKeep0, .daysToKeep1 = pCreate->daysToKeep1, .daysToKeep2 = pCreate->daysToKeep2, - .minRows = pCreate->minRowsPerFileBlock, - .maxRows = pCreate->maxRowsPerFileBlock, + .minRows = pCreate->minRows, + .maxRows = pCreate->maxRows, .fsyncPeriod = pCreate->fsyncPeriod, .commitTime = pCreate->commitTime, .precision = pCreate->precision, @@ -396,29 +405,30 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat mError("db:%s, failed to create since %s", pCreate->db, terrstr()); goto CREATE_DB_OVER; } + mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); - if (mndSetRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } - if (mndSetUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } - if (mndSetCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } - if (mndSetRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } - if (mndSetUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } @@ -447,8 +457,8 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0); pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); - pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); - pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); + pCreate->minRows = htonl(pCreate->minRows); + pCreate->maxRows = htonl(pCreate->maxRows); pCreate->commitTime = htonl(pCreate->commitTime); pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod); @@ -456,7 +466,7 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { SDbObj *pDb = mndAcquireDb(pMnode, pCreate->db); if (pDb != NULL) { - sdbRelease(pMnode->pSdb, pDb); + mndReleaseDb(pMnode, pDb); if (pCreate->ignoreExist) { mDebug("db:%s, already exist, ignore exist is set", pCreate->db); return 0; @@ -485,57 +495,77 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { } static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) { - bool changed = false; + terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) { pDb->cfg.totalBlocks = pAlter->totalBlocks; - changed = true; + terrno = 0; } if (pAlter->daysToKeep0 >= 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) { pDb->cfg.daysToKeep0 = pAlter->daysToKeep0; - changed = true; + terrno = 0; } if (pAlter->daysToKeep1 >= 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) { pDb->cfg.daysToKeep1 = pAlter->daysToKeep1; - changed = true; + terrno = 0; } if (pAlter->daysToKeep2 >= 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) { pDb->cfg.daysToKeep2 = pAlter->daysToKeep2; - changed = true; + terrno = 0; } if (pAlter->fsyncPeriod >= 0 && pAlter->fsyncPeriod != pDb->cfg.fsyncPeriod) { pDb->cfg.fsyncPeriod = pAlter->fsyncPeriod; - changed = true; + terrno = 0; } if (pAlter->walLevel >= 0 && pAlter->walLevel != pDb->cfg.walLevel) { pDb->cfg.walLevel = pAlter->walLevel; - changed = true; + terrno = 0; } if (pAlter->quorum >= 0 && pAlter->quorum != pDb->cfg.quorum) { pDb->cfg.quorum = pAlter->quorum; - changed = true; + terrno = 0; } if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) { pDb->cfg.cacheLastRow = pAlter->cacheLastRow; - changed = true; + terrno = 0; } - if (!changed) { - terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; - return -1; - } + return terrno; +} + +static int32_t mndSetUpdateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { + SSdbRaw *pRedoRaw = mndDbActionEncode(pNewDb); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; return 0; } +static int32_t mndSetUpdateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { + SSdbRaw *pUndoRaw = mndDbActionEncode(pOldDb); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; } + +static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; } + +static int32_t mndSetUpdateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; } + static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbObj *pNewDb) { + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { mError("db:%s, failed to update since %s", pOldDb->name, terrstr()); @@ -544,30 +574,41 @@ static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbO mDebug("trans:%d, used to update db:%s", pTrans->id, pOldDb->name); - SSdbRaw *pRedoRaw = mndDbActionEncode(pNewDb); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetUpdateDbRedoLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - SSdbRaw *pUndoRaw = mndDbActionEncode(pOldDb); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetUpdateDbUndoLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; + } + + if (mndSetUpdateDbCommitLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; + } + + if (mndSetUpdateDbRedoActions(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; + } + + if (mndSetUpdateDbUndoActions(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + goto UPDATE_DB_OVER; } + code = 0; + +UPDATE_DB_OVER: mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { @@ -610,46 +651,82 @@ static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } +static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdbRaw *pRedoRaw = mndDbActionEncode(pDb); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdbRaw *pUndoRaw = mndDbActionEncode(pDb); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdbRaw *pCommitRaw = mndDbActionEncode(pDb); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return 0; } + +static int32_t mndSetDropDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return 0; } + static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pDb) { + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { mError("db:%s, failed to drop since %s", pDb->name, terrstr()); return -1; } + mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); - SSdbRaw *pRedoRaw = mndDbActionEncode(pDb); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); - SSdbRaw *pUndoRaw = mndDbActionEncode(pDb); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropDbUndoLogs(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); - SSdbRaw *pCommitRaw = mndDbActionEncode(pDb); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; + } + + if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; + } + + if (mndSetDropDbUndoActions(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + goto DROP_DB_OVER; } + code = 0; + +DROP_DB_OVER: mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { @@ -751,29 +828,27 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) { static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SSyncDbMsg *pSync = pMsg->rpcMsg.pCont; - - SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); if (pDb == NULL) { mError("db:%s, failed to process sync db msg since %s", pMsg->db, terrstr()); return -1; - } else { - mndReleaseDb(pMnode, pDb); - return 0; } + + mndReleaseDb(pMnode, pDb); + return 0; } static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SCompactDbMsg *pCompact = pMsg->rpcMsg.pCont; - - SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); if (pDb == NULL) { mError("db:%s, failed to process compact db msg since %s", pMsg->db, terrstr()); return -1; - } else { - mndReleaseDb(pMnode, pDb); - return 0; } + + mndReleaseDb(pMnode, pDb); + return 0; } static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 493f20bc9af3319521fd602e2ed8e6648305ca22..9751771773c53fa55ec965a1fed8be02013f7588 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -18,10 +18,10 @@ #include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" -#include "ttime.h" #include "tep.h" +#include "ttime.h" -#define TSDB_DNODE_VER 1 +#define TSDB_DNODE_VER_NUMBER 1 #define TSDB_DNODE_RESERVE_SIZE 64 #define TSDB_CONFIG_OPTION_LEN 16 #define TSDB_CONIIG_VALUE_LEN 48 @@ -101,14 +101,14 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { SSdbRaw *pRaw = mndDnodeActionEncode(&dnodeObj); if (pRaw == NULL) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); + if (sdbSetRawStatus(pRaw, SDB_STATUS_READY) != 0) return -1; mDebug("dnode:%d, will be created while deploy sdb", dnodeObj.id); return sdbWrite(pMnode->pSdb, pRaw); } static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER, sizeof(SDnodeObj) + TSDB_DNODE_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER_NUMBER, sizeof(SDnodeObj) + TSDB_DNODE_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -127,7 +127,7 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_DNODE_VER) { + if (sver != TSDB_DNODE_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to decode dnode since %s", terrstr()); return NULL; @@ -150,21 +150,8 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) { mTrace("dnode:%d, perform insert action", pDnode->id); - - pDnode->rebootTime = 0; - pDnode->lastAccessTime = 0; - pDnode->accessTimes = 0; - pDnode->numOfMnodes = 0; - pDnode->numOfVnodes = 0; - pDnode->numOfQnodes = 0; - pDnode->numOfSupportMnodes = 0; - pDnode->numOfSupportVnodes = 0; - pDnode->numOfSupportQnodes = 0; - pDnode->numOfCores = 0; - pDnode->status = DND_STATUS_OFFLINE; pDnode->offlineReason = DND_REASON_STATUS_NOT_RECEIVED; snprintf(pDnode->ep, TSDB_EP_LEN, "%s:%u", pDnode->fqdn, pDnode->port); - return 0; } @@ -225,7 +212,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode) { int64_t ms = taosGetTimestampMs(); int64_t interval = ABS(pDnode->lastAccessTime - ms); - if (interval > 3000 * pMnode->cfg.statusInterval) { + if (interval > 3500 * pMnode->cfg.statusInterval) { return false; } return true; @@ -267,12 +254,9 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { return DND_REASON_STATUS_INTERVAL_NOT_MATCH; } - int64_t checkTime = 0; - char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (checkTime != pCfg->checkTime)) { + if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (pMnode->checkTime != pCfg->checkTime)) { mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone, - pCfg->checkTime, checkTime); + pCfg->checkTime, pMnode->checkTime); return DND_REASON_TIME_ZONE_NOT_MATCH; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 16733cbf94bd93fc6193d6ca7bf0cd6c39e88f94..869b6e538b711b8782144041c450a410210fb67a 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -19,7 +19,8 @@ #include "mndShow.h" #include "mndTrans.h" -#define SDB_MNODE_VER 1 +#define TSDB_MNODE_VER_NUMBER 1 +#define TSDB_MNODE_RESERVE_SIZE 64 static int32_t mndCreateDefaultMnode(SMnode *pMnode); static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj); @@ -60,8 +61,12 @@ int32_t mndInitMnode(SMnode *pMnode) { void mndCleanupMnode(SMnode *pMnode) {} static SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_MNODE, &mnodeId); + SSdb *pSdb = pMnode->pSdb; + SMnodeObj *pObj = sdbAcquire(pSdb, SDB_MNODE, &mnodeId); + if (pObj == NULL) { + terrno = TSDB_CODE_MND_MNODE_NOT_EXIST; + } + return pObj; } static void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pMnodeObj) { @@ -97,13 +102,14 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) { } static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, SDB_MNODE_VER, sizeof(SMnodeObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, TSDB_MNODE_VER_NUMBER, sizeof(SMnodeObj) + TSDB_MNODE_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; SDB_SET_INT32(pRaw, dataPos, pMnodeObj->id); SDB_SET_INT64(pRaw, dataPos, pMnodeObj->createdTime) SDB_SET_INT64(pRaw, dataPos, pMnodeObj->updateTime) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_MNODE_RESERVE_SIZE) return pRaw; } @@ -112,7 +118,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != SDB_MNODE_VER) { + if (sver != TSDB_MNODE_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to decode mnode since %s", terrstr()); return NULL; @@ -126,6 +132,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, pRow, dataPos, &pMnodeObj->id) SDB_GET_INT64(pRaw, pRow, dataPos, &pMnodeObj->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pMnodeObj->updateTime) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_MNODE_RESERVE_SIZE) return pRow; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index bf657fd27aabdbb22ed3b1c01fb85052b60cb506..a4b73e0f3f2547bf6a013eae16c927e1658a0ad7 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -23,29 +23,25 @@ #define QUERY_ID_SIZE 20 #define QUERY_OBJ_ID_SIZE 18 #define SUBQUERY_INFO_SIZE 6 -#define QUERY_STREAM_SAVE_SIZE 20 +#define QUERY_SAVE_SIZE 20 typedef struct { - char user[TSDB_USER_LEN]; - char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc - int32_t pid; // pid of app that invokes taosc - int64_t appStartTime; // app start time - int32_t id; - int8_t killed; - int8_t align; - uint16_t port; - uint32_t ip; - int64_t stime; - int64_t lastAccess; - int32_t queryId; - int32_t streamId; - int32_t numOfQueries; - int32_t numOfStreams; - SStreamDesc *pStreams; - SQueryDesc *pQueries; + int32_t id; + char user[TSDB_USER_LEN]; + char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc + int64_t appStartTimeMs; // app start time + int32_t pid; // pid of app that invokes taosc + uint32_t ip; + uint16_t port; + int8_t killed; + int64_t loginTimeMs; + int64_t lastAccessTimeMs; + int32_t queryId; + int32_t numOfQueries; + SQueryDesc *pQueries; } SConnObj; -static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime); +static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, const char *app, int64_t startTime); static void mndFreeConn(SConnObj *pConn); static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId); static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); @@ -54,16 +50,12 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg); static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg); static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg); -static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg); static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg); static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); static int32_t mndRetrieveQueries(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); -static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); -static int32_t mndRetrieveStreams(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); -static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); int32_t mndInitProfile(SMnode *pMnode) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -79,7 +71,6 @@ int32_t mndInitProfile(SMnode *pMnode) { mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_HEARTBEAT, mndProcessHeartBeatMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONNECT, mndProcessConnectMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_QUERY, mndProcessKillQueryMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_STREAM, mndProcessKillStreamMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_CONN, mndProcessKillConnectionMsg); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndGetConnsMeta); @@ -88,9 +79,6 @@ int32_t mndInitProfile(SMnode *pMnode) { mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndGetQueryMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndRetrieveQueries); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_QUERIES, mndCancelGetNextQuery); - mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndGetStreamMeta); - mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStreams); - mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream); return 0; } @@ -103,46 +91,43 @@ void mndCleanupProfile(SMnode *pMnode) { } } -static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app, int64_t startTime) { +static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, const char *app, int64_t startTime) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; int32_t connId = atomic_add_fetch_32(&pMgmt->connId, 1); if (connId == 0) atomic_add_fetch_32(&pMgmt->connId, 1); + if (startTime == 0) startTime = taosGetTimestampMs(); - SConnObj connObj = {.pid = pid, - .appStartTime = startTime, - .id = connId, + SConnObj connObj = {.id = connId, + .appStartTimeMs = startTime, + .pid = pid, + .ip = pInfo->clientIp, + .port = pInfo->clientPort, .killed = 0, - .port = port, - .ip = ip, - .stime = taosGetTimestampMs(), - .lastAccess = 0, + .loginTimeMs = taosGetTimestampMs(), + .lastAccessTimeMs = 0, .queryId = 0, - .streamId = 0, .numOfQueries = 0, - .numOfStreams = 0, - .pStreams = NULL, .pQueries = NULL}; - connObj.lastAccess = connObj.stime; - tstrncpy(connObj.user, user, TSDB_USER_LEN); + connObj.lastAccessTimeMs = connObj.loginTimeMs; + tstrncpy(connObj.user, pInfo->user, TSDB_USER_LEN); tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN); int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); if (pConn == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("conn:%d, data:%p failed to put into cache since %s, user:%s", connId, pConn, user, terrstr()); + mError("conn:%d, data:%p failed to put into cache since %s, user:%s", connId, pConn, pInfo->user, terrstr()); return NULL; } else { - mTrace("conn:%d, data:%p created, user:%s", pConn->id, pConn, user); + mTrace("conn:%d, data:%p created, user:%s", pConn->id, pConn, pInfo->user); return pConn; } } static void mndFreeConn(SConnObj *pConn) { tfree(pConn->pQueries); - tfree(pConn->pStreams); mTrace("conn:%d, data:%p destroyed", pConn->id, pConn); } @@ -156,7 +141,7 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId) { } int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; - pConn->lastAccess = keepTime * 1000 + (uint64_t)taosGetTimestampMs(); + pConn->lastAccessTimeMs = keepTime * 1000 + (uint64_t)taosGetTimestampMs(); mTrace("conn:%d, data:%p acquired from cache", pConn->id, pConn); return pConn; @@ -219,7 +204,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { mndReleaseDb(pMnode, pDb); } - SConnObj *pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app, pReq->startTime); + SConnObj *pConn = mndCreateConn(pMnode, &info, pReq->pid, pReq->app, pReq->startTime); if (pConn == NULL) { mError("user:%s, failed to login from %s while create connection since %s", pMsg->user, ip, terrstr()); return -1; @@ -254,16 +239,14 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) { pConn->numOfQueries = 0; - pConn->numOfStreams = 0; int32_t numOfQueries = htonl(pMsg->numOfQueries); - int32_t numOfStreams = htonl(pMsg->numOfStreams); if (numOfQueries > 0) { if (pConn->pQueries == NULL) { - pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_STREAM_SAVE_SIZE); + pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_SAVE_SIZE); } - pConn->numOfQueries = MIN(QUERY_STREAM_SAVE_SIZE, numOfQueries); + pConn->numOfQueries = MIN(QUERY_SAVE_SIZE, numOfQueries); int32_t saveSize = pConn->numOfQueries * sizeof(SQueryDesc); if (saveSize > 0 && pConn->pQueries != NULL) { @@ -271,19 +254,6 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) { } } - if (numOfStreams > 0) { - if (pConn->pStreams == NULL) { - pConn->pStreams = calloc(sizeof(SStreamDesc), QUERY_STREAM_SAVE_SIZE); - } - - pConn->numOfStreams = MIN(QUERY_STREAM_SAVE_SIZE, numOfStreams); - - int32_t saveSize = pConn->numOfStreams * sizeof(SStreamDesc); - if (saveSize > 0 && pConn->pStreams != NULL) { - memcpy(pConn->pStreams, pMsg->pData + numOfQueries * sizeof(SQueryDesc), saveSize); - } - } - return TSDB_CODE_SUCCESS; } @@ -303,7 +273,7 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { SConnObj *pConn = mndAcquireConn(pMnode, pReq->connId); if (pConn == NULL) { - pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app, 0); + pConn = mndCreateConn(pMnode, &info, pReq->pid, pReq->app, 0); if (pConn == NULL) { mError("user:%s, conn:%d is freed and failed to create new conn since %s", pMsg->user, pReq->connId, terrstr()); return -1; @@ -343,11 +313,6 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { pRsp->killConnection = 1; } - if (pConn->streamId != 0) { - pRsp->streamId = htonl(pConn->streamId); - pConn->streamId = 0; - } - if (pConn->queryId != 0) { pRsp->queryId = htonl(pConn->queryId); pConn->queryId = 0; @@ -395,37 +360,6 @@ static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg) { } } -static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SProfileMgmt *pMgmt = &pMnode->profileMgmt; - - SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); - if (pUser == NULL) return 0; - if (!pUser->superUser) { - mndReleaseUser(pMnode, pUser); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - mndReleaseUser(pMnode, pUser); - - SKillStreamMsg *pKill = pMsg->rpcMsg.pCont; - int32_t connId = htonl(pKill->connId); - int32_t streamId = htonl(pKill->streamId); - mDebug("kill stream msg is received, streamId:%d", streamId); - - SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); - if (pConn == NULL) { - mError("connId:%d, failed to kill streamId:%d, conn not exist", connId, streamId); - terrno = TSDB_CODE_MND_INVALID_CONN_ID; - return -1; - } else { - mInfo("connId:%d, streamId:%d is killed by user:%s", connId, streamId, pMsg->user); - pConn->streamId = streamId; - taosCacheRelease(pMgmt->cache, (void **)&pConn, false); - return TSDB_CODE_SUCCESS; - } -} - static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -525,6 +459,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->numOfRows = taosHashGetSize(pMgmt->cache->pHashTable); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } @@ -567,12 +502,12 @@ static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = pConn->stime; + *(int64_t *)pWrite = pConn->loginTimeMs; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - if (pConn->lastAccess < pConn->stime) pConn->lastAccess = pConn->stime; - *(int64_t *)pWrite = pConn->lastAccess; + if (pConn->lastAccessTimeMs < pConn->loginTimeMs) pConn->lastAccessTimeMs = pConn->loginTimeMs; + *(int64_t *)pWrite = pConn->lastAccessTimeMs; cols++; numOfRows++; @@ -623,7 +558,7 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 24; + pShow->bytes[cols] = 22 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "qid"); pSchema[cols].bytes = htonl(pShow->bytes[cols]); @@ -693,6 +628,7 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->numOfRows = 1000000; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } @@ -790,6 +726,7 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pMsg, SShowObj *pShow, char *data, } } + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; return numOfRows; } @@ -798,173 +735,3 @@ static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); } - -static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { - SMnode *pMnode = pMsg->pMnode; - SProfileMgmt *pMgmt = &pMnode->profileMgmt; - - SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); - if (pUser == NULL) return 0; - if (!pUser->superUser) { - mndReleaseUser(pMnode, pUser); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - mndReleaseUser(pMnode, pUser); - - int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "streamId"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "connId"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "user"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "destination"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "ip:port"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 8; - pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 8; - pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "exec"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 8; - pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; - strcpy(pSchema[cols].name, "time(us)"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "sql"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "cycles"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pMeta->numOfColumns = htonl(cols); - pShow->numOfColumns = cols; - - pShow->offset[0] = 0; - for (int32_t i = 1; i < cols; ++i) { - pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; - } - - pShow->numOfRows = 1000000; - pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - - return 0; -} - -static int32_t mndRetrieveStreams(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; - int32_t numOfRows = 0; - SConnObj *pConn = NULL; - int32_t cols = 0; - char *pWrite; - void *pIter; - char ipStr[TSDB_IPv4ADDR_LEN + 6]; - - while (numOfRows < rows) { - pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn); - if (pConn == NULL) { - pShow->pIter = pIter; - break; - } - - if (numOfRows + pConn->numOfStreams >= rows) { - mndCancelGetNextConn(pMnode, pIter); - break; - } - - pShow->pIter = pIter; - for (int32_t i = 0; i < pConn->numOfStreams; ++i) { - SStreamDesc *pDesc = pConn->pStreams + i; - cols = 0; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = htobe64(pDesc->streamId); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = htobe64(pConn->id); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConn->user, pShow->bytes[cols]); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->dstTable, pShow->bytes[cols]); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConn->ip), pConn->port); - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = htobe64(pDesc->ctime); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = htobe64(pDesc->stime); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = htobe64(pDesc->useconds); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->sql, pShow->bytes[cols]); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = (int32_t)htobe64(pDesc->num); - cols++; - - numOfRows++; - } - } - - pShow->numOfReads += numOfRows; - return numOfRows; -} - -static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) { - SProfileMgmt *pMgmt = &pMnode->profileMgmt; - taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); -} diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index db6777ebf8129d41fac21b95d373246e96127bb8..41f09e85a99b55637a44388d2bd1106ba43b961d 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -138,7 +138,7 @@ static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) { ShowMetaFp metaFp = pMgmt->metaFps[type]; if (metaFp == NULL) { terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; - mError("failed to process show-meta msg:%s since no message handle", mndShowStr(type)); + mError("failed to process show-meta msg:%s since %s", mndShowStr(type), terrstr()); return -1; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 822036b599b5c2cfcb086403b9122416b729b76e..a51312a2a924e5d51cd5c8279426faf6e6016b59 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -21,9 +21,10 @@ #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" +#include "mndVgroup.h" #include "tname.h" -#define TSDB_STB_VER_NUM 1 +#define TSDB_STB_VER_NUMBER 1 #define TSDB_STB_RESERVE_SIZE 64 static SSdbRaw *mndStbActionEncode(SStbObj *pStb); @@ -70,7 +71,7 @@ void mndCleanupStb(SMnode *pMnode) {} static SSdbRaw *mndStbActionEncode(SStbObj *pStb) { int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + TSDB_STB_RESERVE_SIZE; - SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUM, size); + SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -103,7 +104,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_STB_VER_NUM) { + if (sver != TSDB_STB_VER_NUMBER) { mError("failed to decode stable since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -176,8 +177,12 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb } SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_STB, stbName); + SSdb *pSdb = pMnode->pSdb; + SStbObj *pStb = sdbAcquire(pSdb, SDB_STB, stbName); + if (pStb == NULL) { + terrno = TSDB_CODE_MND_STB_NOT_EXIST; + } + return pStb; } void mndReleaseStb(SMnode *pMnode, SStbObj *pStb) { @@ -195,7 +200,54 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, char *stbName) { return mndAcquireDb(pMnode, db); } -static int32_t mndCheckStbMsg(SCreateStbMsg *pCreate) { +static SCreateStbInternalMsg *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { + int32_t totalCols = pStb->numOfTags + pStb->numOfColumns; + int32_t contLen = totalCols * sizeof(SSchema) + sizeof(SCreateStbInternalMsg); + + SCreateStbInternalMsg *pCreate = calloc(1, contLen); + if (pCreate == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pCreate->head.contLen = htonl(contLen); + pCreate->head.vgId = htonl(pVgroup->vgId); + memcpy(pCreate->name, pStb->name, TSDB_TABLE_FNAME_LEN); + pCreate->suid = htobe64(pStb->uid); + pCreate->sverson = htonl(pStb->version); + pCreate->ttl = 0; + pCreate->keep = 0; + pCreate->numOfTags = htonl(pStb->numOfTags); + pCreate->numOfColumns = htonl(pStb->numOfColumns); + + memcpy(pCreate->pSchema, pStb->pSchema, totalCols * sizeof(SSchema)); + for (int32_t t = 0; t < totalCols; ++t) { + SSchema *pSchema = &pCreate->pSchema[t]; + pSchema->bytes = htonl(pSchema->bytes); + pSchema->colId = htonl(pSchema->colId); + } + + return pCreate; +} + +static SDropStbInternalMsg *mndBuildDropStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { + int32_t contLen = sizeof(SDropStbInternalMsg); + + SDropStbInternalMsg *pDrop = calloc(1, contLen); + if (pDrop == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pDrop->head.contLen = htonl(contLen); + pDrop->head.vgId = htonl(pVgroup->vgId); + memcpy(pDrop->name, pStb->name, TSDB_TABLE_FNAME_LEN); + pDrop->suid = htobe64(pStb->uid); + + return pDrop; +} + +static int32_t mndCheckCreateStbMsg(SCreateStbMsg *pCreate) { pCreate->numOfColumns = htonl(pCreate->numOfColumns); pCreate->numOfTags = htonl(pCreate->numOfTags); int32_t totalCols = pCreate->numOfColumns + pCreate->numOfTags; @@ -244,6 +296,103 @@ static int32_t mndCheckStbMsg(SCreateStbMsg *pCreate) { return 0; } +static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateStbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdbRaw *pUndoRaw = mndStbActionEncode(pStb); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) continue; + + SCreateStbInternalMsg *pMsg = mndBuildCreateStbMsg(pMnode, pVgroup, pStb); + if (pMsg == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pMsg; + action.contLen = htonl(pMsg->head.contLen); + action.msgType = TSDB_MSG_TYPE_CREATE_STB_IN; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) continue; + + SDropStbInternalMsg *pMsg = mndBuildDropStbMsg(pMnode, pVgroup, pStb); + if (pMsg == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pMsg; + action.contLen = sizeof(SDropStbInternalMsg); + action.msgType = TSDB_MSG_TYPE_DROP_STB_IN; + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + free(pMsg); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCreate, SDbObj *pDb) { SStbObj stbObj = {0}; tstrncpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); @@ -265,6 +414,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre } memcpy(stbObj.pSchema, pCreate->pSchema, totalSize); + int32_t code = 0; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); @@ -272,29 +422,30 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre } mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); - SSdbRaw *pRedoRaw = mndStbActionEncode(&stbObj); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto CREATE_STB_OVER; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); - SSdbRaw *pUndoRaw = mndStbActionEncode(&stbObj); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, &stbObj) != 0) { + mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); + goto CREATE_STB_OVER; } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); - SSdbRaw *pCommitRaw = mndStbActionEncode(&stbObj); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto CREATE_STB_OVER; + } + + if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_STB_OVER; + } + + if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, &stbObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_STB_OVER; } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -302,8 +453,11 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre return -1; } + code = 0; + +CREATE_STB_OVER: mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) { @@ -312,7 +466,7 @@ static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) { mDebug("stb:%s, start to create", pCreate->name); - if (mndCheckStbMsg(pCreate) != 0) { + if (mndCheckCreateStbMsg(pCreate) != 0) { mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } @@ -349,7 +503,10 @@ static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg) { + mndTransHandleActionRsp(pMsg); + return 0; +} static int32_t mndCheckAlterStbMsg(SAlterStbMsg *pAlter) { SSchema *pSchema = &pAlter->schema; @@ -410,9 +567,44 @@ static int32_t mndProcessAlterStbMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessAlterStbInRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessAlterStbInRsp(SMnodeMsg *pMsg) { + mndTransHandleActionRsp(pMsg); + return 0; +} + +static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { + SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropStbUndoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { + SSdbRaw *pUndoRaw = mndStbActionEncode(pStb); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { + SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; } + +static int32_t mndSetDropStbUndoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; } static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pStb) { + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { mError("stb:%s, failed to drop since %s", pStb->name, terrstr()); @@ -420,36 +612,39 @@ static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pStb) { } mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); - SSdbRaw *pRedoRaw = mndStbActionEncode(pStb); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_STB_OVER; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); - SSdbRaw *pUndoRaw = mndStbActionEncode(pStb); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropStbUndoLogs(pMnode, pTrans, pStb) != 0) { + mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); + goto DROP_STB_OVER; } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); - SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_STB_OVER; + } + + if (mndSetDropStbRedoActions(pMnode, pTrans, pStb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_STB_OVER; + } + + if (mndSetDropStbUndoActions(pMnode, pTrans, pStb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_STB_OVER; } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + goto DROP_STB_OVER; } + code = 0; + +DROP_STB_OVER: mndTransDrop(pTrans); return 0; } @@ -484,7 +679,10 @@ static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) { + mndTransHandleActionRsp(pMsg); + return 0; +} static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 5caec6c78dc59bd061e96770c7817510c43a0f6f..54cd6ab501a0a9294b0f35e575c365d44e14b7bd 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -17,17 +17,10 @@ #include "mndTrans.h" #include "mndSync.h" -#define TSDB_TRANS_VER 1 +#define TSDB_TRANS_VER_NUMBER 1 #define TSDB_TRN_ARRAY_SIZE 8 #define TSDB_TRN_RESERVE_SIZE 64 -typedef struct { - SEpSet epSet; - int8_t msgType; - int32_t contLen; - void *pCont; -} STransAction; - static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); @@ -37,11 +30,11 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle); static void mndTransSendRpcRsp(STrans *pTrans, int32_t code); static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); -static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); +static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction); static void mndTransDropLogs(SArray *pArray); static void mndTransDropActions(SArray *pArray); static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray); -static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray); +static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray); static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans); @@ -100,7 +93,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { rawDataLen += (sizeof(STransAction) + pAction->contLen); } - SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER, rawDataLen); + SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER_NUMBER, rawDataLen); if (pRaw == NULL) { mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); return NULL; @@ -167,7 +160,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { return NULL; } - if (sver != TSDB_TRANS_VER) { + if (sver != TSDB_TRANS_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to get check soft ver from raw:%p since %s", pRaw, terrstr()); return NULL; @@ -324,8 +317,12 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOldTrans, STrans *pNewT } STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_TRANS, &transId); + SSdb *pSdb = pMnode->pSdb; + STrans *pTrans = sdbAcquire(pSdb, SDB_TRANS, &transId); + if (pTrans == NULL) { + terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; + } + return pTrans; } void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { @@ -343,10 +340,8 @@ char *mndTransStageStr(ETrnStage stage) { return "commit"; case TRN_STAGE_ROLLBACK: return "rollback"; - case TRN_STAGE_RETRY: - return "retry"; case TRN_STAGE_OVER: - return "stop"; + return "over"; default: return "undefined"; } @@ -388,7 +383,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { return NULL; } - mDebug("trans:%d, data:%p is created", pTrans->id, pTrans); + mDebug("trans:%d, is created", pTrans->id); return pTrans; } @@ -417,7 +412,7 @@ void mndTransDrop(STrans *pTrans) { mndTransDropActions(pTrans->redoActions); mndTransDropActions(pTrans->undoActions); - mDebug("trans:%d, data:%p is dropped", pTrans->id, pTrans); + // mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); tfree(pTrans); } @@ -459,10 +454,8 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) { return code; } -static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { - STransAction action = {.epSet = *pEpSet, .msgType = msgType, .contLen = contLen, .pCont = pCont}; - - void *ptr = taosArrayPush(pArray, &action); +static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction) { + void *ptr = taosArrayPush(pArray, pAction); if (ptr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -471,15 +464,15 @@ static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgTy return 0; } -int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { - int32_t code = mndTransAppendAction(pTrans->redoActions, pEpSet, msgType, contLen, pCont); - mTrace("trans:%d, msg:%s len:%d append to redo actions", pTrans->id, taosMsg[msgType], contLen); +int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction) { + int32_t code = mndTransAppendAction(pTrans->redoActions, pAction); + mTrace("trans:%d, msg:%s append to redo actions, code:0x%x", pTrans->id, taosMsg[pAction->msgType], code); return code; } -int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { - int32_t code = mndTransAppendAction(pTrans->undoActions, pEpSet, msgType, contLen, pCont); - mTrace("trans:%d, msg:%s len:%d append to undo actions", pTrans->id, taosMsg[msgType], contLen); +int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction) { + int32_t code = mndTransAppendAction(pTrans->undoActions, pAction); + mTrace("trans:%d, msg:%s append to undo actions, code:0x%x", pTrans->id, taosMsg[pAction->msgType], code); return code; } @@ -493,7 +486,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { } sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mTrace("trans:%d, start sync", pTrans->id); + mTrace("trans:%d, sync to other nodes", pTrans->id); int32_t code = mndSyncPropose(pMnode, pRaw); if (code != 0) { mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); @@ -515,7 +508,6 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { return -1; } - mDebug("trans:%d, prepare finished", pNewTrans->id); pNewTrans->rpcHandle = pTrans->rpcHandle; mndTransExecute(pMnode, pNewTrans); mndReleaseTrans(pMnode, pNewTrans); @@ -533,7 +525,7 @@ int32_t mndTransCommit(SMnode *pMnode, STrans *pTrans) { sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); if (taosArrayGetSize(pTrans->commitLogs) != 0) { - mTrace("trans:%d, start sync", pTrans->id); + mTrace("trans:%d, sync to other nodes", pTrans->id); int32_t code = mndSyncPropose(pMnode, pRaw); if (code != 0) { mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); @@ -563,7 +555,7 @@ int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) { } sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); - mTrace("trans:%d, start sync", pTrans->id); + mTrace("trans:%d, sync to other nodes", pTrans->id); int32_t code = mndSyncPropose(pMnode, pRaw); if (code != 0) { mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); @@ -596,6 +588,50 @@ void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code) // todo } +void mndTransHandleActionRsp(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + int64_t sig = (int64_t)(pMsg->rpcMsg.ahandle); + int32_t transId = (int32_t)(sig >> 32); + int32_t action = (int32_t)((sig << 32) >> 32); + + STrans *pTrans = mndAcquireTrans(pMnode, transId); + if (pTrans == NULL) { + mError("trans:%d, failed to get transId from vnode rsp since %s", transId, terrstr()); + goto HANDLE_ACTION_RSP_OVER; + } + + SArray *pArray = NULL; + if (pTrans->stage == TRN_STAGE_EXECUTE) { + pArray = pTrans->redoActions; + } else if (pTrans->stage == TRN_STAGE_ROLLBACK) { + pArray = pTrans->undoActions; + } else { + } + + if (pArray == NULL) { + mError("trans:%d, invalid trans stage:%s", transId, mndTransStageStr(pTrans->stage)); + goto HANDLE_ACTION_RSP_OVER; + } + + int32_t actionNum = taosArrayGetSize(pTrans->redoActions); + if (action < 0 || action > actionNum) { + mError("trans:%d, invalid action:%d", transId, action); + goto HANDLE_ACTION_RSP_OVER; + } + + STransAction *pAction = taosArrayGet(pArray, action); + if (pAction != NULL) { + pAction->msgReceived = 1; + pAction->errCode = pMsg->code; + } + + mDebug("trans:%d, action:%d response is received, code:0x%x", transId, action, pMsg->code); + mndTransExecute(pMnode, pTrans); + +HANDLE_ACTION_RSP_OVER: + mndReleaseTrans(pMnode, pTrans); +} + static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) { SSdb *pSdb = pMnode->pSdb; int32_t arraySize = taosArrayGetSize(pArray); @@ -618,7 +654,7 @@ static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) { if (code != 0) { mError("trans:%d, failed to execute redo logs since %s", pTrans->id, terrstr()) } else { - mTrace("trans:%d, execute redo logs finished", pTrans->id) + mDebug("trans:%d, execute redo logs finished", pTrans->id) } } @@ -632,7 +668,7 @@ static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) { if (code != 0) { mError("trans:%d, failed to execute undo logs since %s", pTrans->id, terrstr()) } else { - mTrace("trans:%d, execute undo logs finished", pTrans->id) + mDebug("trans:%d, execute undo logs finished", pTrans->id) } } @@ -646,47 +682,70 @@ static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) { if (code != 0) { mError("trans:%d, failed to execute commit logs since %s", pTrans->id, terrstr()) } else { - mTrace("trans:%d, execute commit logs finished", pTrans->id) + mDebug("trans:%d, execute commit logs finished", pTrans->id) } } return code; } -static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray) { -#if 0 - int32_t arraySize = taosArrayGetSize(pArray); - for (int32_t i = 0; i < arraySize; ++i) { - STransAction *pAction = taosArrayGet(pArray, i); +static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { + int32_t numOfActions = taosArrayGetSize(pArray); + if (numOfActions == 0) return 0; + + for (int32_t action = 0; action < numOfActions; ++action) { + STransAction *pAction = taosArrayGet(pArray, action); + if (pAction == NULL) continue; + if (pAction->msgSent) continue; - SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen}; + int64_t signature = pTrans->id; + signature = (signature << 32); + signature += action; + + SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .ahandle = (void *)signature}; rpcMsg.pCont = rpcMallocCont(pAction->contLen); if (rpcMsg.pCont == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); + + pAction->msgSent = 1; + pAction->msgReceived = 0; + pAction->errCode = 0; + + mDebug("trans:%d, action:%d is sent", pTrans->id, action); mndSendMsgToDnode(pMnode, &pAction->epSet, &rpcMsg); } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; -#else - return 0; -#endif + int32_t numOfReceivedMsgs = 0; + int32_t errorCode = 0; + for (int32_t action = 0; action < numOfActions; ++action) { + STransAction *pAction = taosArrayGet(pArray, action); + if (pAction == NULL) continue; + if (pAction->msgSent && pAction->msgReceived) { + numOfReceivedMsgs++; + if (pAction->errCode != 0) { + errorCode = pAction->errCode; + } + } + } + + if (numOfReceivedMsgs == numOfActions) { + mDebug("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errorCode); + terrno = errorCode; + return errorCode; + } else { + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } } static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { - if (taosArrayGetSize(pTrans->redoActions) <= 0) return 0; - - mTrace("trans:%d, start to execute redo actions", pTrans->id); - return mndTransExecuteActions(pMnode, pTrans->redoActions); + return mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); } static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { - if (taosArrayGetSize(pTrans->undoActions) <= 0) return 0; - - mTrace("trans:%d, start to execute undo actions", pTrans->id); - return mndTransExecuteActions(pMnode, pTrans->undoActions); + return mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); } static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { @@ -694,7 +753,7 @@ static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { if (code == 0) { pTrans->stage = TRN_STAGE_EXECUTE; - mTrace("trans:%d, stage from prepare to execute", pTrans->id); + mDebug("trans:%d, stage from prepare to execute", pTrans->id); } else { pTrans->stage = TRN_STAGE_ROLLBACK; mError("trans:%d, stage from prepare to rollback since %s", pTrans->id, terrstr()); @@ -708,17 +767,17 @@ static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans) { if (code == 0) { pTrans->stage = TRN_STAGE_COMMIT; - mTrace("trans:%d, stage from execute to commit", pTrans->id); + mDebug("trans:%d, stage from execute to commit", pTrans->id); } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { - mTrace("trans:%d, stage keep on execute since %s", pTrans->id, terrstr(code)); + mDebug("trans:%d, stage keep on execute since %s", pTrans->id, tstrerror(code)); return code; } else { if (pTrans->policy == TRN_POLICY_ROLLBACK) { pTrans->stage = TRN_STAGE_ROLLBACK; mError("trans:%d, stage from execute to rollback since %s", pTrans->id, terrstr()); } else { - pTrans->stage = TRN_STAGE_RETRY; - mError("trans:%d, stage from execute to retry since %s", pTrans->id, terrstr()); + pTrans->stage = TRN_STAGE_EXECUTE; + mError("trans:%d, stage keep on execute since %s", pTrans->id, terrstr()); } } @@ -726,29 +785,16 @@ static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans) { } static int32_t mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteCommitLogs(pMnode, pTrans); - - if (code == 0) { - pTrans->stage = TRN_STAGE_OVER; - mTrace("trans:%d, commit stage finished", pTrans->id); - } else { - if (pTrans->policy == TRN_POLICY_ROLLBACK) { - pTrans->stage = TRN_STAGE_ROLLBACK; - mError("trans:%d, stage from commit to rollback since %s", pTrans->id, terrstr()); - } else { - pTrans->stage = TRN_STAGE_RETRY; - mError("trans:%d, stage from commit to retry since %s", pTrans->id, terrstr()); - } - } - - return code; + mndTransExecuteCommitLogs(pMnode, pTrans); + pTrans->stage = TRN_STAGE_OVER; + return 0; } static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteUndoActions(pMnode, pTrans); if (code == 0) { - mTrace("trans:%d, rollbacked", pTrans->id); + mDebug("trans:%d, rollbacked", pTrans->id); } else { pTrans->stage = TRN_STAGE_ROLLBACK; mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr()); @@ -757,20 +803,6 @@ static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { return code; } -static int32_t mndTransPerformRetryStage(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteRedoActions(pMnode, pTrans); - - if (code == 0) { - pTrans->stage = TRN_STAGE_COMMIT; - mTrace("trans:%d, stage from retry to commit", pTrans->id); - } else { - pTrans->stage = TRN_STAGE_RETRY; - mError("trans:%d, stage keep on retry since %s", pTrans->id, terrstr()); - } - - return code; -} - static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { int32_t code = 0; @@ -785,7 +817,7 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { case TRN_STAGE_COMMIT: code = mndTransCommit(pMnode, pTrans); if (code == 0) { - code = mndTransPerformCommitStage(pMnode, pTrans); + mndTransPerformCommitStage(pMnode, pTrans); } break; case TRN_STAGE_ROLLBACK: @@ -794,9 +826,6 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { code = mndTransRollback(pMnode, pTrans); } break; - case TRN_STAGE_RETRY: - code = mndTransPerformRetryStage(pMnode, pTrans); - break; default: mndTransSendRpcRsp(pTrans, 0); return; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 58d687a14865ec7fe104457d287115e4bc0737f0..c96e6a80db3ec6b19f1ea609632a8022c65dad79 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -20,7 +20,7 @@ #include "mndTrans.h" #include "tkey.h" -#define TSDB_USER_VER 1 +#define TSDB_USER_VER_NUMBER 1 #define TSDB_USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); @@ -94,7 +94,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { } static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER, sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER_NUMBER, sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -114,7 +114,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_USER_VER) { + if (sver != TSDB_USER_VER_NUMBER) { mError("failed to decode user since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -175,8 +175,12 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNe } SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_USER, userName); + SSdb *pSdb = pMnode->pSdb; + SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + } + return pUser; } void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index e99fea200b944639aca3fb5bcb33623033c7915d..c65436efcb844d3b9387fdc9467673205d03d4dc 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -21,7 +21,7 @@ #include "mndShow.h" #include "mndTrans.h" -#define TSDB_VGROUP_VER_NUM 1 +#define TSDB_VGROUP_VER_NUMBER 1 #define TSDB_VGROUP_RESERVE_SIZE 64 static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); @@ -70,7 +70,7 @@ int32_t mndInitVgroup(SMnode *pMnode) { void mndCleanupVgroup(SMnode *pMnode) {} SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUMBER, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -98,7 +98,7 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_VGROUP_VER_NUM) { + if (sver != TSDB_VGROUP_VER_NUMBER) { mError("failed to decode vgroup since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -142,14 +142,20 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNe mTrace("vgId:%d, perform update action", pOldVgroup->vgId); pOldVgroup->updateTime = pNewVgroup->updateTime; pOldVgroup->version = pNewVgroup->version; + pOldVgroup->hashBegin = pNewVgroup->hashBegin; + pOldVgroup->hashEnd = pNewVgroup->hashEnd; pOldVgroup->replica = pNewVgroup->replica; memcpy(pOldVgroup->vnodeGid, pNewVgroup->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); return 0; } SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_VGROUP, &vgId); + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = sdbAcquire(pSdb, SDB_VGROUP, &vgId); + if (pVgroup == NULL) { + terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST; + } + return pVgroup; } void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { @@ -158,16 +164,17 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { } SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { - SCreateVnodeMsg *pCreate = malloc(sizeof(SCreateVnodeMsg)); + SCreateVnodeMsg *pCreate = calloc(1, sizeof(SCreateVnodeMsg)); if (pCreate == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pCreate->dnodeId = htonl(pDnode->id); pCreate->vgId = htonl(pVgroup->vgId); + pCreate->dnodeId = htonl(pDnode->id); memcpy(pCreate->db, pDb->name, TSDB_FULL_DB_NAME_LEN); pCreate->dbUid = htobe64(pDb->uid); + pCreate->vgVersion = htonl(pVgroup->version); pCreate->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); pCreate->totalBlocks = htonl(pDb->cfg.totalBlocks); pCreate->daysPerFile = htonl(pDb->cfg.daysPerFile); @@ -193,7 +200,6 @@ SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbOb SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pVgidDnode == NULL) { free(pCreate); - terrno = TSDB_CODE_MND_APP_ERROR; return NULL; } @@ -217,7 +223,7 @@ SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbOb } SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { - SDropVnodeMsg *pDrop = malloc(sizeof(SDropVnodeMsg)); + SDropVnodeMsg *pDrop = calloc(1, sizeof(SDropVnodeMsg)); if (pDrop == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -269,7 +275,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { return -1; } - int32_t alloceVgroups = 0; + int32_t allocedVgroups = 0; int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); uint32_t hashMin = 0; uint32_t hashMax = UINT32_MAX; @@ -281,7 +287,6 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { pVgroup->createdTime = taosGetTimestampMs(); pVgroup->updateTime = pVgroups->createdTime; pVgroup->version = 1; - pVgroup->dbUid = pDb->uid; pVgroup->hashBegin = hashMin + hashInterval * v; if (v == pDb->cfg.numOfVgroups - 1) { pVgroup->hashEnd = hashMax; @@ -290,6 +295,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { } memcpy(pVgroup->dbName, pDb->name, TSDB_FULL_DB_NAME_LEN); + pVgroup->dbUid = pDb->uid; pVgroup->replica = pDb->cfg.replications; if (mndGetAvailableDnode(pMnode, pVgroup) != 0) { @@ -298,22 +304,46 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { return -1; } - alloceVgroups++; + allocedVgroups++; } *ppVgroups = pVgroups; return 0; } -static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { return 0; } +SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup) { + SEpSet epset = {0}; + + for (int32_t v = 0; v < pVgroup->replica; ++v) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) continue; + + if (pVgid->role == TAOS_SYNC_STATE_LEADER) { + epset.inUse = epset.numOfEps; + } + + epset.port[epset.numOfEps] = pDnode->port; + memcpy(&epset.fqdn[epset.numOfEps], pDnode->fqdn, TSDB_FQDN_LEN); + epset.numOfEps++; + mndReleaseDnode(pMnode, pDnode); + } + + return epset; +} + +static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { + mndTransHandleActionRsp(pMsg); + return 0; +} + static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) { - SSdb *pSdb = pMnode->pSdb; - + SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -329,7 +359,7 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; - if (strcmp(pVgroup->dbName, dbName) == 0) { + if (pVgroup->dbUid == pDb->uid) { replica = MAX(replica, pVgroup->replica); numOfVgroups++; } @@ -441,11 +471,25 @@ static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter) { } static int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { - if (dnodeId == 0) { - return 0; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfVnodes = 0; + void *pIter = NULL; + + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + for (int32_t v = 0; v < pVgroup->replica; ++v) { + if (pVgroup->vnodeGid[v].dnodeId == dnodeId) { + numOfVnodes++; + } + } + + sdbRelease(pSdb, pVgroup); } - return 0; + return numOfVnodes; } static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 24155a4e5329f73fc950e25ef23b942b55a11bf0..fb0b95dc4a696f9a7b2751f3735d73d059838c60 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -225,7 +225,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { } return 0; -} +} SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { mDebug("start to open mnode in %s", path); @@ -237,6 +237,9 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { return NULL; } + char timestr[24] = "1970-01-01 00:00:00.00"; + (void)taosParseTime(timestr, &pMnode->checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + pMnode->pSteps = taosArrayInit(24, sizeof(SMnodeStep)); if (pMnode->pSteps == NULL) { free(pMnode); diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index f63f7236fbb56b0edfc49b61b6a2b00736454a65..1c8b82fd92cd3ac8ef536a744533e077c4a79588 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -240,6 +240,8 @@ void sdbRelease(SSdb *pSdb, void *pObj) { } void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { + *ppObj = NULL; + SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return NULL; diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index e37559808e2b5242354be6187c07fc49a732a940..5a0020199f0652376de73ec5c243301b4ac2371f 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -27,12 +27,12 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { pRaw->sver = sver; pRaw->dataLen = dataLen; - mTrace("raw:%p, is created, len:%d", pRaw, dataLen); + // mTrace("raw:%p, is created, len:%d", pRaw, dataLen); return pRaw; } void sdbFreeRaw(SSdbRaw *pRaw) { - mTrace("raw:%p, is freed", pRaw); + // mTrace("raw:%p, is freed", pRaw); free(pRaw); } diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index 3623516624e8ea31c3eedd0bcdfa52cf20d16ef4..6972605afd9c1dd93b7fd0b3198c3837b5351f88 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -19,5 +19,5 @@ target_link_libraries( # test if(${BUILD_TEST}) - #add_subdirectory(test) + add_subdirectory(test) endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index 56e07aca10704f7d8cb79a92c2ad80e4b25fee02..605557d4eace479f8cad786c658b2092b3e63d23 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -71,6 +71,7 @@ struct SVnode { SWal* pWal; SVnodeSync* pSync; SVnodeFS* pFs; + tsem_t canCommit; }; int vnodeScheduleTask(SVnodeTask* task); diff --git a/source/dnode/vnode/impl/src/vnodeCommit.c b/source/dnode/vnode/impl/src/vnodeCommit.c index 7213e31cb46d8bd75f5227e4b88bbd088dd7f340..f5bf60a7e3d56b66a8bf3804fbc2e7515671a4f8 100644 --- a/source/dnode/vnode/impl/src/vnodeCommit.c +++ b/source/dnode/vnode/impl/src/vnodeCommit.c @@ -38,9 +38,10 @@ int vnodeCommit(void *arg) { metaCommit(pVnode->pMeta); tqCommit(pVnode->pTq); - tsdbCommit(pVnode->pTq); + tsdbCommit(pVnode->pTsdb); vnodeBufPoolRecycle(pVnode); + tsem_post(&(pVnode->canCommit)); // TODO return 0; } diff --git a/source/dnode/vnode/impl/src/vnodeInt.c b/source/dnode/vnode/impl/src/vnodeInt.c index 8a6fc8bf5eb7da2545441adf932d5b4740f0ccaa..5deaffe6d220134ac6f8c5f0cfd4607be2ef304a 100644 --- a/source/dnode/vnode/impl/src/vnodeInt.c +++ b/source/dnode/vnode/impl/src/vnodeInt.c @@ -15,53 +15,31 @@ #define _DEFAULT_SOURCE #include "vnodeInt.h" -#include "tqueue.h" int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg) { return 0; } -SVnode *vnodeCreate(int32_t vgId, const char *path, const SVnodeCfg *pCfg) { return NULL; } -void vnodeDrop(SVnode *pVnode) {} + int32_t vnodeCompact(SVnode *pVnode) { return 0; } + int32_t vnodeSync(SVnode *pVnode) { return 0; } int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { return 0; } -SVnodeMsg *vnodeInitMsg(int32_t msgNum) { - SVnodeMsg *pMsg = taosAllocateQitem(msgNum * sizeof(SRpcMsg *) + sizeof(SVnodeMsg)); - if (pMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } else { - pMsg->allocNum = msgNum; - return pMsg; - } +int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + vInfo("query message is processed"); + return 0; } -int32_t vnodeAppendMsg(SVnodeMsg *pMsg, SRpcMsg *pRpcMsg) { - if (pMsg->curNum >= pMsg->allocNum) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pMsg->rpcMsg[pMsg->curNum++] = *pRpcMsg; +int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + vInfo("fetch message is processed"); + return 0; } -void vnodeCleanupMsg(SVnodeMsg *pMsg) { - for (int32_t i = 0; i < pMsg->curNum; ++i) { - rpcFreeCont(pMsg->rpcMsg[i].pCont); - } - taosFreeQitem(pMsg); +int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + vInfo("sync message is processed"); + return 0; } -void vnodeProcessMsg(SVnode *pVnode, SVnodeMsg *pMsg, EVnMsgType msgType) { - switch (msgType) { - case VN_MSG_TYPE_WRITE: - break; - case VN_MSG_TYPE_APPLY: - break; - case VN_MSG_TYPE_SYNC: - break; - case VN_MSG_TYPE_QUERY: - break; - case VN_MSG_TYPE_FETCH: - break; - } -} \ No newline at end of file +int vnodeProcessConsumeReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + vInfo("consume message is processed"); + return 0; +} diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index 70d9c7d4b09d899d91cecb5eaf7fc3159c495a82..c98f3e0800eaa0726c29708864040aa5d1a379ca 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -74,11 +74,14 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { pVnode->path = strdup(path); vnodeOptionsCopy(&(pVnode->config), pVnodeCfg); + tsem_init(&(pVnode->canCommit), 0, 1); + return pVnode; } static void vnodeFree(SVnode *pVnode) { if (pVnode) { + tsem_destroy(&(pVnode->canCommit)); tfree(pVnode->path); free(pVnode); } diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/impl/src/vnodeWrite.c index fc977ce03f6150e58ec324949c2fc7263a8f26e2..5ec03f1fd3f7d7f4f2448d388146df311561dd35 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/impl/src/vnodeWrite.c @@ -27,14 +27,14 @@ int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { } int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { - SRpcMsg *pMsg; + SRpcMsg * pMsg; SVnodeReq *pVnodeReq; for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); // ser request version - void *pBuf = pMsg->pCont; + void * pBuf = pMsg->pCont; int64_t ver = pVnode->state.processed++; taosEncodeFixedU64(&pBuf, ver); @@ -45,68 +45,63 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { walFsync(pVnode->pWal, false); - // Apply each request now - for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { - pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); - SVnodeReq vReq; + // TODO: Integrate RAFT module here - // Apply the request - { - void *ptr = vnodeMalloc(pVnode, pMsg->contLen); - if (ptr == NULL) { - // TODO: handle error - } + return 0; +} + +int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { + SVnodeReq vReq; + void * ptr = vnodeMalloc(pVnode, pMsg->contLen); + if (ptr == NULL) { + // TODO: handle error + } - // TODO: copy here need to be extended - memcpy(ptr, pMsg->pCont, pMsg->contLen); + // TODO: copy here need to be extended + memcpy(ptr, pMsg->pCont, pMsg->contLen); + + // todo: change the interface here + uint64_t ver; + taosDecodeFixedU64(pMsg->pCont, &ver); + if (tqPushMsg(pVnode->pTq, ptr, ver) < 0) { + // TODO: handle error + } - // todo: change the interface here - uint64_t ver; - taosDecodeFixedU64(pMsg->pCont, &ver); - if (tqPushMsg(pVnode->pTq, ptr, ver) < 0) { + vnodeParseReq(pMsg->pCont, &vReq, pMsg->msgType); + + switch (pMsg->msgType) { + case TSDB_MSG_TYPE_CREATE_STB_IN: + case TSDB_MSG_TYPE_CREATE_TABLE: + if (metaCreateTable(pVnode->pMeta, &(vReq.ctReq)) < 0) { // TODO: handle error } - vnodeParseReq(pMsg->pCont, &vReq, pMsg->msgType); - - switch (pMsg->msgType) { - case TSDB_MSG_TYPE_CREATE_TABLE: - if (metaCreateTable(pVnode->pMeta, &(vReq.ctReq)) < 0) { - // TODO: handle error - } - - // TODO: maybe need to clear the requst struct - break; - case TSDB_MSG_TYPE_DROP_TABLE: - if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) { - // TODO: handle error - } - break; - case TSDB_MSG_TYPE_SUBMIT: - if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr) < 0) { - // TODO: handle error - } - break; - default: - break; + // TODO: maybe need to clear the requst struct + break; + case TSDB_MSG_TYPE_DROP_STB_IN: + case TSDB_MSG_TYPE_DROP_TABLE: + if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) { + // TODO: handle error } - - pVnode->state.applied = ver; - } - - // Check if it needs to commit - if (vnodeShouldCommit(pVnode)) { - if (vnodeAsyncCommit(pVnode) < 0) { + break; + case TSDB_MSG_TYPE_SUBMIT: + if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr) < 0) { // TODO: handle error } - } + break; + default: + break; } - return 0; -} + pVnode->state.applied = ver; -int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - // TODO + // Check if it needs to commit + if (vnodeShouldCommit(pVnode)) { + tsem_wait(&(pVnode->canCommit)); + if (vnodeAsyncCommit(pVnode) < 0) { + // TODO: handle error + } + } return 0; } diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index 6f2e6f721ab335e7219e08e762a46041ddce73d7..aa9fc741d95122f8c60a44967e4b996d3b98deb5 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -81,7 +81,8 @@ static void vtBuildCreateStbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { pSchema = vtCreateBasicSchema(); pTagSchema = vtCreateBasicTagSchema(); - SVnodeReq vCreateSTbReq = VNODE_INIT_CREATE_STB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); + SVnodeReq vCreateSTbReq; + vnodeSetCreateStbReq(&vCreateSTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); zs = vnodeBuildReq(NULL, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs); @@ -104,7 +105,8 @@ static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { int tz; SKVRow pTag = vtCreateBasicTag(); - SVnodeReq vCreateCTbReq = VNODE_INIT_CREATE_CTB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pTag); + SVnodeReq vCreateCTbReq; + vnodeSetCreateCtbReq(&vCreateCTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pTag); tz = vnodeBuildReq(NULL, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz); @@ -166,6 +168,21 @@ static void vtClearMsgBatch(SArray *pMsgArr) { taosArrayClear(pMsgArr); } +static void vtProcessAndApplyReqs(SVnode *pVnode, SArray *pMsgArr) { + int rcode; + SRpcMsg *pReq; + SRpcMsg *pRsp; + + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + GTEST_ASSERT_EQ(rcode, 0); + + for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) { + pReq = *(SRpcMsg **)taosArrayGet(pMsgArr, i); + rcode = vnodeApplyWMsg(pVnode, pReq, NULL); + GTEST_ASSERT_EQ(rcode, 0); + } +} + TEST(vnodeApiTest, vnode_simple_create_table_test) { tb_uid_t suid = 1638166374163; SRpcMsg *pMsg; @@ -189,8 +206,7 @@ TEST(vnodeApiTest, vnode_simple_create_table_test) { sprintf(tbname, "st"); vtBuildCreateStbReq(suid, tbname, &pMsg); taosArrayPush(pMsgArr, &pMsg); - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - ASSERT_EQ(rcode, 0); + vtProcessAndApplyReqs(pVnode, pMsgArr); vtClearMsgBatch(pMsgArr); // CREATE A LOT OF CHILD TABLES @@ -203,8 +219,7 @@ TEST(vnodeApiTest, vnode_simple_create_table_test) { } // Process request batch - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - ASSERT_EQ(rcode, 0); + vtProcessAndApplyReqs(pVnode, pMsgArr); // Clear request batch vtClearMsgBatch(pMsgArr); @@ -242,16 +257,14 @@ TEST(vnodeApiTest, vnode_simple_insert_test) { sprintf(tbname, "st"); vtBuildCreateStbReq(suid, tbname, &pMsg); taosArrayPush(pMsgArr, &pMsg); - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - GTEST_ASSERT_EQ(rcode, 0); + vtProcessAndApplyReqs(pVnode, pMsgArr); vtClearMsgBatch(pMsgArr); // 2. CREATE A CHILD TABLE sprintf(tbname, "t0"); vtBuildCreateCtbReq(suid, tbname, &pMsg); taosArrayPush(pMsgArr, &pMsg); - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - GTEST_ASSERT_EQ(rcode, 0); + vtProcessAndApplyReqs(pVnode, pMsgArr); vtClearMsgBatch(pMsgArr); // 3. WRITE A LOT OF TIME-SERIES DATA @@ -260,8 +273,7 @@ TEST(vnodeApiTest, vnode_simple_insert_test) { vtBuildSubmitReq(&pMsg); taosArrayPush(pMsgArr, &pMsg); } - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - GTEST_ASSERT_EQ(rcode, 0); + vtProcessAndApplyReqs(pVnode, pMsgArr); vtClearMsgBatch(pMsgArr); } diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index 9e5aaf61fcd4e6961897e94013f0dfde499b7a8a..4529e868e104e7b27937ed2553da2fed64984e23 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -14,9 +14,7 @@ */ #include "tqInt.h" -#include "osSocket.h" #include "tqMetaStore.h" -#include "osAtomic.h" // static // read next version data @@ -61,15 +59,19 @@ STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogHandle* tqLogHandle, SMemA pTq->path = strdup(path); pTq->tqConfig = tqConfig; pTq->tqLogHandle = tqLogHandle; +#if 0 pTq->tqMemRef.pAllocatorFactory = allocFac; pTq->tqMemRef.pAllocator = allocFac->create(allocFac); if (pTq->tqMemRef.pAllocator == NULL) { // TODO: error code of buffer pool } +#endif pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeGroup, (FTqDeserialize)tqDeserializeGroup, free, 0); if (pTq->tqMeta == NULL) { free(pTq); +#if 0 allocFac->destroy(allocFac, pTq->tqMemRef.pAllocator); +#endi return NULL; } @@ -144,6 +146,7 @@ int tqCreateGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroup // TODO return -1; } + *ppGroup = pGroup; memset(pGroup, 0, sizeof(STqGroup)); pGroup->topicList = tdListNew(sizeof(STqTopic)); diff --git a/source/dnode/vnode/tsdb/CMakeLists.txt b/source/dnode/vnode/tsdb/CMakeLists.txt index 870d75cd20ed3979cbb02a2aa2e96de131e64bf8..30e9d70f125c42990665948614ff570d1badc16b 100644 --- a/source/dnode/vnode/tsdb/CMakeLists.txt +++ b/source/dnode/vnode/tsdb/CMakeLists.txt @@ -1,10 +1,24 @@ aux_source_directory(src TSDB_SRC) -add_library(tsdb ${TSDB_SRC}) +if(0) + add_library(tsdb ${TSDB_SRC}) +else(0) + add_library(tsdb "") + target_sources(tsdb + PRIVATE + "src/tsdbCommit.c" + "src/tsdbMain.c" + "src/tsdbMemTable.c" + "src/tsdbOptions.c" + "src/tsdbWrite.c" + ) +endif(0) + target_include_directories( tsdb PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/tsdb" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + target_link_libraries( tsdb PUBLIC os diff --git a/src/tsdb/inc/tsdbCommit.h b/source/dnode/vnode/tsdb/inc/tsdbCommit.h similarity index 97% rename from src/tsdb/inc/tsdbCommit.h rename to source/dnode/vnode/tsdb/inc/tsdbCommit.h index cde728b1705cd1eead065772978631fb4b36246d..82ba1c9dffc98323d369fc6d8704ada0eb4210c7 100644 --- a/src/tsdb/inc/tsdbCommit.h +++ b/source/dnode/vnode/tsdb/inc/tsdbCommit.h @@ -16,6 +16,7 @@ #ifndef _TD_TSDB_COMMIT_H_ #define _TD_TSDB_COMMIT_H_ +#if 0 typedef struct { int minFid; int midFid; @@ -53,5 +54,6 @@ static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) { return -1; } } +#endif #endif /* _TD_TSDB_COMMIT_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbCompact.h b/source/dnode/vnode/tsdb/inc/tsdbCompact.h similarity index 94% rename from src/tsdb/inc/tsdbCompact.h rename to source/dnode/vnode/tsdb/inc/tsdbCompact.h index 5a382de5e0cc438dbd444d9a0895fd5f55094476..441dfda6ade603c2dd1240962543307ca3bb83f9 100644 --- a/src/tsdb/inc/tsdbCompact.h +++ b/source/dnode/vnode/tsdb/inc/tsdbCompact.h @@ -19,8 +19,12 @@ extern "C" { #endif +#if 0 + void *tsdbCompactImpl(STsdbRepo *pRepo); +#endif + #ifdef __cplusplus } #endif diff --git a/src/tsdb/inc/tsdbFS.h b/source/dnode/vnode/tsdb/inc/tsdbFS.h similarity index 99% rename from src/tsdb/inc/tsdbFS.h rename to source/dnode/vnode/tsdb/inc/tsdbFS.h index e89e10f7667e8aa5388ebfa4d2c5b54f1bf3e57f..03207567835c85cabd6ff8a97f3288d5481ff080 100644 --- a/src/tsdb/inc/tsdbFS.h +++ b/source/dnode/vnode/tsdb/inc/tsdbFS.h @@ -16,6 +16,8 @@ #ifndef _TD_TSDB_FS_H_ #define _TD_TSDB_FS_H_ +#if 0 + #define TSDB_FS_VERSION 0 // ================== TSDB global config @@ -113,4 +115,6 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS* pFs) { return 0; } +#endif + #endif /* _TD_TSDB_FS_H_ */ diff --git a/src/tsdb/inc/tsdbFile.h b/source/dnode/vnode/tsdb/inc/tsdbFile.h similarity index 99% rename from src/tsdb/inc/tsdbFile.h rename to source/dnode/vnode/tsdb/inc/tsdbFile.h index b9d5431de6bc3864a4a13ea30356033de76da178..73a7de02499bb316b27728de85b754cdc108c253 100644 --- a/src/tsdb/inc/tsdbFile.h +++ b/source/dnode/vnode/tsdb/inc/tsdbFile.h @@ -16,6 +16,8 @@ #ifndef _TS_TSDB_FILE_H_ #define _TS_TSDB_FILE_H_ +#if 0 + #define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_DELIMITER 0xF00AFA0F #define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF @@ -364,4 +366,5 @@ static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) { return true; } +#endif #endif /* _TS_TSDB_FILE_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbHealth.h b/source/dnode/vnode/tsdb/inc/tsdbHealth.h similarity index 98% rename from src/tsdb/inc/tsdbHealth.h rename to source/dnode/vnode/tsdb/inc/tsdbHealth.h index 324f4312e05fc0ca0200c319728bf692bf476bf6..d7b70ac053749863241fdd946842c7face2ed961 100644 --- a/src/tsdb/inc/tsdbHealth.h +++ b/source/dnode/vnode/tsdb/inc/tsdbHealth.h @@ -16,10 +16,14 @@ #ifndef _TD_TSDB_HEALTH_H_ #define _TD_TSDB_HEALTH_H_ +#if 0 + bool tsdbUrgeQueryFree(STsdbRepo* pRepo); int32_t tsdbInsertNewBlock(STsdbRepo* pRepo); bool tsdbIdleMemEnough(); bool tsdbAllowNewBlock(STsdbRepo* pRepo); +#endif + #endif /* _TD_TSDB_BUFFER_H_ */ diff --git a/src/tsdb/inc/tsdbReadImpl.h b/source/dnode/vnode/tsdb/inc/tsdbReadImpl.h similarity index 99% rename from src/tsdb/inc/tsdbReadImpl.h rename to source/dnode/vnode/tsdb/inc/tsdbReadImpl.h index 814c4d130599768e8237145559c47e50e64db4db..a9bd76c2b162461fed199842533016b93a6a4f04 100644 --- a/src/tsdb/inc/tsdbReadImpl.h +++ b/source/dnode/vnode/tsdb/inc/tsdbReadImpl.h @@ -15,6 +15,7 @@ #ifndef _TD_TSDB_READ_IMPL_H_ #define _TD_TSDB_READ_IMPL_H_ +#if 0 #include "tfs.h" #include "tsdb.h" @@ -150,4 +151,6 @@ static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) { return 0; } +#endif + #endif /*_TD_TSDB_READ_IMPL_H_*/ diff --git a/src/tsdb/inc/tsdbRowMergeBuf.h b/source/dnode/vnode/tsdb/inc/tsdbRowMergeBuf.h similarity index 98% rename from src/tsdb/inc/tsdbRowMergeBuf.h rename to source/dnode/vnode/tsdb/inc/tsdbRowMergeBuf.h index cefa9b27fbd2a6116e2d7180f102440af9595e93..1531d532b9a2ba410747e833a8881289b5dd1370 100644 --- a/src/tsdb/inc/tsdbRowMergeBuf.h +++ b/source/dnode/vnode/tsdb/inc/tsdbRowMergeBuf.h @@ -16,6 +16,8 @@ #ifndef TSDB_ROW_MERGE_BUF_H #define TSDB_ROW_MERGE_BUF_H +#if 0 + #ifdef __cplusplus extern "C" { #endif @@ -42,4 +44,6 @@ static FORCE_INLINE void tsdbFreeMergeBuf(SMergeBuf buf) { } #endif +#endif + #endif /* ifndef TSDB_ROW_MERGE_BUF_H */ diff --git a/src/tsdb/inc/tsdbint.h b/source/dnode/vnode/tsdb/inc/tsdbint.h similarity index 99% rename from src/tsdb/inc/tsdbint.h rename to source/dnode/vnode/tsdb/inc/tsdbint.h index 2354269c6ee861632ff0d88c2efe52470a689a48..bdd1bd6f71f25e6c566ea3f7992d8b9197689855 100644 --- a/src/tsdb/inc/tsdbint.h +++ b/source/dnode/vnode/tsdb/inc/tsdbint.h @@ -16,6 +16,7 @@ #ifndef _TD_TSDB_INT_H_ #define _TD_TSDB_INT_H_ +#if 0 // // TODO: remove the include // #include // #include @@ -144,4 +145,5 @@ static FORCE_INLINE int tsdbGetNextMaxTables(int tid) { } #endif +#endif #endif /* _TD_TSDB_INT_H_ */ diff --git a/source/dnode/vnode/tsdb/src/tsdbCommit.c b/source/dnode/vnode/tsdb/src/tsdbCommit.c index a747c7333eabaabc17160943a7288b62927d34a8..1247dcd728117ee834ea28b951b5ebf7d4e29940 100644 --- a/source/dnode/vnode/tsdb/src/tsdbCommit.c +++ b/source/dnode/vnode/tsdb/src/tsdbCommit.c @@ -30,4 +30,1673 @@ int tsdbCommit(STsdb *pTsdb) { pTsdb->imem = NULL; // tsem_post(&(pTsdb->canCommit)); return 0; -} \ No newline at end of file +} + +#if 0 +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "tsdbint.h" + +extern int32_t tsTsdbMetaCompactRatio; + +#define TSDB_MAX_SUBBLOCKS 8 +static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { + if (key < 0) { + return (int)((key + 1) / tsTickPerDay[precision] / days - 1); + } else { + return (int)((key / tsTickPerDay[precision] / days)); + } +} + +typedef struct { + SRtn rtn; // retention snapshot + SFSIter fsIter; // tsdb file iterator + int niters; // memory iterators + SCommitIter *iters; + bool isRFileSet; // read and commit FSET + SReadH readh; + SDFileSet wSet; + bool isDFileSame; + bool isLFileSame; + TSKEY minKey; + TSKEY maxKey; + SArray * aBlkIdx; // SBlockIdx array + STable * pTable; + SArray * aSupBlk; // Table super-block array + SArray * aSubBlk; // table sub-block array + SDataCols * pDataCols; +} SCommitH; + +#define TSDB_COMMIT_REPO(ch) TSDB_READ_REPO(&(ch->readh)) +#define TSDB_COMMIT_REPO_ID(ch) REPO_ID(TSDB_READ_REPO(&(ch->readh))) +#define TSDB_COMMIT_WRITE_FSET(ch) (&((ch)->wSet)) +#define TSDB_COMMIT_TABLE(ch) ((ch)->pTable) +#define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD) +#define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA) +#define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST) +#define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh)) +#define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh)) +#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock) +#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) + +static int tsdbCommitMeta(STsdbRepo *pRepo); +static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact); +static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); +static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile); +static int tsdbCommitTSData(STsdbRepo *pRepo); +static void tsdbStartCommit(STsdbRepo *pRepo); +static void tsdbEndCommit(STsdbRepo *pRepo, int eno); +static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); +static int tsdbCreateCommitIters(SCommitH *pCommith); +static void tsdbDestroyCommitIters(SCommitH *pCommith); +static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key); +static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo); +static void tsdbDestroyCommitH(SCommitH *pCommith); +static int tsdbGetFidLevel(int fid, SRtn *pRtn); +static int tsdbNextCommitFid(SCommitH *pCommith); +static int tsdbCommitToTable(SCommitH *pCommith, int tid); +static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable); +static int tsdbComparKeyBlock(const void *arg1, const void *arg2); +static int tsdbWriteBlockInfo(SCommitH *pCommih); +static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData); +static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx); +static int tsdbMoveBlock(SCommitH *pCommith, int bidx); +static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks); +static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, + bool isLastOneBlock); +static void tsdbResetCommitFile(SCommitH *pCommith); +static void tsdbResetCommitTable(SCommitH *pCommith); +static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); +static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError); +static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo); +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, + TSKEY maxKey, int maxRows, int8_t update); + +void *tsdbCommitData(STsdbRepo *pRepo) { + if (pRepo->imem == NULL) { + return NULL; + } + tsdbStartCommit(pRepo); + + // Commit to update meta file + if (tsdbCommitMeta(pRepo) < 0) { + tsdbError("vgId:%d error occurs while committing META data since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + // Create the iterator to read from cache + if (tsdbCommitTSData(pRepo) < 0) { + tsdbError("vgId:%d error occurs while committing TS data since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS); + return NULL; + +_err: + ASSERT(terrno != TSDB_CODE_SUCCESS); + pRepo->code = terrno; + + tsdbEndCommit(pRepo, terrno); + return NULL; +} + +int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) { + SDiskID did; + SDFileSet nSet; + STsdbFS * pfs = REPO_FS(pRepo); + int level; + + ASSERT(pSet->fid >= pRtn->minFid); + + level = tsdbGetFidLevel(pSet->fid, pRtn); + + tfsAllocDisk(level, &(did.level), &(did.id)); + if (did.level == TFS_UNDECIDED_LEVEL) { + terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + return -1; + } + + if (did.level > TSDB_FSET_LEVEL(pSet)) { + // Need to move the FSET to higher level + tsdbInitDFileSet(&nSet, did, REPO_ID(pRepo), pSet->fid, FS_TXN_VERSION(pfs)); + + if (tsdbCopyDFileSet(pSet, &nSet) < 0) { + tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno)); + return -1; + } + + if (tsdbUpdateDFileSet(pfs, &nSet) < 0) { + return -1; + } + + tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id); + } else { + // On a correct level + if (tsdbUpdateDFileSet(pfs, pSet) < 0) { + return -1; + } + } + + return 0; +} + +int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, + SBlockIdx *pIdx) { + size_t nSupBlocks; + size_t nSubBlocks; + uint32_t tlen; + SBlockInfo *pBlkInfo; + int64_t offset; + SBlock * pBlock; + + memset(pIdx, 0, sizeof(*pIdx)); + + nSupBlocks = taosArrayGetSize(pSupA); + nSubBlocks = (pSubA == NULL) ? 0 : taosArrayGetSize(pSubA); + + if (nSupBlocks <= 0) { + // No data (data all deleted) + return 0; + } + + tlen = (uint32_t)(sizeof(SBlockInfo) + sizeof(SBlock) * (nSupBlocks + nSubBlocks) + sizeof(TSCKSUM)); + if (tsdbMakeRoom(ppBuf, tlen) < 0) return -1; + pBlkInfo = *ppBuf; + + pBlkInfo->delimiter = TSDB_FILE_DELIMITER; + pBlkInfo->tid = TABLE_TID(pTable); + pBlkInfo->uid = TABLE_UID(pTable); + + memcpy((void *)(pBlkInfo->blocks), taosArrayGet(pSupA, 0), nSupBlocks * sizeof(SBlock)); + if (nSubBlocks > 0) { + memcpy((void *)(pBlkInfo->blocks + nSupBlocks), taosArrayGet(pSubA, 0), nSubBlocks * sizeof(SBlock)); + + for (int i = 0; i < nSupBlocks; i++) { + pBlock = pBlkInfo->blocks + i; + + if (pBlock->numOfSubBlocks > 1) { + pBlock->offset += (sizeof(SBlockInfo) + sizeof(SBlock) * nSupBlocks); + } + } + } + + taosCalcChecksumAppend(0, (uint8_t *)pBlkInfo, tlen); + + if (tsdbAppendDFile(pHeadf, (void *)pBlkInfo, tlen, &offset) < 0) { + return -1; + } + + tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(pBlkInfo, tlen - sizeof(TSCKSUM))); + + // Set pIdx + pBlock = taosArrayGetLast(pSupA); + + pIdx->tid = TABLE_TID(pTable); + pIdx->uid = TABLE_UID(pTable); + pIdx->hasLast = pBlock->last ? 1 : 0; + pIdx->maxKey = pBlock->keyLast; + pIdx->numOfBlocks = (uint32_t)nSupBlocks; + pIdx->len = tlen; + pIdx->offset = (uint32_t)offset; + + return 0; +} + +int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { + SBlockIdx *pBlkIdx; + size_t nidx = taosArrayGetSize(pIdxA); + int tlen = 0, size; + int64_t offset; + + if (nidx <= 0) { + // All data are deleted + pHeadf->info.offset = 0; + pHeadf->info.len = 0; + return 0; + } + + for (size_t i = 0; i < nidx; i++) { + pBlkIdx = (SBlockIdx *)taosArrayGet(pIdxA, i); + + size = tsdbEncodeSBlockIdx(NULL, pBlkIdx); + if (tsdbMakeRoom(ppBuf, tlen + size) < 0) return -1; + + void *ptr = POINTER_SHIFT(*ppBuf, tlen); + tsdbEncodeSBlockIdx(&ptr, pBlkIdx); + + tlen += size; + } + + tlen += sizeof(TSCKSUM); + if (tsdbMakeRoom(ppBuf, tlen) < 0) return -1; + taosCalcChecksumAppend(0, (uint8_t *)(*ppBuf), tlen); + + if (tsdbAppendDFile(pHeadf, *ppBuf, tlen, &offset) < tlen) { + return -1; + } + + tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(*ppBuf, tlen - sizeof(TSCKSUM))); + pHeadf->info.offset = (uint32_t)offset; + pHeadf->info.len = tlen; + + return 0; +} + + +// =================== Commit Meta Data +static int tsdbInitCommitMetaFile(STsdbRepo *pRepo, SMFile* pMf, bool open) { + STsdbFS * pfs = REPO_FS(pRepo); + SMFile * pOMFile = pfs->cstatus->pmf; + SDiskID did; + + // Create/Open a meta file or open the existing file + if (pOMFile == NULL) { + // Create a new meta file + did.level = TFS_PRIMARY_LEVEL; + did.id = TFS_PRIMARY_ID; + tsdbInitMFile(pMf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); + + if (open && tsdbCreateMFile(pMf, true) < 0) { + tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMf)); + } else { + tsdbInitMFileEx(pMf, pOMFile); + if (open && tsdbOpenMFile(pMf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } + + return 0; +} + +static int tsdbCommitMeta(STsdbRepo *pRepo) { + STsdbFS * pfs = REPO_FS(pRepo); + SMemTable *pMem = pRepo->imem; + SMFile * pOMFile = pfs->cstatus->pmf; + SMFile mf; + SActObj * pAct = NULL; + SActCont * pCont = NULL; + SListNode *pNode = NULL; + + ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0); + + if (listNEles(pMem->actList) <= 0) { + // no meta data to commit, just keep the old meta file + tsdbUpdateMFile(pfs, pOMFile); + if (tsTsdbMetaCompactRatio > 0) { + if (tsdbInitCommitMetaFile(pRepo, &mf, false) < 0) { + return -1; + } + int ret = tsdbCompactMetaFile(pRepo, pfs, &mf); + if (ret < 0) tsdbError("compact meta file error"); + + return ret; + } + return 0; + } else { + if (tsdbInitCommitMetaFile(pRepo, &mf, true) < 0) { + return -1; + } + } + + // Loop to write + while ((pNode = tdListPopHead(pMem->actList)) != NULL) { + pAct = (SActObj *)pNode->data; + if (pAct->act == TSDB_UPDATE_META) { + pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); + if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len, false) < 0) { + tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, + tstrerror(terrno)); + tsdbCloseMFile(&mf); + (void)tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache + return -1; + } + } else if (pAct->act == TSDB_DROP_META) { + if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) { + tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, + tstrerror(terrno)); + tsdbCloseMFile(&mf); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache + return -1; + } + } else { + ASSERT(false); + } + } + + if (tsdbUpdateMFileHeader(&mf) < 0) { + tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno)); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache + return -1; + } + + TSDB_FILE_FSYNC(&mf); + tsdbCloseMFile(&mf); + tsdbUpdateMFile(pfs, &mf); + + if (tsTsdbMetaCompactRatio > 0 && tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) { + tsdbError("compact meta file error"); + } + + return 0; +} + +int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord) { + int tlen = 0; + tlen += taosEncodeFixedU64(buf, pRecord->uid); + tlen += taosEncodeFixedI64(buf, pRecord->offset); + tlen += taosEncodeFixedI64(buf, pRecord->size); + + return tlen; +} + +void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord) { + buf = taosDecodeFixedU64(buf, &(pRecord->uid)); + buf = taosDecodeFixedI64(buf, &(pRecord->offset)); + buf = taosDecodeFixedI64(buf, &(pRecord->size)); + + return buf; +} + +void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) { + STsdbCfg *pCfg = REPO_CFG(pRepo); + TSKEY minKey, midKey, maxKey, now; + + now = taosGetTimestamp(pCfg->precision); + minKey = now - pCfg->keep * tsTickPerDay[pCfg->precision]; + midKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision]; + maxKey = now - pCfg->keep1 * tsTickPerDay[pCfg->precision]; + + pRtn->minKey = minKey; + pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision)); + pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision)); + pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision)); + tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey, + pRtn->minFid, pRtn->midFid, pRtn->maxFid); +} + +static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact) { + char buf[64] = "\0"; + void * pBuf = buf; + SKVRecord rInfo; + int64_t offset; + + // Seek to end of meta file + offset = tsdbSeekMFile(pMFile, 0, SEEK_END); + if (offset < 0) { + return -1; + } + + rInfo.offset = offset; + rInfo.uid = uid; + rInfo.size = contLen; + + int tlen = tsdbEncodeKVRecord((void **)(&pBuf), &rInfo); + if (tsdbAppendMFile(pMFile, buf, tlen, NULL) < tlen) { + return -1; + } + + if (tsdbAppendMFile(pMFile, cont, contLen, NULL) < contLen) { + return -1; + } + + tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM))); + + SHashObj* cache = compact ? pfs->metaCacheComp : pfs->metaCache; + + pMFile->info.nRecords++; + + SKVRecord *pRecord = taosHashGet(cache, (void *)&uid, sizeof(uid)); + if (pRecord != NULL) { + pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord)); + } else { + pMFile->info.nRecords++; + } + taosHashPut(cache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); + + return 0; +} + +static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { + SKVRecord rInfo = {0}; + char buf[128] = "\0"; + + SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid)); + if (pRecord == NULL) { + tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid); + return -1; + } + + rInfo.offset = -pRecord->offset; + rInfo.uid = pRecord->uid; + rInfo.size = pRecord->size; + + void *pBuf = buf; + tsdbEncodeKVRecord(&pBuf, &rInfo); + + if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) { + return -1; + } + + pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord)); + pMFile->info.nDels++; + pMFile->info.nRecords--; + pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); + + taosHashRemove(pfs->metaCache, (void *)(&uid), sizeof(uid)); + return 0; +} + +static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { + float delPercent = (float)(pMFile->info.nDels) / (float)(pMFile->info.nRecords); + float tombPercent = (float)(pMFile->info.tombSize) / (float)(pMFile->info.size); + float compactRatio = (float)(tsTsdbMetaCompactRatio)/100; + + if (delPercent < compactRatio && tombPercent < compactRatio) { + return 0; + } + + if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) { + tsdbError("open meta file %s compact fail", pMFile->f.rname); + return -1; + } + + tsdbInfo("begin compact tsdb meta file, ratio:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 ",size:%" PRId64, + tsTsdbMetaCompactRatio, pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size); + + SMFile mf; + SDiskID did; + + // first create tmp meta file + did.level = TFS_PRIMARY_LEVEL; + did.id = TFS_PRIMARY_ID; + tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)) + 1); + + if (tsdbCreateMFile(&mf, true) < 0) { + tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d meta file %s is created to compact meta data", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); + + // second iterator metaCache + int code = -1; + int64_t maxBufSize = 1024; + SKVRecord *pRecord; + void *pBuf = NULL; + + pBuf = malloc((size_t)maxBufSize); + if (pBuf == NULL) { + goto _err; + } + + // init Comp + assert(pfs->metaCacheComp == NULL); + pfs->metaCacheComp = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (pfs->metaCacheComp == NULL) { + goto _err; + } + + pRecord = taosHashIterate(pfs->metaCache, NULL); + while (pRecord) { + if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { + tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), + tstrerror(terrno)); + goto _err; + } + if (pRecord->size > maxBufSize) { + maxBufSize = pRecord->size; + void* tmp = realloc(pBuf, (size_t)maxBufSize); + if (tmp == NULL) { + goto _err; + } + pBuf = tmp; + } + int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size); + if (nread < 0) { + tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), + tstrerror(terrno)); + goto _err; + } + + if (nread < pRecord->size) { + tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", + REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); + goto _err; + } + + if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, true) < 0) { + tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid, + tstrerror(terrno)); + goto _err; + } + + pRecord = taosHashIterate(pfs->metaCache, pRecord); + } + code = 0; + +_err: + if (code == 0) TSDB_FILE_FSYNC(&mf); + tsdbCloseMFile(&mf); + tsdbCloseMFile(pMFile); + + if (code == 0) { + // rename meta.tmp -> meta + tsdbInfo("vgId:%d meta file rename %s -> %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf), TSDB_FILE_FULL_NAME(pMFile)); + taosRename(mf.f.aname,pMFile->f.aname); + tstrncpy(mf.f.aname, pMFile->f.aname, TSDB_FILENAME_LEN); + tstrncpy(mf.f.rname, pMFile->f.rname, TSDB_FILENAME_LEN); + // update current meta file info + pfs->nstatus->pmf = NULL; + tsdbUpdateMFile(pfs, &mf); + + taosHashCleanup(pfs->metaCache); + pfs->metaCache = pfs->metaCacheComp; + pfs->metaCacheComp = NULL; + } else { + // remove meta.tmp file + remove(mf.f.aname); + taosHashCleanup(pfs->metaCacheComp); + pfs->metaCacheComp = NULL; + } + + tfree(pBuf); + + ASSERT(mf.info.nDels == 0); + ASSERT(mf.info.tombSize == 0); + + tsdbInfo("end compact tsdb meta file,code:%d,nRecords:%" PRId64 ",size:%" PRId64, + code,mf.info.nRecords,mf.info.size); + return code; +} + +// =================== Commit Time-Series Data +static int tsdbCommitTSData(STsdbRepo *pRepo) { + SMemTable *pMem = pRepo->imem; + SCommitH commith; + SDFileSet *pSet = NULL; + int fid; + + memset(&commith, 0, sizeof(commith)); + + if (pMem->numOfRows <= 0) { + // No memory data, just apply retention on each file on disk + if (tsdbApplyRtn(pRepo) < 0) { + return -1; + } + return 0; + } + + // Resource initialization + if (tsdbInitCommitH(&commith, pRepo) < 0) { + return -1; + } + + // Skip expired memory data and expired FSET + tsdbSeekCommitIter(&commith, commith.rtn.minKey); + while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { + if (pSet->fid < commith.rtn.minFid) { + tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + } else { + break; + } + } + + // Loop to commit to each file + fid = tsdbNextCommitFid(&(commith)); + while (true) { + // Loop over both on disk and memory + if (pSet == NULL && fid == TSDB_IVLD_FID) break; + + if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) { + // Only has existing FSET but no memory data to commit in this + // existing FSET, only check if file in correct retention + if (tsdbApplyRtnOnFSet(pRepo, pSet, &(commith.rtn)) < 0) { + tsdbDestroyCommitH(&commith); + return -1; + } + + pSet = tsdbFSIterNext(&(commith.fsIter)); + } else { + // Has memory data to commit + SDFileSet *pCSet; + int cfid; + + if (pSet == NULL || pSet->fid > fid) { + // Commit to a new FSET with fid: fid + pCSet = NULL; + cfid = fid; + } else { + // Commit to an existing FSET + pCSet = pSet; + cfid = pSet->fid; + pSet = tsdbFSIterNext(&(commith.fsIter)); + } + + if (tsdbCommitToFile(&commith, pCSet, cfid) < 0) { + tsdbDestroyCommitH(&commith); + return -1; + } + + fid = tsdbNextCommitFid(&commith); + } + } + + tsdbDestroyCommitH(&commith); + return 0; +} + +static void tsdbStartCommit(STsdbRepo *pRepo) { + SMemTable *pMem = pRepo->imem; + + ASSERT(pMem->numOfRows > 0 || listNEles(pMem->actList) > 0); + + tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64 " meta rows: %d", + REPO_ID(pRepo), pMem->keyFirst, pMem->keyLast, pMem->numOfRows, listNEles(pMem->actList)); + + tsdbStartFSTxn(pRepo, pMem->pointsAdd, pMem->storageAdd); + + pRepo->code = TSDB_CODE_SUCCESS; +} + +static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { + if (eno != TSDB_CODE_SUCCESS) { + tsdbEndFSTxnWithError(REPO_FS(pRepo)); + } else { + tsdbEndFSTxn(pRepo); + } + + tsdbInfo("vgId:%d commit over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); + + if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno); + + SMemTable *pIMem = pRepo->imem; + (void)tsdbLockRepo(pRepo); + pRepo->imem = NULL; + (void)tsdbUnlockRepo(pRepo); + tsdbUnRefMemTable(pRepo, pIMem); + tsem_post(&(pRepo->readyToCommit)); +} + +#if 0 +static bool tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { + for (int i = 0; i < nIters; i++) { + TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter); + if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return true; + } + return false; +} +#endif + +static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + + ASSERT(pSet == NULL || pSet->fid == fid); + + tsdbResetCommitFile(pCommith); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &(pCommith->minKey), &(pCommith->maxKey)); + + // Set and open files + if (tsdbSetAndOpenCommitFile(pCommith, pSet, fid) < 0) { + return -1; + } + + // Loop to commit each table data + for (int tid = 1; tid < pCommith->niters; tid++) { + SCommitIter *pIter = pCommith->iters + tid; + + if (pIter->pTable == NULL) continue; + + if (tsdbCommitToTable(pCommith, tid) < 0) { + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + } + + if (tsdbWriteBlockIdx(TSDB_COMMIT_HEAD_FILE(pCommith), pCommith->aBlkIdx, (void **)(&(TSDB_COMMIT_BUF(pCommith)))) < + 0) { + tsdbError("vgId:%d failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + + if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) { + tsdbError("vgId:%d failed to update FSET %d header since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + + // Close commit file + tsdbCloseCommitFile(pCommith, false); + + if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) { + return -1; + } + + return 0; +} + +static int tsdbCreateCommitIters(SCommitH *pCommith) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + SMemTable *pMem = pRepo->imem; + STsdbMeta *pMeta = pRepo->tsdbMeta; + + pCommith->niters = pMem->maxTables; + pCommith->iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter)); + if (pCommith->iters == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (tsdbRLockRepoMeta(pRepo) < 0) return -1; + + // reference all tables + for (int i = 0; i < pMem->maxTables; i++) { + if (pMeta->tables[i] != NULL) { + tsdbRefTable(pMeta->tables[i]); + pCommith->iters[i].pTable = pMeta->tables[i]; + } + } + + if (tsdbUnlockRepoMeta(pRepo) < 0) return -1; + + for (int i = 0; i < pMem->maxTables; i++) { + if ((pCommith->iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && + (TABLE_UID(pCommith->iters[i].pTable) == pMem->tData[i]->uid)) { + if ((pCommith->iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + tSkipListIterNext(pCommith->iters[i].pIter); + } + } + + return 0; +} + +static void tsdbDestroyCommitIters(SCommitH *pCommith) { + if (pCommith->iters == NULL) return; + + for (int i = 1; i < pCommith->niters; i++) { + if (pCommith->iters[i].pTable != NULL) { + tsdbUnRefTable(pCommith->iters[i].pTable); + tSkipListDestroyIter(pCommith->iters[i].pIter); + } + } + + free(pCommith->iters); + pCommith->iters = NULL; + pCommith->niters = 0; +} + +// Skip all keys until key (not included) +static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key) { + for (int i = 0; i < pCommith->niters; i++) { + SCommitIter *pIter = pCommith->iters + i; + if (pIter->pTable == NULL || pIter->pIter == NULL) continue; + + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, key - 1, INT32_MAX, NULL, NULL, 0, true, NULL); + } +} + +static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo) { + STsdbCfg *pCfg = REPO_CFG(pRepo); + + memset(pCommith, 0, sizeof(*pCommith)); + tsdbGetRtnSnap(pRepo, &(pCommith->rtn)); + + TSDB_FSET_SET_CLOSED(TSDB_COMMIT_WRITE_FSET(pCommith)); + + // Init read handle + if (tsdbInitReadH(&(pCommith->readh), pRepo) < 0) { + return -1; + } + + // Init file iterator + tsdbFSIterInit(&(pCommith->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD); + + if (tsdbCreateCommitIters(pCommith) < 0) { + tsdbDestroyCommitH(pCommith); + return -1; + } + + pCommith->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx)); + if (pCommith->aBlkIdx == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; + } + + pCommith->aSupBlk = taosArrayInit(1024, sizeof(SBlock)); + if (pCommith->aSupBlk == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; + } + + pCommith->aSubBlk = taosArrayInit(1024, sizeof(SBlock)); + if (pCommith->aSubBlk == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; + } + + pCommith->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); + if (pCommith->pDataCols == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; + } + + return 0; +} + +static void tsdbDestroyCommitH(SCommitH *pCommith) { + pCommith->pDataCols = tdFreeDataCols(pCommith->pDataCols); + pCommith->aSubBlk = taosArrayDestroy(pCommith->aSubBlk); + pCommith->aSupBlk = taosArrayDestroy(pCommith->aSupBlk); + pCommith->aBlkIdx = taosArrayDestroy(pCommith->aBlkIdx); + tsdbDestroyCommitIters(pCommith); + tsdbDestroyReadH(&(pCommith->readh)); + tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); +} + +static int tsdbNextCommitFid(SCommitH *pCommith) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + int fid = TSDB_IVLD_FID; + + for (int i = 0; i < pCommith->niters; i++) { + SCommitIter *pIter = pCommith->iters + i; + if (pIter->pTable == NULL || pIter->pIter == NULL) continue; + + TSKEY nextKey = tsdbNextIterKey(pIter->pIter); + if (nextKey == TSDB_DATA_TIMESTAMP_NULL) { + continue; + } else { + int tfid = (int)(TSDB_KEY_FID(nextKey, pCfg->daysPerFile, pCfg->precision)); + if (fid == TSDB_IVLD_FID || fid > tfid) { + fid = tfid; + } + } + } + + return fid; +} + +static int tsdbCommitToTable(SCommitH *pCommith, int tid) { + SCommitIter *pIter = pCommith->iters + tid; + TSKEY nextKey = tsdbNextIterKey(pIter->pIter); + + tsdbResetCommitTable(pCommith); + + TSDB_RLOCK_TABLE(pIter->pTable); + + // Set commit table + if (tsdbSetCommitTable(pCommith, pIter->pTable) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + + // No disk data and no memory data, just return + if (pCommith->readh.pBlkIdx == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return 0; + } + + // Must has disk data or has memory data + int nBlocks; + int bidx = 0; + SBlock *pBlock; + + if (pCommith->readh.pBlkIdx) { + if (tsdbLoadBlockInfo(&(pCommith->readh), NULL) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + + nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; + } else { + nBlocks = 0; + } + + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + + while (true) { + if (pBlock == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) break; + + if ((nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey) || + (pBlock && (!pBlock->last) && tsdbComparKeyBlock((void *)(&nextKey), pBlock) > 0)) { + if (tsdbMoveBlock(pCommith, bidx) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + + bidx++; + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + } else if (pBlock && (pBlock->last || tsdbComparKeyBlock((void *)(&nextKey), pBlock) == 0)) { + // merge pBlock data and memory data + if (tsdbMergeMemData(pCommith, pIter, bidx) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + + bidx++; + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + nextKey = tsdbNextIterKey(pIter->pIter); + } else { + // Only commit memory data + if (pBlock == NULL) { + if (tsdbCommitMemData(pCommith, pIter, pCommith->maxKey, false) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + } else { + if (tsdbCommitMemData(pCommith, pIter, pBlock->keyFirst - 1, true) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + } + nextKey = tsdbNextIterKey(pIter->pIter); + } + } + + TSDB_RUNLOCK_TABLE(pIter->pTable); + + if (tsdbWriteBlockInfo(pCommith) < 0) { + tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), + TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); + return -1; + } + + return 0; +} + +static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) { + STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); + + pCommith->pTable = pTable; + + if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (pCommith->isRFileSet) { + if (tsdbSetReadTable(&(pCommith->readh), pTable) < 0) { + return -1; + } + } else { + pCommith->readh.pBlkIdx = NULL; + } + return 0; +} + +static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { + TSKEY key = *(TSKEY *)arg1; + SBlock *pBlock = (SBlock *)arg2; + + if (key < pBlock->keyFirst) { + return -1; + } else if (key > pBlock->keyLast) { + return 1; + } else { + return 0; + } +} + +int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, + bool isLast, bool isSuper, void **ppBuf, void **ppCBuf) { + STsdbCfg * pCfg = REPO_CFG(pRepo); + SBlockData *pBlockData; + int64_t offset = 0; + int rowsToWrite = pDataCols->numOfRows; + + ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock); + ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock); + + // Make buffer space + if (tsdbMakeRoom(ppBuf, TSDB_BLOCK_STATIS_SIZE(pDataCols->numOfCols)) < 0) { + return -1; + } + pBlockData = (SBlockData *)(*ppBuf); + + // Get # of cols not all NULL(not including key column) + int nColsNotAllNull = 0; + for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column + SDataCol * pDataCol = pDataCols->cols + ncol; + SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; + + if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it + continue; + } + + memset(pBlockCol, 0, sizeof(*pBlockCol)); + + pBlockCol->colId = pDataCol->colId; + pBlockCol->type = pDataCol->type; + if (tDataTypes[pDataCol->type].statisFunc) { + (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), + &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), + &(pBlockCol->numOfNull)); + } + nColsNotAllNull++; + } + + ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols); + + // Compress the data if neccessary + int tcol = 0; // counter of not all NULL and written columns + uint32_t toffset = 0; + int32_t tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull); + int32_t lsize = tsize; + int32_t keyLen = 0; + for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { + // All not NULL columns finish + if (ncol != 0 && tcol >= nColsNotAllNull) break; + + SDataCol * pDataCol = pDataCols->cols + ncol; + SBlockCol *pBlockCol = pBlockData->cols + tcol; + + if (ncol != 0 && (pDataCol->colId != pBlockCol->colId)) continue; + + int32_t flen; // final length + int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite); + void * tptr; + + // Make room + if (tsdbMakeRoom(ppBuf, lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { + return -1; + } + pBlockData = (SBlockData *)(*ppBuf); + pBlockCol = pBlockData->cols + tcol; + tptr = POINTER_SHIFT(pBlockData, lsize); + + if (pCfg->compression == TWO_STAGE_COMP && + tsdbMakeRoom(ppCBuf, tlen + COMP_OVERFLOW_BYTES) < 0) { + return -1; + } + + // Compress or just copy + if (pCfg->compression) { + flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, + tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, + tlen + COMP_OVERFLOW_BYTES); + } else { + flen = tlen; + memcpy(tptr, pDataCol->pData, flen); + } + + // Add checksum + ASSERT(flen > 0); + flen += sizeof(TSCKSUM); + taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); + tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM))); + + if (ncol != 0) { + tsdbSetBlockColOffset(pBlockCol, toffset); + pBlockCol->len = flen; + tcol++; + } else { + keyLen = flen; + } + + toffset += flen; + lsize += flen; + } + + pBlockData->delimiter = TSDB_FILE_DELIMITER; + pBlockData->uid = TABLE_UID(pTable); + pBlockData->numOfCols = nColsNotAllNull; + + taosCalcChecksumAppend(0, (uint8_t *)pBlockData, tsize); + tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(pBlockData, tsize - sizeof(TSCKSUM))); + + // Write the whole block to file + if (tsdbAppendDFile(pDFile, (void *)pBlockData, lsize, &offset) < lsize) { + return -1; + } + + // Update pBlock membership vairables + pBlock->last = isLast; + pBlock->offset = offset; + pBlock->algorithm = pCfg->compression; + pBlock->numOfRows = rowsToWrite; + pBlock->len = lsize; + pBlock->keyLen = keyLen; + pBlock->numOfSubBlocks = isSuper ? 1 : 0; + pBlock->numOfCols = nColsNotAllNull; + pBlock->keyFirst = dataColsKeyFirst(pDataCols); + pBlock->keyLast = dataColsKeyLast(pDataCols); + + tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 + " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, + REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, + pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); + + return 0; +} + +static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, + bool isSuper) { + return tsdbWriteBlockImpl(TSDB_COMMIT_REPO(pCommith), TSDB_COMMIT_TABLE(pCommith), pDFile, pDataCols, pBlock, isLast, + isSuper, (void **)(&(TSDB_COMMIT_BUF(pCommith))), + (void **)(&(TSDB_COMMIT_COMP_BUF(pCommith)))); +} + + +static int tsdbWriteBlockInfo(SCommitH *pCommih) { + SDFile * pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); + SBlockIdx blkIdx; + STable * pTable = TSDB_COMMIT_TABLE(pCommih); + + if (tsdbWriteBlockInfoImpl(pHeadf, pTable, pCommih->aSupBlk, pCommih->aSubBlk, (void **)(&(TSDB_COMMIT_BUF(pCommih))), + &blkIdx) < 0) { + return -1; + } + + if (blkIdx.numOfBlocks == 0) { + return 0; + } + + if (taosArrayPush(pCommih->aBlkIdx, (void *)(&blkIdx)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + SMergeInfo mInfo; + int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); + SDFile * pDFile; + bool isLast; + SBlock block; + + while (true) { + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, defaultRows, pCommith->pDataCols, NULL, 0, + pCfg->update, &mInfo); + + if (pCommith->pDataCols->numOfRows <= 0) break; + + if (toData || pCommith->pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; + } else { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isLast = true; + } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; + + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) { + return -1; + } + } + + return 0; +} + +static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + int nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; + SBlock * pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + TSKEY keyLimit; + int16_t colId = 0; + SMergeInfo mInfo; + SBlock subBlocks[TSDB_MAX_SUBBLOCKS]; + SBlock block, supBlock; + SDFile * pDFile; + + if (bidx == nBlocks - 1) { + keyLimit = pCommith->maxKey; + } else { + keyLimit = pBlock[1].keyFirst - 1; + } + + SSkipListIterator titer = *(pIter->pIter); + if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1) < 0) return -1; + + tsdbLoadDataFromCache(pIter->pTable, &titer, keyLimit, INT32_MAX, NULL, pCommith->readh.pDCols[0]->cols[0].pData, + pCommith->readh.pDCols[0]->numOfRows, pCfg->update, &mInfo); + + if (mInfo.nOperations == 0) { + // no new data to insert (all updates denied) + if (tsdbMoveBlock(pCommith, bidx) < 0) { + return -1; + } + *(pIter->pIter) = titer; + } else if (pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed == 0) { + // Ignore the block + ASSERT(0); + *(pIter->pIter) = titer; + } else if (tsdbCanAddSubBlock(pCommith, pBlock, &mInfo)) { + // Add a sub-block + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, INT32_MAX, pCommith->pDataCols, + pCommith->readh.pDCols[0]->cols[0].pData, pCommith->readh.pDCols[0]->numOfRows, pCfg->update, + &mInfo); + if (pBlock->last) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, pBlock->last, false) < 0) return -1; + + if (pBlock->numOfSubBlocks == 1) { + subBlocks[0] = *pBlock; + subBlocks[0].numOfSubBlocks = 0; + } else { + memcpy(subBlocks, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), + sizeof(SBlock) * pBlock->numOfSubBlocks); + } + subBlocks[pBlock->numOfSubBlocks] = block; + supBlock = *pBlock; + supBlock.keyFirst = mInfo.keyFirst; + supBlock.keyLast = mInfo.keyLast; + supBlock.numOfSubBlocks++; + supBlock.numOfRows = pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed; + supBlock.offset = taosArrayGetSize(pCommith->aSubBlk) * sizeof(SBlock); + + if (tsdbCommitAddBlock(pCommith, &supBlock, subBlocks, supBlock.numOfSubBlocks) < 0) return -1; + } else { + if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; + if (tsdbMergeBlockData(pCommith, pIter, pCommith->readh.pDCols[0], keyLimit, bidx == (nBlocks - 1)) < 0) return -1; + } + + return 0; +} + +static int tsdbMoveBlock(SCommitH *pCommith, int bidx) { + SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + SDFile *pDFile; + SBlock block; + bool isSameFile; + + ASSERT(pBlock->numOfSubBlocks > 0); + + if (pBlock->last) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isSameFile = pCommith->isLFileSame; + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isSameFile = pCommith->isDFileSame; + } + + if (isSameFile) { + if (pBlock->numOfSubBlocks == 1) { + if (tsdbCommitAddBlock(pCommith, pBlock, NULL, 0) < 0) { + return -1; + } + } else { + block = *pBlock; + block.offset = sizeof(SBlock) * taosArrayGetSize(pCommith->aSubBlk); + + if (tsdbCommitAddBlock(pCommith, &block, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), + pBlock->numOfSubBlocks) < 0) { + return -1; + } + } + } else { + if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; + if (tsdbWriteBlock(pCommith, pDFile, pCommith->readh.pDCols[0], &block, pBlock->last, true) < 0) return -1; + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; + } + + return 0; +} + +static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks) { + if (taosArrayPush(pCommith->aSupBlk, pSupBlock) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (pSubBlocks && taosArrayAddBatch(pCommith->aSubBlk, pSubBlocks, nSubBlocks) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, bool isLastOneBlock) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + SBlock block; + SDFile * pDFile; + bool isLast; + int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); + + int biter = 0; + while (true) { + tsdbLoadAndMergeFromCache(pCommith->readh.pDCols[0], &biter, pIter, pCommith->pDataCols, keyLimit, defaultRows, + pCfg->update); + + if (pCommith->pDataCols->numOfRows == 0) break; + + if (isLastOneBlock) { + if (pCommith->pDataCols->numOfRows < pCfg->minRowsPerFileBlock) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isLast = true; + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; + } + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; + } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; + } + + return 0; +} + +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, + TSKEY maxKey, int maxRows, int8_t update) { + TSKEY key1 = INT64_MAX; + TSKEY key2 = INT64_MAX; + STSchema *pSchema = NULL; + + ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); + tdResetDataCols(pTarget); + + while (true) { + key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); + SMemRow row = tsdbNextIterRow(pCommitIter->pIter); + if (row == NULL || memRowKey(row) > maxKey) { + key2 = INT64_MAX; + } else { + key2 = memRowKey(row); + } + + if (key1 == INT64_MAX && key2 == INT64_MAX) break; + + if (key1 < key2) { + for (int i = 0; i < pDataCols->numOfCols; i++) { + //TODO: dataColAppendVal may fail + dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, + pTarget->maxPoints); + } + + pTarget->numOfRows++; + (*iter)++; + } else if (key1 > key2) { + if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row)); + ASSERT(pSchema != NULL); + } + + tdAppendMemRowToDataCol(row, pSchema, pTarget, true); + + tSkipListIterNext(pCommitIter->pIter); + } else { + if (update != TD_ROW_OVERWRITE_UPDATE) { + //copy disk data + for (int i = 0; i < pDataCols->numOfCols; i++) { + //TODO: dataColAppendVal may fail + dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, + pTarget->maxPoints); + } + + if(update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++; + } + if (update != TD_ROW_DISCARD_UPDATE) { + //copy mem data + if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row)); + ASSERT(pSchema != NULL); + } + + tdAppendMemRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE); + } + (*iter)++; + tSkipListIterNext(pCommitIter->pIter); + } + + if (pTarget->numOfRows >= maxRows) break; + } +} + +static void tsdbResetCommitFile(SCommitH *pCommith) { + pCommith->isRFileSet = false; + pCommith->isDFileSame = false; + pCommith->isLFileSame = false; + taosArrayClear(pCommith->aBlkIdx); +} + +static void tsdbResetCommitTable(SCommitH *pCommith) { + taosArrayClear(pCommith->aSubBlk); + taosArrayClear(pCommith->aSupBlk); + pCommith->pTable = NULL; +} + +static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { + SDiskID did; + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith); + + tfsAllocDisk(tsdbGetFidLevel(fid, &(pCommith->rtn)), &(did.level), &(did.id)); + if (did.level == TFS_UNDECIDED_LEVEL) { + terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + return -1; + } + + // Open read FSET + if (pSet) { + if (tsdbSetAndOpenReadFSet(&(pCommith->readh), pSet) < 0) { + return -1; + } + + pCommith->isRFileSet = true; + + if (tsdbLoadBlockIdx(&(pCommith->readh)) < 0) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + + tsdbDebug("vgId:%d FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet), + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + } else { + pCommith->isRFileSet = false; + } + + // Set and open commit FSET + if (pSet == NULL || did.level > TSDB_FSET_LEVEL(pSet)) { + // Create a new FSET to write data + tsdbInitDFileSet(pWSet, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo))); + + if (tsdbCreateDFileSet(pWSet, true) < 0) { + tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), + TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno)); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + } + return -1; + } + + pCommith->isDFileSame = false; + pCommith->isLFileSame = false; + + tsdbDebug("vgId:%d FSET %d at level %d disk id %d is created to commit", REPO_ID(pRepo), TSDB_FSET_FID(pWSet), + TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet)); + } else { + did.level = TSDB_FSET_LEVEL(pSet); + did.id = TSDB_FSET_ID(pSet); + + pCommith->wSet.fid = fid; + pCommith->wSet.state = 0; + + // TSDB_FILE_HEAD + SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); + tsdbInitDFile(pWHeadf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); + if (tsdbCreateDFile(pWHeadf, true) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), + tstrerror(terrno)); + + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + + // TSDB_FILE_DATA + SDFile *pRDataf = TSDB_READ_DATA_FILE(&(pCommith->readh)); + SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith); + tsdbInitDFileEx(pWDataf, pRDataf); + if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + pCommith->isDFileSame = true; + + // TSDB_FILE_LAST + SDFile *pRLastf = TSDB_READ_LAST_FILE(&(pCommith->readh)); + SDFile *pWLastf = TSDB_COMMIT_LAST_FILE(pCommith); + if (pRLastf->info.size < 32 * 1024) { + tsdbInitDFileEx(pWLastf, pRLastf); + pCommith->isLFileSame = true; + + if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } else { + tsdbInitDFile(pWLastf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST); + pCommith->isLFileSame = false; + + if (tsdbCreateDFile(pWLastf, true) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + (void)tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } + } + + return 0; +} + +static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError) { + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + } + + if (!hasError) { + TSDB_FSET_FSYNC(TSDB_COMMIT_WRITE_FSET(pCommith)); + } + tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); +} + +static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + int mergeRows = pBlock->numOfRows + pInfo->rowsInserted - pInfo->rowsDeleteSucceed; + + ASSERT(mergeRows > 0); + + if (pBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pInfo->nOperations <= pCfg->maxRowsPerFileBlock) { + if (pBlock->last) { + if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true; + } else { + if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true; + } + } + + return false; +} + +int tsdbApplyRtn(STsdbRepo *pRepo) { + SRtn rtn; + SFSIter fsiter; + STsdbFS * pfs = REPO_FS(pRepo); + SDFileSet *pSet; + + // Get retention snapshot + tsdbGetRtnSnap(pRepo, &rtn); + + tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); + while ((pSet = tsdbFSIterNext(&fsiter))) { + if (pSet->fid < rtn.minFid) { + tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + continue; + } + + if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) { + return -1; + } + } + + return 0; +} +#endif \ No newline at end of file diff --git a/src/tsdb/src/tsdbCompact.c b/source/dnode/vnode/tsdb/src/tsdbCompact.c similarity index 100% rename from src/tsdb/src/tsdbCompact.c rename to source/dnode/vnode/tsdb/src/tsdbCompact.c diff --git a/src/tsdb/src/tsdbFS.c b/source/dnode/vnode/tsdb/src/tsdbFS.c similarity index 100% rename from src/tsdb/src/tsdbFS.c rename to source/dnode/vnode/tsdb/src/tsdbFS.c diff --git a/src/tsdb/src/tsdbFile.c b/source/dnode/vnode/tsdb/src/tsdbFile.c similarity index 100% rename from src/tsdb/src/tsdbFile.c rename to source/dnode/vnode/tsdb/src/tsdbFile.c diff --git a/src/tsdb/src/tsdbHealth.c b/source/dnode/vnode/tsdb/src/tsdbHealth.c similarity index 100% rename from src/tsdb/src/tsdbHealth.c rename to source/dnode/vnode/tsdb/src/tsdbHealth.c diff --git a/source/dnode/vnode/tsdb/src/tsdbIdx.c b/source/dnode/vnode/tsdb/src/tsdbIdx.c deleted file mode 100644 index 6dea4a4e57392be988126c579648f39a8270b9bf..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/tsdb/src/tsdbIdx.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/src/tsdbMain.c b/source/dnode/vnode/tsdb/src/tsdbMain.c index d67d45660da5a02636a71dcb6d4461a3ddb8af41..c8bcfc69062cdac8e140edd6143a5c9c39568ab7 100644 --- a/source/dnode/vnode/tsdb/src/tsdbMain.c +++ b/source/dnode/vnode/tsdb/src/tsdbMain.c @@ -92,4 +92,1014 @@ static int tsdbOpenImpl(STsdb *pTsdb) { static void tsdbCloseImpl(STsdb *pTsdb) { // TODO -} \ No newline at end of file +} +#if 0 +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// no test file errors here +#include "taosdef.h" +#include "tsdbint.h" +#include "ttimer.h" +#include "tthread.h" + +#define IS_VALID_PRECISION(precision) \ + (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO)) +#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP +#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) + +static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); +static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH); +static void tsdbFreeRepo(STsdbRepo *pRepo); +static void tsdbStartStream(STsdbRepo *pRepo); +static void tsdbStopStream(STsdbRepo *pRepo); +static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh); +static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, SBlockIdx *pIdx); + +// Function declaration +int32_t tsdbCreateRepo(int repoid) { + char tsdbDir[TSDB_FILENAME_LEN] = "\0"; + char dataDir[TSDB_FILENAME_LEN] = "\0"; + + tsdbGetRootDir(repoid, tsdbDir); + if (tfsMkdir(tsdbDir) < 0) { + goto _err; + } + + tsdbGetDataDir(repoid, dataDir); + if (tfsMkdir(dataDir) < 0) { + goto _err; + } + + // TODO: need to create current file with nothing in + + return 0; + +_err: + tsdbError("vgId:%d failed to create TSDB repository since %s", repoid, tstrerror(terrno)); + return -1; +} + +int32_t tsdbDropRepo(int repoid) { + char tsdbDir[TSDB_FILENAME_LEN] = "\0"; + + tsdbGetRootDir(repoid, tsdbDir); + return tfsRmdir(tsdbDir); +} + +STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { + STsdbRepo *pRepo; + STsdbCfg config = *pCfg; + + terrno = TSDB_CODE_SUCCESS; + + // Check and set default configurations + if (tsdbCheckAndSetDefaultCfg(&config) < 0) { + tsdbError("vgId:%d failed to open TSDB repository since %s", config.tsdbId, tstrerror(terrno)); + return NULL; + } + + // Create new TSDB object + if ((pRepo = tsdbNewRepo(&config, pAppH)) == NULL) { + tsdbError("vgId:%d failed to open TSDB repository while creating TSDB object since %s", config.tsdbId, + tstrerror(terrno)); + return NULL; + } + + // Open meta + if (tsdbOpenMeta(pRepo) < 0) { + tsdbError("vgId:%d failed to open TSDB repository while opening Meta since %s", config.tsdbId, tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; + } + + if (tsdbOpenBufPool(pRepo) < 0) { + tsdbError("vgId:%d failed to open TSDB repository while opening buffer pool since %s", config.tsdbId, + tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; + } + + if (tsdbOpenFS(pRepo) < 0) { + tsdbError("vgId:%d failed to open TSDB repository while opening FS since %s", config.tsdbId, tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; + } + + // TODO: Restore information from data + if ((!(pRepo->state & TSDB_STATE_BAD_DATA)) && tsdbRestoreInfo(pRepo) < 0) { + tsdbError("vgId:%d failed to open TSDB repository while restore info since %s", config.tsdbId, tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; + } + + pRepo->mergeBuf = NULL; + + tsdbStartStream(pRepo); + + tsdbDebug("vgId:%d, TSDB repository opened", REPO_ID(pRepo)); + + return pRepo; +} + +// Note: all working thread and query thread must stopped when calling this function +int tsdbCloseRepo(STsdbRepo *repo, int toCommit) { + if (repo == NULL) return 0; + + STsdbRepo *pRepo = repo; + int vgId = REPO_ID(pRepo); + + terrno = TSDB_CODE_SUCCESS; + + tsdbStopStream(pRepo); + if(pRepo->pthread){ + taosDestoryThread(pRepo->pthread); + pRepo->pthread = NULL; + } + + if (toCommit) { + tsdbSyncCommit(repo); + } + + tsem_wait(&(pRepo->readyToCommit)); + + tsdbUnRefMemTable(pRepo, pRepo->mem); + tsdbUnRefMemTable(pRepo, pRepo->imem); + pRepo->mem = NULL; + pRepo->imem = NULL; + + tsdbCloseFS(pRepo); + tsdbCloseBufPool(pRepo); + tsdbCloseMeta(pRepo); + tsdbFreeRepo(pRepo); + tsdbDebug("vgId:%d repository is closed", vgId); + + if (terrno != TSDB_CODE_SUCCESS) { + return -1; + } else { + return 0; + } +} + +STsdbCfg *tsdbGetCfg(const STsdbRepo *repo) { + ASSERT(repo != NULL); + return &((STsdbRepo *)repo)->config; +} + +int tsdbLockRepo(STsdbRepo *pRepo) { + int code = pthread_mutex_lock(&pRepo->mutex); + if (code != 0) { + tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + pRepo->repoLocked = true; + return 0; +} + +int tsdbUnlockRepo(STsdbRepo *pRepo) { + ASSERT(IS_REPO_LOCKED(pRepo)); + pRepo->repoLocked = false; + int code = pthread_mutex_unlock(&pRepo->mutex); + if (code != 0) { + tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +int tsdbCheckCommit(STsdbRepo *pRepo) { + ASSERT(pRepo->mem != NULL); + STsdbCfg *pCfg = &(pRepo->config); + + STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); + ASSERT(pBufBlock != NULL); + if ((pRepo->mem->extraBuffList != NULL) || + ((listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 3) && (pBufBlock->remain < TSDB_BUFFER_RESERVE))) { + // trigger commit + if (tsdbAsyncCommit(pRepo) < 0) return -1; + } + + return 0; +} + +STsdbMeta *tsdbGetMeta(STsdbRepo *pRepo) { return pRepo->tsdbMeta; } + +STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo) { return NULL; } + +int tsdbGetState(STsdbRepo *repo) { return repo->state; } + +int8_t tsdbGetCompactState(STsdbRepo *repo) { return (int8_t)(repo->compactState); } + +void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) { + ASSERT(repo != NULL); + STsdbRepo *pRepo = repo; + *totalPoints = pRepo->stat.pointsWritten; + *totalStorage = pRepo->stat.totalStorage; + *compStorage = pRepo->stat.compStorage; +} + +int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { + // TODO: think about multithread cases + if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; + + STsdbCfg * pRCfg = &repo->config; + + ASSERT(pRCfg->tsdbId == pCfg->tsdbId); + ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); + ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); + ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); + ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); + ASSERT(pRCfg->precision == pCfg->precision); + + bool configChanged = false; + if (pRCfg->compression != pCfg->compression) { + configChanged = true; + } + if (pRCfg->keep != pCfg->keep) { + configChanged = true; + } + if (pRCfg->keep1 != pCfg->keep1) { + configChanged = true; + } + if (pRCfg->keep2 != pCfg->keep2) { + configChanged = true; + } + if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { + configChanged = true; + } + if (pRCfg->totalBlocks != pCfg->totalBlocks) { + configChanged = true; + } + + if (!configChanged) { + tsdbError("vgId:%d no config changed", REPO_ID(repo)); + } + + int code = pthread_mutex_lock(&repo->save_mutex); + if (code != 0) { + tsdbError("vgId:%d failed to lock tsdb save config mutex since %s", REPO_ID(repo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + + STsdbCfg * pSaveCfg = &repo->save_config; + *pSaveCfg = repo->config; + + pSaveCfg->compression = pCfg->compression; + pSaveCfg->keep = pCfg->keep; + pSaveCfg->keep1 = pCfg->keep1; + pSaveCfg->keep2 = pCfg->keep2; + pSaveCfg->cacheLastRow = pCfg->cacheLastRow; + pSaveCfg->totalBlocks = pCfg->totalBlocks; + + tsdbInfo("vgId:%d old config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d),totalBlocks(%d)", + REPO_ID(repo), + pRCfg->compression, pRCfg->keep, pRCfg->keep1,pRCfg->keep2, + pRCfg->cacheLastRow, pRCfg->totalBlocks); + tsdbInfo("vgId:%d new config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d),totalBlocks(%d)", + REPO_ID(repo), + pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, + pSaveCfg->cacheLastRow,pSaveCfg->totalBlocks); + + repo->config_changed = true; + + pthread_mutex_unlock(&repo->save_mutex); + + // schedule a commit msg and wait for the new config applied + tsdbSyncCommitConfig(repo); + + return 0; +#if 0 + STsdbRepo *pRepo = (STsdbRepo *)repo; + STsdbCfg config = pRepo->config; + STsdbCfg * pRCfg = &pRepo->config; + + if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; + + ASSERT(pRCfg->tsdbId == pCfg->tsdbId); + ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); + ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); + ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); + ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); + ASSERT(pRCfg->precision == pCfg->precision); + + bool configChanged = false; + if (pRCfg->compression != pCfg->compression) { + tsdbAlterCompression(pRepo, pCfg->compression); + config.compression = pCfg->compression; + configChanged = true; + } + if (pRCfg->keep != pCfg->keep) { + if (tsdbAlterKeep(pRepo, pCfg->keep) < 0) { + tsdbError("vgId:%d failed to configure repo when alter keep since %s", REPO_ID(pRepo), tstrerror(terrno)); + config.keep = pCfg->keep; + return -1; + } + configChanged = true; + } + if (pRCfg->totalBlocks != pCfg->totalBlocks) { + tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks); + config.totalBlocks = pCfg->totalBlocks; + configChanged = true; + } + if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { + config.cacheLastRow = pCfg->cacheLastRow; + configChanged = true; + } + + if (configChanged) { + if (tsdbSaveConfig(pRepo->rootDir, &config) < 0) { + tsdbError("vgId:%d failed to configure repository while save config since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } + + return 0; +#endif +} + +uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) { + // TODO + return 0; +#if 0 + STsdbRepo *pRepo = (STsdbRepo *)repo; + // STsdbMeta *pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + uint32_t magic = 0; + char * fname = NULL; + + struct stat fState; + + tsdbDebug("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex); + ASSERT(*index <= eindex); + + if (name[0] == 0) { // get the file from index or after, but not larger than eindex + int fid = (*index) / TSDB_FILE_TYPE_MAX; + + if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { + if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { + fname = tsdbGetMetaFileName(pRepo->rootDir); + *index = TSDB_META_FILE_INDEX; + magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); + sprintf(name, "tsdb/%s", TSDB_META_FILE_NAME); + } else { + return 0; + } + } else { + SFileGroup *pFGroup = + taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); + if (pFGroup->fileId == fid) { + SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX]; + fname = strdup(TSDB_FILE_NAME(pFile)); + magic = pFile->info.magic; + char *tfname = strdup(fname); + sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); + tfree(tfname); + } else { + if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) { + SFile *pFile = &pFGroup->files[0]; + fname = strdup(TSDB_FILE_NAME(pFile)); + *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; + magic = pFile->info.magic; + char *tfname = strdup(fname); + sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); + tfree(tfname); + } else { + return 0; + } + } + } + } else { // get the named file at the specified index. If not there, return 0 + fname = malloc(256); + sprintf(fname, "%s/vnode/vnode%d/%s", TFS_PRIMARY_PATH(), REPO_ID(pRepo), name); + if (access(fname, F_OK) != 0) { + tfree(fname); + return 0; + } + if (*index == TSDB_META_FILE_INDEX) { // get meta file + tsdbGetStoreInfo(fname, &magic, size); + } else { + char tfname[TSDB_FILENAME_LEN] = "\0"; + sprintf(tfname, "vnode/vnode%d/tsdb/%s/%s", REPO_ID(pRepo), TSDB_DATA_DIR_NAME, basename(name)); + tsdbGetFileInfoImpl(tfname, &magic, size); + } + tfree(fname); + return magic; + } + + if (stat(fname, &fState) < 0) { + tfree(fname); + return 0; + } + + *size = fState.st_size; + // magic = *size; + + tfree(fname); + return magic; +#endif +} + +void tsdbGetRootDir(int repoid, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb", repoid); +} + +void tsdbGetDataDir(int repoid, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data", repoid); +} + +static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { + // Check tsdbId + if (pCfg->tsdbId < 0) { + tsdbError("vgId:%d invalid vgroup ID", pCfg->tsdbId); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + + // Check precision + if (pCfg->precision == -1) { + pCfg->precision = TSDB_DEFAULT_PRECISION; + } else { + if (!IS_VALID_PRECISION(pCfg->precision)) { + tsdbError("vgId:%d invalid precision configuration %d", pCfg->tsdbId, pCfg->precision); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + } + + // Check compression + if (pCfg->compression == -1) { + pCfg->compression = TSDB_DEFAULT_COMPRESSION; + } else { + if (!IS_VALID_COMPRESSION(pCfg->compression)) { + tsdbError("vgId:%d invalid compression configuration %d", pCfg->tsdbId, pCfg->precision); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + } + + // Check daysPerFile + if (pCfg->daysPerFile == -1) { + pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; + } else { + if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { + tsdbError( + "vgId:%d invalid daysPerFile configuration! daysPerFile %d TSDB_MIN_DAYS_PER_FILE %d TSDB_MAX_DAYS_PER_FILE " + "%d", + pCfg->tsdbId, pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + } + + // Check minRowsPerFileBlock and maxRowsPerFileBlock + if (pCfg->minRowsPerFileBlock == -1) { + pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; + } else { + if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { + tsdbError( + "vgId:%d invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MIN_ROW_FBLOCK %d " + "TSDB_MAX_MIN_ROW_FBLOCK %d", + pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + } + + if (pCfg->maxRowsPerFileBlock == -1) { + pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; + } else { + if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { + tsdbError( + "vgId:%d invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAX_ROW_FBLOCK %d " + "TSDB_MAX_MAX_ROW_FBLOCK %d", + pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + } + + if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { + tsdbError("vgId:%d invalid configuration! minRowsPerFileBlock %d maxRowsPerFileBlock %d", pCfg->tsdbId, + pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + + // Check keep + if (pCfg->keep == -1) { + pCfg->keep = TSDB_DEFAULT_KEEP; + } else { + if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) { + tsdbError( + "vgId:%d invalid keep configuration! keep %d TSDB_MIN_KEEP %d " + "TSDB_MAX_KEEP %d", + pCfg->tsdbId, pCfg->keep, TSDB_MIN_KEEP, TSDB_MAX_KEEP); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + } + + if (pCfg->keep1 == 0) { + pCfg->keep1 = pCfg->keep; + } + + if (pCfg->keep2 == 0) { + pCfg->keep2 = pCfg->keep; + } + + // update check + if (pCfg->update < TD_ROW_DISCARD_UPDATE || pCfg->update > TD_ROW_PARTIAL_UPDATE) + pCfg->update = TD_ROW_DISCARD_UPDATE; + + // update cacheLastRow + if (pCfg->cacheLastRow != 0) { + if (pCfg->cacheLastRow > 3) + pCfg->cacheLastRow = 1; + } + return 0; +} + +static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { + STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(*pRepo)); + if (pRepo == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + + pRepo->state = TSDB_STATE_OK; + pRepo->code = TSDB_CODE_SUCCESS; + pRepo->compactState = 0; + pRepo->config = *pCfg; + if (pAppH) { + pRepo->appH = *pAppH; + } + pRepo->repoLocked = false; + pRepo->pthread = NULL; + + int code = pthread_mutex_init(&(pRepo->mutex), NULL); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + tsdbFreeRepo(pRepo); + return NULL; + } + + code = pthread_mutex_init(&(pRepo->save_mutex), NULL); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + tsdbFreeRepo(pRepo); + return NULL; + } + pRepo->config_changed = false; + atomic_store_8(&pRepo->hasCachedLastColumn, 0); + + code = tsem_init(&(pRepo->readyToCommit), 0, 1); + if (code != 0) { + code = errno; + terrno = TAOS_SYSTEM_ERROR(code); + tsdbFreeRepo(pRepo); + return NULL; + } + + pRepo->tsdbMeta = tsdbNewMeta(pCfg); + if (pRepo->tsdbMeta == NULL) { + tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbFreeRepo(pRepo); + return NULL; + } + + pRepo->pPool = tsdbNewBufPool(pCfg); + if (pRepo->pPool == NULL) { + tsdbError("vgId:%d failed to create buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbFreeRepo(pRepo); + return NULL; + } + + pRepo->fs = tsdbNewFS(pCfg); + if (pRepo->fs == NULL) { + tsdbError("vgId:%d failed to TSDB file system since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbFreeRepo(pRepo); + return NULL; + } + + return pRepo; +} + +static void tsdbFreeRepo(STsdbRepo *pRepo) { + if (pRepo) { + tsdbFreeFS(pRepo->fs); + tsdbFreeBufPool(pRepo->pPool); + tsdbFreeMeta(pRepo->tsdbMeta); + tsdbFreeMergeBuf(pRepo->mergeBuf); + // tsdbFreeMemTable(pRepo->mem); + // tsdbFreeMemTable(pRepo->imem); + tsem_destroy(&(pRepo->readyToCommit)); + pthread_mutex_destroy(&pRepo->mutex); + free(pRepo); + } +} + +static void tsdbStartStream(STsdbRepo *pRepo) { + STsdbMeta *pMeta = pRepo->tsdbMeta; + + for (int i = 0; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable && pTable->type == TSDB_STREAM_TABLE) { + pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, + tsdbGetTableSchemaImpl(pTable, false, false, -1), 0); + } + } +} + +static void tsdbStopStream(STsdbRepo *pRepo) { + STsdbMeta *pMeta = pRepo->tsdbMeta; + + for (int i = 0; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable && pTable->type == TSDB_STREAM_TABLE) { + (*pRepo->appH.cqDropFunc)(pTable->cqhandle); + } + } +} + +static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh) { + //tsdbInfo("tsdbRestoreLastColumns of table %s", pTable->name->data); + + STSchema *pSchema = tsdbGetTableLatestSchema(pTable); + if (pSchema == NULL) { + tsdbError("tsdbGetTableLatestSchema of table %s fail", pTable->name->data); + return 0; + } + + SBlock* pBlock; + int numColumns; + int32_t blockIdx; + SDataStatis* pBlockStatis = NULL; + SMemRow row = NULL; + // restore last column data with last schema + + int err = 0; + + numColumns = schemaNCols(pSchema); + if (numColumns <= pTable->restoreColumnNum) { + pTable->hasRestoreLastColumn = true; + return 0; + } + if (pTable->lastColSVersion != schemaVersion(pSchema)) { + if (tsdbInitColIdCacheWithSchema(pTable, pSchema) < 0) { + return -1; + } + } + + row = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); + if (row == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + err = -1; + goto out; + } + + memRowSetType(row, SMEM_ROW_DATA); + tdInitDataRow(memRowDataBody(row), pSchema); + + // first load block index info + if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { + err = -1; + goto out; + } + + pBlockStatis = calloc(numColumns, sizeof(SDataStatis)); + if (pBlockStatis == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + err = -1; + goto out; + } + memset(pBlockStatis, 0, numColumns * sizeof(SDataStatis)); + for(int32_t i = 0; i < numColumns; ++i) { + STColumn *pCol = schemaColAt(pSchema, i); + pBlockStatis[i].colId = pCol->colId; + } + + // load block from backward + SBlockIdx *pIdx = pReadh->pBlkIdx; + blockIdx = (int32_t)(pIdx->numOfBlocks - 1); + + while (numColumns > pTable->restoreColumnNum && blockIdx >= 0) { + bool loadStatisData = false; + pBlock = pReadh->pBlkInfo->blocks + blockIdx; + blockIdx -= 1; + + // load block data + if (tsdbLoadBlockData(pReadh, pBlock, NULL) < 0) { + err = -1; + goto out; + } + + // file block with sub-blocks has no statistics data + if (pBlock->numOfSubBlocks <= 1) { + tsdbLoadBlockStatis(pReadh, pBlock); + tsdbGetBlockStatis(pReadh, pBlockStatis, (int)numColumns); + loadStatisData = true; + } + + for (int16_t i = 0; i < numColumns && numColumns > pTable->restoreColumnNum; ++i) { + STColumn *pCol = schemaColAt(pSchema, i); + // ignore loaded columns + if (pTable->lastCols[i].bytes != 0) { + continue; + } + + // ignore block which has no not-null colId column + if (loadStatisData && pBlockStatis[i].numOfNull == pBlock->numOfRows) { + continue; + } + + // OK,let's load row from backward to get not-null column + for (int32_t rowId = pBlock->numOfRows - 1; rowId >= 0; rowId--) { + SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; + const void* pColData = tdGetColDataOfRow(pDataCol, rowId); + tdAppendColVal(memRowDataBody(row), pColData, pCol->type, pCol->offset); + //SDataCol *pDataCol = readh.pDCols[0]->cols + j; + void *value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); + if (isNull(value, pCol->type)) { + continue; + } + + int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pCol->colId); + if (idx == -1) { + tsdbError("tsdbRestoreLastColumns restore vgId:%d,table:%s cache column %d fail", REPO_ID(pRepo), pTable->name->data, pCol->colId); + continue; + } + // save not-null column + uint16_t bytes = IS_VAR_DATA_TYPE(pCol->type) ? varDataTLen(pColData) : pCol->bytes; + SDataCol *pLastCol = &(pTable->lastCols[idx]); + pLastCol->pData = malloc(bytes); + pLastCol->bytes = bytes; + pLastCol->colId = pCol->colId; + memcpy(pLastCol->pData, value, bytes); + + // save row ts(in column 0) + pDataCol = pReadh->pDCols[0]->cols + 0; + pCol = schemaColAt(pSchema, 0); + tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->offset); + pLastCol->ts = memRowKey(row); + + pTable->restoreColumnNum += 1; + + tsdbDebug("tsdbRestoreLastColumns restore vgId:%d,table:%s cache column %d, %" PRId64, REPO_ID(pRepo), pTable->name->data, pLastCol->colId, pLastCol->ts); + break; + } + } + } + +out: + taosTZfree(row); + tfree(pBlockStatis); + + if (err == 0 && numColumns <= pTable->restoreColumnNum) { + pTable->hasRestoreLastColumn = true; + } + + return err; +} + +static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, SBlockIdx *pIdx) { + ASSERT(pTable->lastRow == NULL); + if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { + return -1; + } + + SBlock* pBlock = pReadh->pBlkInfo->blocks + pIdx->numOfBlocks - 1; + + if (tsdbLoadBlockData(pReadh, pBlock, NULL) < 0) { + return -1; + } + + // Get the data in row + + STSchema *pSchema = tsdbGetTableSchema(pTable); + pTable->lastRow = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); + if (pTable->lastRow == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + memRowSetType(pTable->lastRow, SMEM_ROW_DATA); + tdInitDataRow(memRowDataBody(pTable->lastRow), pSchema); + for (int icol = 0; icol < schemaNCols(pSchema); icol++) { + STColumn *pCol = schemaColAt(pSchema, icol); + SDataCol *pDataCol = pReadh->pDCols[0]->cols + icol; + tdAppendColVal(memRowDataBody(pTable->lastRow), tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, + pCol->offset); + } + + return 0; +} + +int tsdbRestoreInfo(STsdbRepo *pRepo) { + SFSIter fsiter; + SReadH readh; + SDFileSet *pSet; + STsdbMeta *pMeta = pRepo->tsdbMeta; + STsdbCfg * pCfg = REPO_CFG(pRepo); + + if (tsdbInitReadH(&readh, pRepo) < 0) { + return -1; + } + + tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); + + if (CACHE_LAST_NULL_COLUMN(pCfg)) { + for (int i = 1; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + pTable->restoreColumnNum = 0; + pTable->hasRestoreLastColumn = false; + } + } + + while ((pSet = tsdbFSIterNext(&fsiter)) != NULL) { + if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + if (tsdbLoadBlockIdx(&readh) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + for (int i = 1; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + + //tsdbInfo("tsdbRestoreInfo restore vgId:%d,table:%s", REPO_ID(pRepo), pTable->name->data); + + if (tsdbSetReadTable(&readh, pTable) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); + SBlockIdx *pIdx = readh.pBlkIdx; + if (pIdx && lastKey < pIdx->maxKey) { + pTable->lastKey = pIdx->maxKey; + + if (CACHE_LAST_ROW(pCfg) && tsdbRestoreLastRow(pRepo, pTable, &readh, pIdx) != 0) { + tsdbDestroyReadH(&readh); + return -1; + } + } + + // restore NULL columns + if (pIdx && CACHE_LAST_NULL_COLUMN(pCfg) && !pTable->hasRestoreLastColumn) { + if (tsdbRestoreLastColumns(pRepo, pTable, &readh) != 0) { + tsdbDestroyReadH(&readh); + return -1; + } + } + } + } + + tsdbDestroyReadH(&readh); + + if (CACHE_LAST_NULL_COLUMN(pCfg)) { + atomic_store_8(&pRepo->hasCachedLastColumn, 1); + } + + return 0; +} + +int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) { + bool cacheLastRow = false, cacheLastCol = false; + SFSIter fsiter; + SReadH readh; + SDFileSet *pSet; + STsdbMeta *pMeta = pRepo->tsdbMeta; + int tableNum = 0; + int maxTableIdx = 0; + int cacheLastRowTableNum = 0; + int cacheLastColTableNum = 0; + + bool need_free_last_row = CACHE_LAST_ROW(oldCfg) && !CACHE_LAST_ROW(&(pRepo->config)); + bool need_free_last_col = CACHE_LAST_NULL_COLUMN(oldCfg) && !CACHE_LAST_NULL_COLUMN(&(pRepo->config)); + + if (CACHE_LAST_ROW(&(pRepo->config)) || CACHE_LAST_NULL_COLUMN(&(pRepo->config))) { + tsdbInfo("tsdbCacheLastData cache last data since cacheLast option changed"); + cacheLastRow = !CACHE_LAST_ROW(oldCfg) && CACHE_LAST_ROW(&(pRepo->config)); + cacheLastCol = !CACHE_LAST_NULL_COLUMN(oldCfg) && CACHE_LAST_NULL_COLUMN(&(pRepo->config)); + } + + // calc max table idx and table num + for (int i = 1; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + tableNum += 1; + maxTableIdx = i; + if (cacheLastCol) { + pTable->restoreColumnNum = 0; + pTable->hasRestoreLastColumn = false; + } + } + + // if close last option,need to free data + if (need_free_last_row || need_free_last_col) { + if (need_free_last_col) { + atomic_store_8(&pRepo->hasCachedLastColumn, 0); + } + tsdbInfo("free cache last data since cacheLast option changed"); + for (int i = 1; i <= maxTableIdx; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + if (need_free_last_row) { + taosTZfree(pTable->lastRow); + pTable->lastRow = NULL; + } + if (need_free_last_col) { + tsdbFreeLastColumns(pTable); + pTable->hasRestoreLastColumn = false; + } + } + } + + if (!cacheLastRow && !cacheLastCol) { + return 0; + } + + cacheLastRowTableNum = cacheLastRow ? tableNum : 0; + cacheLastColTableNum = cacheLastCol ? tableNum : 0; + + if (tsdbInitReadH(&readh, pRepo) < 0) { + return -1; + } + + tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); + + while ((pSet = tsdbFSIterNext(&fsiter)) != NULL && (cacheLastRowTableNum > 0 || cacheLastColTableNum > 0)) { + if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + if (tsdbLoadBlockIdx(&readh) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + for (int i = 1; i <= maxTableIdx; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + + //tsdbInfo("tsdbRestoreInfo restore vgId:%d,table:%s", REPO_ID(pRepo), pTable->name->data); + + if (tsdbSetReadTable(&readh, pTable) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + SBlockIdx *pIdx = readh.pBlkIdx; + + if (pIdx && cacheLastRowTableNum > 0 && pTable->lastRow == NULL) { + pTable->lastKey = pIdx->maxKey; + + if (tsdbRestoreLastRow(pRepo, pTable, &readh, pIdx) != 0) { + tsdbDestroyReadH(&readh); + return -1; + } + cacheLastRowTableNum -= 1; + } + + // restore NULL columns + if (pIdx && cacheLastColTableNum > 0 && !pTable->hasRestoreLastColumn) { + if (tsdbRestoreLastColumns(pRepo, pTable, &readh) != 0) { + tsdbDestroyReadH(&readh); + return -1; + } + if (pTable->hasRestoreLastColumn) { + cacheLastColTableNum -= 1; + } + } + } + } + + tsdbDestroyReadH(&readh); + + if (cacheLastCol) { + atomic_store_8(&pRepo->hasCachedLastColumn, 1); + } + + return 0; +} +#endif \ No newline at end of file diff --git a/src/tsdb/src/tsdbRead.c b/source/dnode/vnode/tsdb/src/tsdbRead.c similarity index 100% rename from src/tsdb/src/tsdbRead.c rename to source/dnode/vnode/tsdb/src/tsdbRead.c diff --git a/src/tsdb/src/tsdbReadImpl.c b/source/dnode/vnode/tsdb/src/tsdbReadImpl.c similarity index 100% rename from src/tsdb/src/tsdbReadImpl.c rename to source/dnode/vnode/tsdb/src/tsdbReadImpl.c diff --git a/src/tsdb/src/tsdbRowMergeBuf.c b/source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c similarity index 100% rename from src/tsdb/src/tsdbRowMergeBuf.c rename to source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c diff --git a/source/dnode/vnode/tsdb/src/tsdbSMA.c b/source/dnode/vnode/tsdb/src/tsdbSMA.c deleted file mode 100644 index 6dea4a4e57392be988126c579648f39a8270b9bf..0000000000000000000000000000000000000000 --- a/source/dnode/vnode/tsdb/src/tsdbSMA.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/src/tsdb/src/tsdbScan.c b/source/dnode/vnode/tsdb/src/tsdbScan.c similarity index 100% rename from src/tsdb/src/tsdbScan.c rename to source/dnode/vnode/tsdb/src/tsdbScan.c diff --git a/src/tsdb/src/tsdbSync.c b/source/dnode/vnode/tsdb/src/tsdbSync.c similarity index 100% rename from src/tsdb/src/tsdbSync.c rename to source/dnode/vnode/tsdb/src/tsdbSync.c diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 636cf0a9a84453458ec0d06b84231f926a4637b5..027532bbb1d1514604fc8277862e764cec13ec51 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -11,3 +11,4 @@ add_subdirectory(executor) add_subdirectory(planner) add_subdirectory(function) add_subdirectory(qcom) +add_subdirectory(qworker) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 1134c7763a88c22e80ad523cbdb63b84313f1872..e3e84857605733ffab22d8263bc0a18e5353385e 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -605,7 +605,7 @@ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* p SName *name = taosArrayGet(pReq->pTableName, i); STableMeta *pTableMeta = NULL; - snprintf(dbName, sizeof(dbName), "%s.%s", name->acctId, name->dbname); + snprintf(dbName, sizeof(dbName), "%d.%s", name->acctId, name->dbname); CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, dbName, name->tname, &pTableMeta)); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index e14c58d4122f1f830bcac17c2c358ef6632ef15c..29c80ae9ec05453dd23605c39d43fa562c2aa2d9 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -61,8 +61,8 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep2 = htonl(3650); - pReq->minRowsPerFileBlock = htonl(100); - pReq->maxRowsPerFileBlock = htonl(4096); + pReq->minRows = htonl(100); + pReq->maxRows = htonl(4096); pReq->commitTime = htonl(3600); pReq->fsyncPeriod = htonl(3000); pReq->walLevel = 1; diff --git a/source/libs/executor/src/tfilter.c b/source/libs/executor/src/tfilter.c index 48662f34436d05c9c2ef873bb52d6e1aa515f4f3..97dccb5c7b69405f91934362262e339985a5060d 100644 --- a/source/libs/executor/src/tfilter.c +++ b/source/libs/executor/src/tfilter.c @@ -1175,7 +1175,7 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g void* key = NULL; len = 0; - taosHashGetKey((SHashObj *)data, p, &key, &len); + taosHashGetKey(p, &key, &len); void *fdata = NULL; if (IS_VAR_DATA_TYPE(type)) { diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index a258cb834dcbe4a15845c690b706d46b09674c78..2584a847ff2343db27a2f2acaf00a3f68c7aad68 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -18,71 +18,69 @@ #include "index.h" #include "index_fst.h" -#include "tlog.h" -#include "thash.h" #include "taos.h" +#include "thash.h" +#include "tlog.h" #ifdef USE_LUCENE #include #endif - #ifdef __cplusplus extern "C" { #endif -typedef enum {kTypeValue, kTypeDeletion} STermValueType ; +typedef enum { kTypeValue, kTypeDeletion } STermValueType; typedef struct SIndexStat { - int32_t totalAdded; // - int32_t totalDeled; // - int32_t totalUpdated; // - int32_t totalTerms; // - int32_t distinctCol; // distinct column -} SIndexStat; + int32_t totalAdded; // + int32_t totalDeled; // + int32_t totalUpdated; // + int32_t totalTerms; // + int32_t distinctCol; // distinct column +} SIndexStat; struct SIndex { -#ifdef USE_LUCENE - index_t *index; -#endif - void *cache; - void *tindex; - SHashObj *colObj;// < field name, field id> - - int64_t suid; // current super table id, -1 is normal table - int colId; // field id allocated to cache - int32_t cVersion; // current version allocated to cache - - SIndexStat stat; - pthread_mutex_t mtx; -}; +#ifdef USE_LUCENE + index_t* index; +#endif + void* cache; + void* tindex; + SHashObj* colObj; // < field name, field id> + + int64_t suid; // current super table id, -1 is normal table + int colId; // field id allocated to cache + int32_t cVersion; // current version allocated to cache + + SIndexStat stat; + pthread_mutex_t mtx; +}; struct SIndexOpts { -#ifdef USE_LUCENE - void *opts; -#endif +#ifdef USE_LUCENE + void* opts; +#endif #ifdef USE_INVERTED_INDEX - int32_t cacheSize; // MB + int32_t cacheSize; // MB // add cache module later #endif - }; struct SIndexMultiTermQuery { - EIndexOperatorType opera; - SArray *query; + EIndexOperatorType opera; + SArray* query; }; // field and key; typedef struct SIndexTerm { - int64_t suid; - SIndexOperOnColumn operType; // oper type, add/del/update - uint8_t colType; // term data type, str/interger/json - char *colName; - int32_t nColName; - char *colVal; - int32_t nColVal; + int64_t suid; + SIndexOperOnColumn operType; // oper type, add/del/update + uint8_t colType; // term data type, str/interger/json + char* colName; + int32_t nColName; + char* colVal; + int32_t nColVal; } SIndexTerm; typedef struct SIndexTermQuery { @@ -90,15 +88,42 @@ typedef struct SIndexTermQuery { EIndexQueryType qType; } SIndexTermQuery; - - -#define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); }} while(0) -#define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); }} while(0) -#define indexWarn(...) do { if (sDebugFlag & DEBUG_WARN) { taosPrintLog("index WARN ", 255, __VA_ARGS__); }} while(0) -#define indexInfo(...) do { if (sDebugFlag & DEBUG_INFO) { taosPrintLog("index ", 255, __VA_ARGS__); }} while(0) -#define indexDebug(...) do { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); }} while(0) -#define indexTrace(...) do { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); }} while(0) - +#define indexFatal(...) \ + do { \ + if (sDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("index FATAL ", 255, __VA_ARGS__); \ + } \ + } while (0) +#define indexError(...) \ + do { \ + if (sDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("index ERROR ", 255, __VA_ARGS__); \ + } \ + } while (0) +#define indexWarn(...) \ + do { \ + if (sDebugFlag & DEBUG_WARN) { \ + taosPrintLog("index WARN ", 255, __VA_ARGS__); \ + } \ + } while (0) +#define indexInfo(...) \ + do { \ + if (sDebugFlag & DEBUG_INFO) { \ + taosPrintLog("index ", 255, __VA_ARGS__); \ + } \ + } while (0) +#define indexDebug(...) \ + do { \ + if (sDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ + } \ + } while (0) +#define indexTrace(...) \ + do { \ + if (sDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \ + } \ + } while (0) #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 97a7b835f6a5afb348847276ba427a11b2113c83..fb4f478ae99fdc7654536a98d67d5e3d8ac407fa 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -22,8 +22,10 @@ // ----------------- key structure in skiplist --------------------- /* A data row, the format is like below: - * content: |<--totalLen-->|<-- fieldid-->|<--field type -->|<-- value len--->|<-- value -->|<-- uid -->|<--version--->|<-- itermType -->| - * len : |<--int32_t -->|<-- int16_t-->|<-- int8_t --->|<--- int32_t --->|<--valuelen->|<--uint64_t->|<-- int32_t-->|<-- int8_t --->| + * content: |<--totalLen-->|<-- fieldid-->|<--field type-->|<-- value len--->| + * |<-- value -->|<--uid -->|<--version--->|<-- itermType -->| + * len : |<--int32_t -->|<-- int16_t-->|<-- int8_t --->|<--- int32_t --->| + * <--valuelen->|<--uint64_t->| * <-- int32_t-->|<-- int8_t --->| */ #ifdef __cplusplus @@ -31,25 +33,22 @@ extern "C" { #endif typedef struct IndexCache { - T_REF_DECLARE() - SSkipList *skiplist; + T_REF_DECLARE() + SSkipList* skiplist; } IndexCache; +// +IndexCache* indexCacheCreate(); -// -IndexCache *indexCacheCreate(); +void indexCacheDestroy(void* cache); -void indexCacheDestroy(void *cache); +int indexCachePut(void* cache, SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid); -int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid); - -//int indexCacheGet(void *cache, uint64_t *rst); -int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s); +// int indexCacheGet(void *cache, uint64_t *rst); +int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s); #ifdef __cplusplus } #endif - - #endif diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 0dcc25831c6bb3940e19d8541653e47a24736143..3c572787fc4b4fa17f7e870b5af44aa95b3ef115 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -20,30 +20,30 @@ extern "C" { #endif -#include "tarray.h" -#include "index_fst_util.h" -#include "index_fst_registry.h" -#include "index_fst_counting_writer.h" #include "index_fst_automation.h" +#include "index_fst_counting_writer.h" +#include "index_fst_registry.h" +#include "index_fst_util.h" +#include "tarray.h" -#define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a) +#define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a) -typedef struct Fst Fst; -typedef struct FstNode FstNode; +typedef struct Fst Fst; +typedef struct FstNode FstNode; typedef struct StreamWithState StreamWithState; -typedef enum { Included, Excluded, Unbounded} FstBound; +typedef enum { Included, Excluded, Unbounded } FstBound; typedef struct FstBoundWithData { - FstSlice data; + FstSlice data; FstBound type; } FstBoundWithData; typedef struct FstStreamBuilder { - Fst *fst; - AutomationCtx *aut; - FstBoundWithData *min; - FstBoundWithData *max; + Fst* fst; + AutomationCtx* aut; + FstBoundWithData* min; + FstBoundWithData* max; } FstStreamBuilder, FstStreamWithStateBuilder; typedef struct FstRange { @@ -51,17 +51,14 @@ typedef struct FstRange { uint64_t end; } FstRange; +typedef enum { GE, GT, LE, LT } RangeType; +typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State; +typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType; -typedef enum {GE, GT, LE, LT} RangeType; -typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal} State; -typedef enum {Ordered, OutOfOrdered, DuplicateKey} OrderType; - - -FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data); -bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice); -bool fstBoundWithDataIsEmpty(FstBoundWithData *bound); -bool fstBoundWithDataIsIncluded(FstBoundWithData *bound); - +FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data); +bool fstBoundWithDataExceededBy(FstBoundWithData* bound, FstSlice* slice); +bool fstBoundWithDataIsEmpty(FstBoundWithData* bound); +bool fstBoundWithDataIsIncluded(FstBoundWithData* bound); typedef struct FstOutput { bool null; @@ -69,270 +66,263 @@ typedef struct FstOutput { } FstOutput; /* - * + * * UnFinished node and helper function - * TODO: simple function name + * TODO: simple function name */ typedef struct FstUnFinishedNodes { - SArray *stack; // } FstUnFinishedNodes; + SArray* stack; // } FstUnFinishedNodes; } FstUnFinishedNodes; -#define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack) +#define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack) -FstUnFinishedNodes *fstUnFinishedNodesCreate(); -void fstUnFinishedNodesDestroy(FstUnFinishedNodes *node); -void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal); -FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes); -FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr); -FstBuilderNode *fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes); -void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *node, Output out); -void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *node, CompiledAddr addr); -void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *node, FstSlice bs, Output out); -uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs); -uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out); +FstUnFinishedNodes* fstUnFinishedNodesCreate(); +void fstUnFinishedNodesDestroy(FstUnFinishedNodes* node); +void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal); +void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* node, Output out); +void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* node, CompiledAddr addr); +void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* node, FstSlice bs, Output out); +uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes* node, FstSlice bs); +FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes); +FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr); +FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes); +uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstSlice bs, Output in, Output* out); typedef struct FstBuilder { - FstCountingWriter *wrt; // The FST raw data is written directly to `wtr`. - FstUnFinishedNodes *unfinished; // The stack of unfinished nodes - FstRegistry* registry; // A map of finished nodes. - FstSlice last; // The last word added - CompiledAddr lastAddr; // The address of the last compiled node - uint64_t len; // num of keys added + FstCountingWriter* wrt; // The FST raw data is written directly to `wtr`. + FstUnFinishedNodes* unfinished; // The stack of unfinished nodes + FstRegistry* registry; // A map of finished nodes. + FstSlice last; // The last word added + CompiledAddr lastAddr; // The address of the last compiled node + uint64_t len; // num of keys added } FstBuilder; +FstBuilder* fstBuilderCreate(void* w, FstType ty); -FstBuilder *fstBuilderCreate(void *w, FstType ty); - - -void fstBuilderDestroy(FstBuilder *b); -void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in); -bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in); -OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup); -void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate); -CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn); -void* fstBuilerIntoInner(FstBuilder *b); -void fstBuilderFinish(FstBuilder *b); - - - +void fstBuilderDestroy(FstBuilder* b); +void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in); +bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in); +void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate); +void* fstBuilerIntoInner(FstBuilder* b); +void fstBuilderFinish(FstBuilder* b); +OrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup); +CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn); typedef struct FstTransitions { - FstNode *node; - FstRange range; + FstNode* node; + FstRange range; } FstTransitions; -//FstState and relation function +// FstState and relation function typedef struct FstState { - State state; + State state; uint8_t val; } FstState; FstState fstStateCreateFrom(FstSlice* data, CompiledAddr addr); FstState fstStateCreate(State state); -//compile -void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uint8_t inp); -void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTransition *trn); -void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node); +// compile +void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp); +void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn); +void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node); // set_comm_input void fstStateSetCommInput(FstState* state, uint8_t inp); // comm_input -uint8_t fstStateCommInput(FstState* state, bool *null); +uint8_t fstStateCommInput(FstState* state, bool* null); // input_len uint64_t fstStateInputLen(FstState* state); - -// end_addr -uint64_t fstStateEndAddrForOneTransNext(FstState* state, FstSlice *data); -uint64_t fstStateEndAddrForOneTrans(FstState *state, FstSlice *data, PackSizes sizes); -uint64_t fstStateEndAddrForAnyTrans(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans); -// input -uint8_t fstStateInput(FstState *state, FstNode *node); -uint8_t fstStateInputForAnyTrans(FstState *state, FstNode *node, uint64_t i); +// end_addr +uint64_t fstStateEndAddrForOneTransNext(FstState* state, FstSlice* data); +uint64_t fstStateEndAddrForOneTrans(FstState* state, FstSlice* data, PackSizes sizes); +uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans); +// input +uint8_t fstStateInput(FstState* state, FstNode* node); +uint8_t fstStateInputForAnyTrans(FstState* state, FstNode* node, uint64_t i); // trans_addr -CompiledAddr fstStateTransAddr(FstState *state, FstNode *node); -CompiledAddr fstStateTransAddrForAnyTrans(FstState *state, FstNode *node, uint64_t i); +CompiledAddr fstStateTransAddr(FstState* state, FstNode* node); +CompiledAddr fstStateTransAddrForAnyTrans(FstState* state, FstNode* node, uint64_t i); -// sizes -PackSizes fstStateSizes(FstState *state, FstSlice *data); -// Output -Output fstStateOutput(FstState *state, FstNode *node); -Output fstStateOutputForAnyTrans(FstState *state, FstNode *node, uint64_t i); +// sizes +PackSizes fstStateSizes(FstState* state, FstSlice* data); +// Output +Output fstStateOutput(FstState* state, FstNode* node); +Output fstStateOutputForAnyTrans(FstState* state, FstNode* node, uint64_t i); // anyTrans specify function -void fstStateSetFinalState(FstState *state, bool yes); -bool fstStateIsFinalState(FstState *state); -void fstStateSetStateNtrans(FstState *state, uint8_t n); +void fstStateSetFinalState(FstState* state, bool yes); +bool fstStateIsFinalState(FstState* state); +void fstStateSetStateNtrans(FstState* state, uint8_t n); // state_ntrans -uint8_t fstStateStateNtrans(FstState *state, bool *null); -uint64_t fstStateTotalTransSize(FstState *state, uint64_t version, PackSizes size, uint64_t nTrans); -uint64_t fstStateTransIndexSize(FstState *state, uint64_t version, uint64_t nTrans); -uint64_t fstStateNtransLen(FstState *state); -uint64_t fstStateNtrans(FstState *state, FstSlice *slice); -Output fstStateFinalOutput(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans); -uint64_t fstStateFindInput(FstState *state, FstNode *node, uint8_t b, bool *null); - - - -#define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext) +uint8_t fstStateStateNtrans(FstState* state, bool* null); +uint64_t fstStateTotalTransSize(FstState* state, uint64_t version, PackSizes size, uint64_t nTrans); +uint64_t fstStateTransIndexSize(FstState* state, uint64_t version, uint64_t nTrans); +uint64_t fstStateNtransLen(FstState* state); +uint64_t fstStateNtrans(FstState* state, FstSlice* slice); +Output fstStateFinalOutput(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans); +uint64_t fstStateFindInput(FstState* state, FstNode* node, uint8_t b, bool* null); + +#define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext) #define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) #define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans) -#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal) - +#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal) typedef struct FstLastTransition { uint8_t inp; Output out; } FstLastTransition; -/* +/* * FstBuilderNodeUnfinished and helper function - * TODO: simple function name + * TODO: simple function name */ typedef struct FstBuilderNodeUnfinished { - FstBuilderNode *node; - FstLastTransition* last; + FstBuilderNode* node; + FstLastTransition* last; } FstBuilderNodeUnfinished; +void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* node, CompiledAddr addr); - -void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *node, CompiledAddr addr); -void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *node, Output out); +void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* node, Output out); /* - * FstNode and helper function + * FstNode and helper function */ typedef struct FstNode { FstSlice data; - uint64_t version; + uint64_t version; FstState state; - CompiledAddr start; - CompiledAddr end; + CompiledAddr start; + CompiledAddr end; bool isFinal; uint64_t nTrans; PackSizes sizes; - Output finalOutput; + Output finalOutput; } FstNode; -// If this node is final and has a terminal output value, then it is, returned. Otherwise, a zero output is returned +// If this node is final and has a terminal output value, then it is, returned. +// Otherwise, a zero output is returned #define FST_NODE_FINAL_OUTPUT(node) node->finalOutput -// Returns true if and only if this node corresponds to a final or "match", state in the finite state transducer. +// Returns true if and only if this node corresponds to a final or "match", +// state in the finite state transducer. #define FST_NODE_IS_FINAL(node) node->isFinal -// Returns the number of transitions in this node, The maximum number of transitions is 256. +// Returns the number of transitions in this node, The maximum number of +// transitions is 256. #define FST_NODE_LEN(node) node->nTrans // Returns true if and only if this node has zero transitions. #define FST_NODE_IS_EMPTYE(node) (node->nTrans == 0) // Return the address of this node. -#define FST_NODE_ADDR(node) node->start +#define FST_NODE_ADDR(node) node->start + +FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* data); +void fstNodeDestroy(FstNode* fstNode); +FstTransitions fstNodeTransitionIter(FstNode* node); +FstTransitions* fstNodeTransitions(FstNode* node); +bool fstNodeGetTransitionAt(FstNode* node, uint64_t i, FstTransition* res); +bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res); +bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res); -FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *data); -void fstNodeDestroy(FstNode *fstNode); +bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode); -FstTransitions fstNodeTransitionIter(FstNode *node); -FstTransitions* fstNodeTransitions(FstNode *node); -bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *res); -bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res); -bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res); -bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode); -FstSlice fstNodeAsSlice(FstNode *node); +FstSlice fstNodeAsSlice(FstNode* node); -// ops +// ops typedef struct FstIndexedValue { uint64_t index; uint64_t value; } FstIndexedValue; -FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out); -void fstLastTransitionDestroy(FstLastTransition *trn); - +FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out); +void fstLastTransitionDestroy(FstLastTransition* trn); typedef struct FstMeta { uint64_t version; - CompiledAddr rootAddr; + CompiledAddr rootAddr; FstType ty; uint64_t len; uint32_t checkSum; } FstMeta; typedef struct Fst { - FstMeta *meta; - FstSlice *data; // - FstNode *root; // + FstMeta* meta; + FstSlice* data; // + FstNode* root; // } Fst; -// refactor simple function - -Fst* fstCreate(FstSlice *data); -void fstDestroy(Fst *fst); +// refactor simple function -bool fstGet(Fst *fst, FstSlice *b, Output *out); -FstNode* fstGetNode(Fst *fst, CompiledAddr); -FstNode* fstGetRoot(Fst *fst); -FstType fstGetType(Fst *fst); -CompiledAddr fstGetRootAddr(Fst *fst); -Output fstEmptyFinalOutput(Fst *fst, bool *null); -FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx); -FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx); +Fst* fstCreate(FstSlice* data); +void fstDestroy(Fst* fst); -// into stream to expand later -StreamWithState* streamBuilderIntoStream(FstStreamBuilder *sb); +bool fstGet(Fst* fst, FstSlice* b, Output* out); +FstNode* fstGetNode(Fst* fst, CompiledAddr); +FstNode* fstGetRoot(Fst* fst); +FstType fstGetType(Fst* fst); +CompiledAddr fstGetRootAddr(Fst* fst); +Output fstEmptyFinalOutput(Fst* fst, bool* null); +FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx); -bool fstVerify(Fst *fst); +FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx); +// into stream to expand later +StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb); +bool fstVerify(Fst* fst); -//refactor this function -bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); +// refactor this function +bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr); typedef struct StreamState { - FstNode *node; + FstNode* node; uint64_t trans; - FstOutput out; - void *autState; -} StreamState; + FstOutput out; + void* autState; +} StreamState; -void streamStateDestroy(void *s); +void streamStateDestroy(void* s); typedef struct StreamWithState { - Fst *fst; - AutomationCtx *aut; - SArray *inp; - FstOutput emptyOutput; - SArray *stack; // - FstBoundWithData *endAt; + Fst* fst; + AutomationCtx* aut; + SArray* inp; + FstOutput emptyOutput; + SArray* stack; // + FstBoundWithData* endAt; } StreamWithState; typedef struct StreamWithStateResult { - FstSlice data; + FstSlice data; FstOutput out; - void *state; + void* state; } StreamWithStateResult; -StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state); -void swsResultDestroy(StreamWithStateResult *result); +StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state); +void swsResultDestroy(StreamWithStateResult* result); -typedef void* (*StreamCallback)(void *); -StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) ; -void streamWithStateDestroy(StreamWithState *sws); -bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min); +typedef void* (*StreamCallback)(void*); +StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max); -StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); +void streamWithStateDestroy(StreamWithState* sws); -FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut); -// set up bound range -// refator, simple code by marco +bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min); -FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type); +StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback); + +FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut); +// set up bound range +// refator, simple code by marco +FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index 8050b85b0878e378bbe40098a4e2fbd0b1535b8a..c7269eda0f30a845c5b9a1810a5a4aea4baa56eb 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -21,62 +21,56 @@ extern "C" { #include "index_fst_util.h" - typedef struct AutomationCtx AutomationCtx; -typedef enum AutomationType { - AUTOMATION_PREFIX, - AUTMMATION_MATCH -} AutomationType; +typedef enum AutomationType { AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType; typedef struct StartWith { - AutomationCtx *autoSelf; + AutomationCtx* autoSelf; } StartWith; typedef struct Complement { - AutomationCtx *autoSelf; + AutomationCtx* autoSelf; } Complement; -// automation +// automation typedef struct AutomationCtx { - AutomationType type; - void *stdata; - char *data; + AutomationType type; + void* stdata; + char* data; } AutomationCtx; - -typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY} ValueType; -typedef enum StartWithStateKind { Done, Running } StartWithStateKind; +typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType; +typedef enum StartWithStateKind { Done, Running } StartWithStateKind; typedef struct StartWithStateValue { StartWithStateKind kind; - ValueType type; + ValueType type; union { - int val; - char *ptr; - SArray *arr; + int val; + char* ptr; + SArray* arr; // add more type - } ; + }; } StartWithStateValue; -StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val); -StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv); -void startWithStateValueDestroy(void *sv); - +StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val); +StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv); +void startWithStateValueDestroy(void* sv); typedef struct AutomationFunc { - void* (*start)(AutomationCtx *ctx) ; - bool (*isMatch)(AutomationCtx *ctx, void *); - bool (*canMatch)(AutomationCtx *ctx, void *data); - bool (*willAlwaysMatch)(AutomationCtx *ctx, void *state); - void* (*accept)(AutomationCtx *ctx, void *state, uint8_t byte); - void* (*acceptEof)(AutomationCtx *ct, void *state); -} AutomationFunc; - -AutomationCtx *automCtxCreate(void *data, AutomationType atype); -void automCtxDestroy(AutomationCtx *ctx); - -extern AutomationFunc automFuncs[]; + void* (*start)(AutomationCtx* ctx); + bool (*isMatch)(AutomationCtx* ctx, void*); + bool (*canMatch)(AutomationCtx* ctx, void* data); + bool (*willAlwaysMatch)(AutomationCtx* ctx, void* state); + void* (*accept)(AutomationCtx* ctx, void* state, uint8_t byte); + void* (*acceptEof)(AutomationCtx* ct, void* state); +} AutomationFunc; + +AutomationCtx* automCtxCreate(void* data, AutomationType atype); +void automCtxDestroy(AutomationCtx* ctx); + +extern AutomationFunc automFuncs[]; #ifdef __cplusplus } #endif diff --git a/source/libs/index/inc/index_fst_common.h b/source/libs/index/inc/index_fst_common.h index 9c802faa33f52982a34e9d236956ec2e71c845f0..8335e437fb148b5f83f3b0db8fc2c6f63a4caa72 100644 --- a/source/libs/index/inc/index_fst_common.h +++ b/source/libs/index/inc/index_fst_common.h @@ -1,9 +1,9 @@ #ifndef __INDEX_FST_COMM_H__ #define __INDEX_FST_COMM_H__ - +#include "tutil.h" extern const uint8_t COMMON_INPUTS[]; -extern char const COMMON_INPUTS_INV[]; +extern const char COMMON_INPUTS_INV[]; #ifdef __cplusplus extern "C" { diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index ea090389bba1ff1a12b91ebcac2d15827916286a..ba0bd52e10280cd0139a6dc852b61c7f75b91a77 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -22,57 +22,60 @@ extern "C" { #include "tfile.h" - -#define DefaultMem 1024*1024 +#define DefaultMem 1024 * 1024 static char tmpFile[] = "./index"; -typedef enum WriterType {TMemory, TFile} WriterType; +typedef enum WriterType { TMemory, TFile } WriterType; typedef struct WriterCtx { - int (*write)(struct WriterCtx *ctx, uint8_t *buf, int len); - int (*read)(struct WriterCtx *ctx, uint8_t *buf, int len); - int (*flush)(struct WriterCtx *ctx); - WriterType type; + int (*write)(struct WriterCtx* ctx, uint8_t* buf, int len); + int (*read)(struct WriterCtx* ctx, uint8_t* buf, int len); + int (*flush)(struct WriterCtx* ctx); + int (*readFrom)(struct WriterCtx* ctx, uint8_t* buf, int len, int32_t offset); + WriterType type; union { - int fd; - void *mem; + struct { + int fd; + bool readOnly; + } file; + struct { + int32_t capa; + char* buf; + } mem; }; int32_t offset; int32_t limit; } WriterCtx; -static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len); -static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len); -static int writeCtxDoFlush(WriterCtx *ctx); +static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len); +static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len); +static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset); +static int writeCtxDoFlush(WriterCtx* ctx); -WriterCtx* writerCtxCreate(WriterType type, bool readOnly); -void writerCtxDestroy(WriterCtx *w); +WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity); +void writerCtxDestroy(WriterCtx* w); typedef uint32_t CheckSummer; - typedef struct FstCountingWriter { - void* wrt; // wrap any writer that counts and checksum bytes written - uint64_t count; - CheckSummer summer; + void* wrt; // wrap any writer that counts and checksum bytes written + uint64_t count; + CheckSummer summer; } FstCountingWriter; -int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len); - -int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len); - -int fstCountingWriterFlush(FstCountingWriter *write); +int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len); +int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len); -uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write); +int fstCountingWriterFlush(FstCountingWriter* write); -FstCountingWriter *fstCountingWriterCreate(void *wtr, bool readOnly); -void fstCountingWriterDestroy(FstCountingWriter *w); +uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write); +FstCountingWriter* fstCountingWriterCreate(void* wtr); +void fstCountingWriterDestroy(FstCountingWriter* w); -void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes); -uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n); - +void fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes); +uint8_t fstCountingWriterPackUint(FstCountingWriter* writer, uint64_t n); #define FST_WRITER_COUNT(writer) (writer->count) #define FST_WRITER_INTER_WRITER(writer) (writer->wtr) @@ -83,5 +86,3 @@ uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n); #endif #endif - - diff --git a/source/libs/index/inc/index_fst_node.h b/source/libs/index/inc/index_fst_node.h index 87eb7cb746e8650f45014d4a7a354de71b32bba6..32de7ea6211d4ab449fc6ef0d0fe8355ff380299 100644 --- a/source/libs/index/inc/index_fst_node.h +++ b/source/libs/index/inc/index_fst_node.h @@ -20,35 +20,36 @@ extern "C" { #endif -#include "index_fst_util.h" #include "index_fst_counting_writer.h" +#include "index_fst_util.h" -#define FST_BUILDER_NODE_IS_FINAL(bn) (bn->isFinal) -#define FST_BUILDER_NODE_TRANS_ISEMPTY(bn) (taosArrayGetSize(bn->trans) == 0) +#define FST_BUILDER_NODE_IS_FINAL(bn) (bn->isFinal) +#define FST_BUILDER_NODE_TRANS_ISEMPTY(bn) (taosArrayGetSize(bn->trans) == 0) #define FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn) (bn->finalOutput == 0) typedef struct FstTransition { - uint8_t inp; //The byte input associated with this transition. - Output out; //The output associated with this transition - CompiledAddr addr; //The address of the node that this transition points to + uint8_t inp; // The byte input associated with this transition. + Output out; // The output associated with this transition + CompiledAddr addr; // The address of the node that this transition points to } FstTransition; typedef struct FstBuilderNode { - bool isFinal; - Output finalOutput; - SArray *trans; // -} FstBuilderNode; + bool isFinal; + Output finalOutput; + SArray* trans; // +} FstBuilderNode; -FstBuilderNode *fstBuilderNodeDefault(); +FstBuilderNode* fstBuilderNodeDefault(); -FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src); +FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src); -void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src); +void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src); -//bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); -bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2); +// bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, +// CompiledAddr lastAddr, CompiledAddr startAddr); +bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2); -void fstBuilderNodeDestroy(FstBuilderNode *node); +void fstBuilderNodeDestroy(FstBuilderNode* node); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_fst_registry.h b/source/libs/index/inc/index_fst_registry.h index 1d89e57e525ea4b5b30242481b1ddad962a954e9..f94a21d8dd7fd6887f531793af74f2d5023f66b7 100644 --- a/source/libs/index/inc/index_fst_registry.h +++ b/source/libs/index/inc/index_fst_registry.h @@ -19,49 +19,48 @@ extern "C" { #endif +#include "index_fst_node.h" #include "index_fst_util.h" #include "tarray.h" -#include "index_fst_node.h" typedef struct FstRegistryCell { - CompiledAddr addr; - FstBuilderNode *node; + CompiledAddr addr; + FstBuilderNode* node; } FstRegistryCell; #define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS) -#define FST_REGISTRY_CELL_INSERT(cell, tAddr) do {cell->addr = tAddr;} while(0) - +#define FST_REGISTRY_CELL_INSERT(cell, tAddr) \ + do { \ + cell->addr = tAddr; \ + } while (0) -//typedef struct FstRegistryCache { -// SArray *cells; +// typedef struct FstRegistryCache { +// SArray *cells; // uint32_t start; // uint32_t end; //} FstRegistryCache; -typedef enum {FOUND, NOTFOUND, REJECTED} FstRegistryEntryState; +typedef enum { FOUND, NOTFOUND, REJECTED } FstRegistryEntryState; typedef struct FstRegistryEntry { FstRegistryEntryState state; - CompiledAddr addr; - FstRegistryCell *cell; -} FstRegistryEntry; + CompiledAddr addr; + FstRegistryCell* cell; +} FstRegistryEntry; - - -// Registry relation function +// Registry relation function typedef struct FstRegistry { - SArray *table; // - uint64_t tableSize; // num of rows - uint64_t mruSize; // num of columns -} FstRegistry; + SArray* table; // + uint64_t tableSize; // num of rows + uint64_t mruSize; // num of columns +} FstRegistry; -// +// FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize); -void fstRegistryDestroy(FstRegistry *registry); - +void fstRegistryDestroy(FstRegistry* registry); -FstRegistryEntry* fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode); -void fstRegistryEntryDestroy(FstRegistryEntry *entry); +FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNode); +void fstRegistryEntryDestroy(FstRegistryEntry* entry); #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_fst_util.h b/source/libs/index/inc/index_fst_util.h index 24b2508678e4664704ffe4425f779d84a876c296..71a378584d401e966d7b2966ff215916c40c8232 100644 --- a/source/libs/index/inc/index_fst_util.h +++ b/source/libs/index/inc/index_fst_util.h @@ -13,7 +13,6 @@ * along with this program. If not, see . */ - #ifndef __INDEX_FST_UTIL_H__ #define __INDEX_FST_UTIL_H__ @@ -21,16 +20,15 @@ extern "C" { #endif -#include "tarray.h" #include "index_fst_common.h" +#include "tarray.h" typedef uint64_t FstType; -typedef uint64_t CompiledAddr; -typedef uint64_t Output; -typedef uint8_t PackSizes; +typedef uint64_t CompiledAddr; +typedef uint64_t Output; +typedef uint8_t PackSizes; - -//A sentinel value used to indicate an empty final state +// A sentinel value used to indicate an empty final state extern const CompiledAddr EMPTY_ADDRESS; /// A sentinel value used to indicate an invalid state. extern const CompiledAddr NONE_ADDRESS; @@ -38,9 +36,9 @@ extern const CompiledAddr NONE_ADDRESS; // This version number is written to every finite state transducer created by // this version When a finite state transducer is read, its version number is // checked against this value. -extern const uint64_t VERSION; -// The threshold (in number of transitions) at which an index is created for -// a node's transitions. This speeds up lookup time at the expense of FST size +extern const uint64_t VERSION; +// The threshold (in number of transitions) at which an index is created for +// a node's transitions. This speeds up lookup time at the expense of FST size extern const uint64_t TRANS_INDEX_THRESHOLD; // high 4 bits is transition address packed size. @@ -48,73 +46,75 @@ extern const uint64_t TRANS_INDEX_THRESHOLD; // // `0` is a legal value which means there are no transitions/outputs - -#define FST_SET_TRANSITION_PACK_SIZE(v, sz) do {v = (v & 0b00001111) | (sz << 4); } while(0) -#define FST_GET_TRANSITION_PACK_SIZE(v) (((v) & 0b11110000) >> 4) -#define FST_SET_OUTPUT_PACK_SIZE(v, sz) do { v = (v & 0b11110000) | sz; } while(0) -#define FST_GET_OUTPUT_PACK_SIZE(v) ((v) & 0b00001111) - -#define COMMON_INPUT(idx) COMMON_INPUTS_INV[(idx) - 1] - -#define COMMON_INDEX(v, max, val) do { \ - val = ((uint16_t)COMMON_INPUTS[v] + 1)%256; \ - val = val > max ? 0: val; \ -} while(0) - - -//uint8_t commonInput(uint8_t idx); -//uint8_t commonIdx(uint8_t v, uint8_t max); - -uint8_t packSize(uint64_t n); -uint64_t unpackUint64(uint8_t *ch, uint8_t sz); +#define FST_SET_TRANSITION_PACK_SIZE(v, sz) \ + do { \ + v = (v & 0b00001111) | (sz << 4); \ + } while (0) +#define FST_GET_TRANSITION_PACK_SIZE(v) (((v)&0b11110000) >> 4) +#define FST_SET_OUTPUT_PACK_SIZE(v, sz) \ + do { \ + v = (v & 0b11110000) | sz; \ + } while (0) +#define FST_GET_OUTPUT_PACK_SIZE(v) ((v)&0b00001111) + +#define COMMON_INPUT(idx) COMMON_INPUTS_INV[(idx)-1] + +#define COMMON_INDEX(v, max, val) \ + do { \ + val = ((uint16_t)COMMON_INPUTS[v] + 1) % 256; \ + val = val > max ? 0 : val; \ + } while (0) + +// uint8_t commonInput(uint8_t idx); +// uint8_t commonIdx(uint8_t v, uint8_t max); + +uint8_t packSize(uint64_t n); +uint64_t unpackUint64(uint8_t* ch, uint8_t sz); uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr); -CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr); - +CompiledAddr unpackDelta(char* data, uint64_t len, uint64_t nodeAddr); typedef struct FstString { - uint8_t *data; + uint8_t* data; uint32_t len; - int32_t ref; + int32_t ref; } FstString; typedef struct FstSlice { - FstString *str; - int32_t start; - int32_t end; + FstString* str; + int32_t start; + int32_t end; } FstSlice; -FstSlice fstSliceCreate(uint8_t *data, uint64_t len); -FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end); -FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end); -bool fstSliceIsEmpty(FstSlice *s); -int fstSliceCompare(FstSlice *s1, FstSlice *s2); -void fstSliceDestroy(FstSlice *s); -uint8_t *fstSliceData(FstSlice *s, int32_t *sz); +FstSlice fstSliceCreate(uint8_t* data, uint64_t len); +FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end); +FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end); +bool fstSliceIsEmpty(FstSlice* s); +int fstSliceCompare(FstSlice* s1, FstSlice* s2); +void fstSliceDestroy(FstSlice* s); +uint8_t* fstSliceData(FstSlice* s, int32_t* sz); #define FST_SLICE_LEN(s) (s->end - s->start + 1) -//// stack +//// stack // -//typedef (*StackFreeElemFn)(void *elem); +// typedef (*StackFreeElemFn)(void *elem); // -//typedef struct FstStack { -// void *first; -// void *end; -// size_t elemSize; +// typedef struct FstStack { +// void *first; +// void *end; +// size_t elemSize; // size_t nElem; // StackFreeElemFn fn; //} FstStack; // // -//FstStack* fstStackCreate(size_t elemSize, stackFreeElem); -//void *fstStackPush(FstStack *s, void *elem); -//void *fstStackTop(FstStack *s); -//size_t fstStackLen(FstStack *s); -//void fstStackDestory(FstStack *); +// FstStack* fstStackCreate(size_t elemSize, stackFreeElem); +// void *fstStackPush(FstStack *s, void *elem); +// void *fstStackTop(FstStack *s); +// size_t fstStackLen(FstStack *s); +// void fstStackDestory(FstStack *); // - - #ifdef __cplusplus } #endif diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index c3f4bd25e5e6a792120f1ac8880bfbc57915dfef..b19b887e9f86a482634b8c0877c1e4c079ac9f44 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -17,30 +17,106 @@ #include "index.h" #include "indexInt.h" +#include "index_fst.h" +#include "index_fst_counting_writer.h" +#include "index_tfile.h" #include "tlockfree.h" -#include "tskiplist.h" #ifdef __cplusplus extern "C" { #endif +// tfile header content +// |<---suid--->|<---version--->|<-------colName------>|<---type-->|<--fstOffset->| +// |<-uint64_t->|<---int32_t--->|<--TSDB_COL_NAME_LEN-->|<-uint8_t->|<---int32_t-->| + +#pragma pack(push, 1) +typedef struct TFileHeader { + uint64_t suid; + int32_t version; + char colName[TSDB_COL_NAME_LEN]; // + uint8_t colType; + int32_t fstOffset; +} TFileHeader; +#pragma pack(pop) + +#define TFILE_HEADER_SIZE (sizeof(TFileHeader)) +#define TFILE_HEADER_NO_FST (TFILE_HEADER_SIZE - sizeof(int32_t)) +//#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t)) + +typedef struct TFileCacheKey { + uint64_t suid; + uint8_t colType; + char* colName; + int32_t nColName; +} TFileCacheKey; + +// table cache +// refactor to LRU cache later +typedef struct TFileCache { + SHashObj* tableCache; + int16_t capacity; + // add more param +} TFileCache; + +typedef struct TFileWriter { + FstBuilder* fb; + WriterCtx* ctx; + TFileHeader header; + uint32_t offset; +} TFileWriter; + +// multi reader and single write +typedef struct TFileReader { + T_REF_DECLARE() + Fst* fst; + WriterCtx* ctx; + TFileHeader header; +} TFileReader; + typedef struct IndexTFile { - T_REF_DECLARE() + char* path; + TFileCache* cache; + TFileWriter* tw; } IndexTFile; +typedef struct TFileWriterOpt { + uint64_t suid; + int8_t colType; + char* colName; + int32_t nColName; + int32_t version; +} TFileWriterOpt; + +typedef struct TFileReaderOpt { + uint64_t suid; + char* colName; + int32_t nColName; +} TFileReaderOpt; +// tfile cache, manage tindex reader +TFileCache* tfileCacheCreate(const char* path); +void tfileCacheDestroy(TFileCache* tcache); +TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key); +void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader); -IndexTFile *indexTFileCreate(); +TFileReader* tfileReaderCreate(WriterCtx* ctx); +void tfileReaderDestroy(TFileReader* reader); +int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result); +TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header); +void tfileWriterDestroy(TFileWriter* tw); +int tfileWriterPut(TFileWriter* tw, void* data); +int tfileWriterFinish(TFileWriter* tw); -int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result); +// +IndexTFile* indexTFileCreate(const char* path); +int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid); +int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result); #ifdef __cplusplus } - #endif - - #endif diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h new file mode 100644 index 0000000000000000000000000000000000000000..21c5ca155b3bb252429be2dd6c30a344694b168a --- /dev/null +++ b/source/libs/index/inc/index_util.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef __INDEX_UTIL_H__ +#define __INDEX_UTIL_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ + do { \ + memcpy((void*)buf, (void*)(&key->mem), sizeof(key->mem)); \ + buf += sizeof(key->mem); \ + } while (0) + +#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ + do { \ + memcpy((void*)buf, (void*)key->mem, len); \ + buf += len; \ + } while (0) + +#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ + do { \ + type c = var; \ + assert(sizeof(var) == sizeof(type)); \ + memcpy((void*)buf, (void*)&c, sizeof(c)); \ + buf += sizeof(c); \ + } while (0) + +#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ + do { \ + memcpy((void*)buf, (void*)var, len); \ + buf += len; \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index ec83e84a3b22541fb1ba7906551d94a906917ea0..e88cecfd623c86d39ae28c9906c976f60c27927f 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -23,138 +23,146 @@ #include "lucene++/Lucene_c.h" #endif -static int uidCompare(const void *a, const void *b) { - uint64_t u1 = *(uint64_t *)a; - uint64_t u2 = *(uint64_t *)b; - if (u1 == u2) { return 0; } - else { return u1 < u2 ? -1 : 1; } +static int uidCompare(const void* a, const void* b) { + uint64_t u1 = *(uint64_t*)a; + uint64_t u2 = *(uint64_t*)b; + if (u1 == u2) { + return 0; + } else { + return u1 < u2 ? -1 : 1; + } } typedef struct SIdxColInfo { - int colId; // generated by index internal + int colId; // generated by index internal int cVersion; -} SIdxColInfo; +} SIdxColInfo; static pthread_once_t isInit = PTHREAD_ONCE_INIT; -static void indexInit(); +static void indexInit(); +static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result); +static int indexMergeCacheIntoTindex(SIndex* sIdx); -static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *term, SArray **result); -static int indexMergeCacheIntoTindex(SIndex *sIdx); +static void indexInterResultsDestroy(SArray* results); +static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* finalResult); -static void indexInterResultsDestroy(SArray *results); -static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *finalResult); - -int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) { +int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { pthread_once(&isInit, indexInit); - SIndex *sIdx = calloc(1, sizeof(SIndex)); - if (sIdx == NULL) { return -1; } + SIndex* sIdx = calloc(1, sizeof(SIndex)); + if (sIdx == NULL) { + return -1; + } -#ifdef USE_LUCENE - index_t *index = index_open(path); +#ifdef USE_LUCENE + index_t* index = index_open(path); sIdx->index = index; #endif - sIdx->cache = (void*)indexCacheCreate(); - sIdx->tindex = NULL; - sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - sIdx->colId = 1; - sIdx->cVersion = 1; +#ifdef USE_INVERTED_INDEX + sIdx->cache = (void*)indexCacheCreate(); + sIdx->tindex = NULL; + sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + sIdx->colId = 1; + sIdx->cVersion = 1; pthread_mutex_init(&sIdx->mtx, NULL); - *index = sIdx; - return 0; + *index = sIdx; + return 0; +#endif + + *index = NULL; + return -1; } -void indexClose(SIndex *sIdx) { -#ifdef USE_LUCENE - index_close(sIdex->index); +void indexClose(SIndex* sIdx) { +#ifdef USE_LUCENE + index_close(sIdex->index); sIdx->index = NULL; #endif #ifdef USE_INVERTED_INDEX indexCacheDestroy(sIdx->cache); - taosHashCleanup(sIdx->colObj); + taosHashCleanup(sIdx->colObj); pthread_mutex_destroy(&sIdx->mtx); #endif - free(sIdx); + free(sIdx); return; } -int indexPut(SIndex *index, SIndexMultiTerm * fVals, uint64_t uid) { - -#ifdef USE_LUCENE - index_document_t *doc = index_document_create(); - - char buf[16] = {0}; - sprintf(buf, "%d", uid); - - for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm *p = taosArrayGetP(fVals, i); - index_document_add(doc, (const char *)(p->key), p->nKey, (const char *)(p->val), p->nVal, 1); - } - index_document_add(doc, NULL, 0, buf, strlen(buf), 0); +int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { +#ifdef USE_LUCENE + index_document_t* doc = index_document_create(); + + char buf[16] = {0}; + sprintf(buf, "%d", uid); + + for (int i = 0; i < taosArrayGetSize(fVals); i++) { + SIndexTerm* p = taosArrayGetP(fVals, i); + index_document_add(doc, (const char*)(p->key), p->nKey, (const char*)(p->val), p->nVal, 1); + } + index_document_add(doc, NULL, 0, buf, strlen(buf), 0); - index_put(index->index, doc); - index_document_destroy(doc); + index_put(index->index, doc); + index_document_destroy(doc); #endif #ifdef USE_INVERTED_INDEX - - //TODO(yihao): reduce the lock range - pthread_mutex_lock(&index->mtx); + + // TODO(yihao): reduce the lock range + pthread_mutex_lock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm *p = taosArrayGetP(fVals, i); - SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); + SIndexTerm* p = taosArrayGetP(fVals, i); + SIdxColInfo* fi = taosHashGet(index->colObj, p->colName, p->nColName); if (fi == NULL) { SIdxColInfo tfi = {.colId = index->colId}; - index->cVersion++; + index->cVersion++; index->colId++; - taosHashPut(index->colObj, p->colName, p->nColName, &tfi, sizeof(tfi)); + taosHashPut(index->colObj, p->colName, p->nColName, &tfi, sizeof(tfi)); } else { - //TODO, del + // TODO, del } - } + } pthread_mutex_unlock(&index->mtx); for (int i = 0; i < taosArrayGetSize(fVals); i++) { - SIndexTerm *p = taosArrayGetP(fVals, i); - SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName); - assert(fi != NULL); - int32_t colId = fi->colId; + SIndexTerm* p = taosArrayGetP(fVals, i); + SIdxColInfo* fi = taosHashGet(index->colObj, p->colName, p->nColName); + assert(fi != NULL); + int32_t colId = fi->colId; int32_t version = index->cVersion; - int ret = indexCachePut(index->cache, p, colId, version, uid); + int ret = indexCachePut(index->cache, p, colId, version, uid); if (ret != 0) { - return ret; + return ret; } } #endif return 0; } -int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) { -#ifdef USE_LUCENE - EIndexOperatorType opera = multiQuerys->opera; +int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) { +#ifdef USE_LUCENE + EIndexOperatorType opera = multiQuerys->opera; - int nQuery = taosArrayGetSize(multiQuerys->query); - char **fields = malloc(sizeof(char *) * nQuery); - char **keys = malloc(sizeof(char *) * nQuery); - int *types = malloc(sizeof(int) * nQuery); + int nQuery = taosArrayGetSize(multiQuerys->query); + char** fields = malloc(sizeof(char*) * nQuery); + char** keys = malloc(sizeof(char*) * nQuery); + int* types = malloc(sizeof(int) * nQuery); for (int i = 0; i < nQuery; i++) { - SIndexTermQuery *p = taosArrayGet(multiQuerys->query, i); - SIndexTerm *term = p->field_value; - - fields[i] = calloc(1, term->nKey + 1); - keys[i] = calloc(1, term->nVal + 1); - - memcpy(fields[i], term->key, term->nKey); - memcpy(keys[i], term->val, term->nVal); - types[i] = (int)(p->type); - } - int *tResult = NULL; - int tsz= 0; - index_multi_search(index->index, (const char **)fields, (const char **)keys, types, nQuery, opera, &tResult, &tsz); - + SIndexTermQuery* p = taosArrayGet(multiQuerys->query, i); + SIndexTerm* term = p->field_value; + + fields[i] = calloc(1, term->nKey + 1); + keys[i] = calloc(1, term->nVal + 1); + + memcpy(fields[i], term->key, term->nKey); + memcpy(keys[i], term->val, term->nVal); + types[i] = (int)(p->type); + } + int* tResult = NULL; + int tsz = 0; + index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz); + for (int i = 0; i < tsz; i++) { taosArrayPush(result, &tResult[i]); } @@ -169,148 +177,153 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result #endif #ifdef USE_INVERTED_INDEX - EIndexOperatorType opera = multiQuerys->opera; // relation of querys - - SArray *interResults = taosArrayInit(4, POINTER_BYTES); - int nQuery = taosArrayGetSize(multiQuerys->query); + EIndexOperatorType opera = multiQuerys->opera; // relation of querys + + SArray* interResults = taosArrayInit(4, POINTER_BYTES); + int nQuery = taosArrayGetSize(multiQuerys->query); for (size_t i = 0; i < nQuery; i++) { - SIndexTermQuery *qTerm = taosArrayGet(multiQuerys->query, i); - SArray *tResult = NULL; - indexTermSearch(index, qTerm, &tResult); - taosArrayPush(interResults, (void *)&tResult); - } + SIndexTermQuery* qTerm = taosArrayGet(multiQuerys->query, i); + SArray* tResult = NULL; + indexTermSearch(index, qTerm, &tResult); + taosArrayPush(interResults, (void*)&tResult); + } indexMergeFinalResults(interResults, opera, result); indexInterResultsDestroy(interResults); - + #endif return 1; } - - -int indexDelete(SIndex *index, SIndexMultiTermQuery *query) { +int indexDelete(SIndex* index, SIndexMultiTermQuery* query) { #ifdef USE_INVERTED_INDEX #endif - + return 1; } -int indexRebuild(SIndex *index, SIndexOpts *opts) { +int indexRebuild(SIndex* index, SIndexOpts* opts){ #ifdef USE_INVERTED_INDEX #endif } - -SIndexOpts *indexOptsCreate() { -#ifdef USE_LUCENE +SIndexOpts* indexOptsCreate() { +#ifdef USE_LUCENE #endif -return NULL; + return NULL; } -void indexOptsDestroy(SIndexOpts *opts) { -#ifdef USE_LUCENE +void indexOptsDestroy(SIndexOpts* opts){ +#ifdef USE_LUCENE #endif -} -/* - * @param: oper - * -*/ - -SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) { - SIndexMultiTermQuery *p = (SIndexMultiTermQuery *)malloc(sizeof(SIndexMultiTermQuery)); - if (p == NULL) { return NULL; } - p->opera = opera; - p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); +} /* + * @param: oper + * + */ + +SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { + SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); + if (p == NULL) { + return NULL; + } + p->opera = opera; + p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); return p; } -void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) { +void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) { for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) { - SIndexTermQuery *p = (SIndexTermQuery *)taosArrayGet(pQuery->query, i); + SIndexTermQuery* p = (SIndexTermQuery*)taosArrayGet(pQuery->query, i); indexTermDestroy(p->term); } - taosArrayDestroy(pQuery->query); + taosArrayDestroy(pQuery->query); free(pQuery); }; -int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType){ - SIndexTermQuery q = {.qType = qType, .term = term}; +int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EIndexQueryType qType) { + SIndexTermQuery q = {.qType = qType, .term = term}; taosArrayPush(pQuery->query, &q); return 0; } +SIndexTerm* indexTermCreate(int64_t suid, + SIndexOperOnColumn oper, + uint8_t colType, + const char* colName, + int32_t nColName, + const char* colVal, + int32_t nColVal) { + SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); + if (t == NULL) { + return NULL; + } -SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, int32_t nColName, const char *colVal, int32_t nColVal) { - SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm))); - if (t == NULL) { return NULL; } - - t->suid = suid; - t->operType= oper; + t->suid = suid; + t->operType = oper; t->colType = colType; - t->colName = (char *)calloc(1, nColName + 1); + t->colName = (char*)calloc(1, nColName + 1); memcpy(t->colName, colName, nColName); t->nColName = nColName; - t->colVal = (char *)calloc(1, nColVal + 1); + t->colVal = (char*)calloc(1, nColVal + 1); memcpy(t->colVal, colVal, nColVal); t->nColVal = nColVal; return t; } -void indexTermDestroy(SIndexTerm *p) { +void indexTermDestroy(SIndexTerm* p) { free(p->colName); free(p->colVal); free(p); -} +} -SIndexMultiTerm *indexMultiTermCreate() { - return taosArrayInit(4, sizeof(SIndexTerm *)); +SIndexMultiTerm* indexMultiTermCreate() { + return taosArrayInit(4, sizeof(SIndexTerm*)); } -int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) { - taosArrayPush(terms, &term); - return 0; +int indexMultiTermAdd(SIndexMultiTerm* terms, SIndexTerm* term) { + taosArrayPush(terms, &term); + return 0; } -void indexMultiTermDestroy(SIndexMultiTerm *terms) { +void indexMultiTermDestroy(SIndexMultiTerm* terms) { for (int32_t i = 0; i < taosArrayGetSize(terms); i++) { - SIndexTerm *p = taosArrayGetP(terms, i); + SIndexTerm* p = taosArrayGetP(terms, i); indexTermDestroy(p); } taosArrayDestroy(terms); } void indexInit() { - //do nothing + // do nothing } -static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result) { - int32_t version = -1; - int16_t colId = -1; - SIdxColInfo *colInfo = NULL; +static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result) { + int32_t version = -1; + int16_t colId = -1; + SIdxColInfo* colInfo = NULL; - SIndexTerm *term = query->term; - const char *colName = term->colName; - int32_t nColName = term->nColName; + SIndexTerm* term = query->term; + const char* colName = term->colName; + int32_t nColName = term->nColName; - pthread_mutex_lock(&sIdx->mtx); - colInfo = taosHashGet(sIdx->colObj, colName, nColName); + pthread_mutex_lock(&sIdx->mtx); + colInfo = taosHashGet(sIdx->colObj, colName, nColName); if (colInfo == NULL) { - pthread_mutex_unlock(&sIdx->mtx); - return -1; + pthread_mutex_unlock(&sIdx->mtx); + return -1; } - colId = colInfo->colId; + colId = colInfo->colId; version = colInfo->cVersion; - pthread_mutex_unlock(&sIdx->mtx); - + pthread_mutex_unlock(&sIdx->mtx); + *result = taosArrayInit(4, sizeof(uint64_t)); - //TODO: iterator mem and tidex - STermValueType s; + // TODO: iterator mem and tidex + STermValueType s; if (0 == indexCacheSearch(sIdx->cache, query, colId, version, *result, &s)) { if (s == kTypeDeletion) { indexInfo("col: %s already drop by other opera", term->colName); - // coloum already drop by other oper, no need to query tindex + // coloum already drop by other oper, no need to query tindex return 0; } else { if (0 != indexTFileSearch(sIdx->tindex, query, *result)) { - indexError("corrupt at index(TFile) col:%s val: %s", term->colName, term->colVal); - return -1; - } + indexError("corrupt at index(TFile) col:%s val: %s", term->colName, term->colVal); + return -1; + } } } else { indexError("corrupt at index(cache) col:%s val: %s", term->colName, term->colVal); @@ -318,36 +331,42 @@ static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result } return 0; } -static void indexInterResultsDestroy(SArray *results) { - if (results == NULL) { return; } +static void indexInterResultsDestroy(SArray* results) { + if (results == NULL) { + return; + } size_t sz = taosArrayGetSize(results); for (size_t i = 0; i < sz; i++) { - SArray *p = taosArrayGetP(results, i); - taosArrayDestroy(p); - } + SArray* p = taosArrayGetP(results, i); + taosArrayDestroy(p); + } taosArrayDestroy(results); - } -static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *fResults) { - //refactor, merge interResults into fResults by oType - SArray *first = taosArrayGetP(interResults, 0); - taosArraySort(first, uidCompare); +static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) { + // refactor, merge interResults into fResults by oType + SArray* first = taosArrayGetP(interResults, 0); + taosArraySort(first, uidCompare); + taosArrayRemoveDuplicate(first, uidCompare, NULL); + if (oType == MUST) { - + // just one column index, enhance later + taosArrayAddAll(fResults, first); } else if (oType == SHOULD) { - + // just one column index, enhance later + taosArrayAddAll(fResults, first); // tag1 condistion || tag2 condition } else if (oType == NOT) { - - // not use currently + // just one column index, enhance later + taosArrayAddAll(fResults, first); + // not use currently } return 0; } -static int indexMergeCacheIntoTindex(SIndex *sIdx) { +static int indexMergeCacheIntoTindex(SIndex* sIdx) { if (sIdx == NULL) { - return -1; + return -1; } - indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); + indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); return 0; } diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 3c52275a4c96f2f0f12f6fae012c9681f2a37c99..6170642707434b4daaf22bc14c9f8058f841e611 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -14,160 +14,154 @@ */ #include "index_cache.h" +#include "index_util.h" #include "tcompare.h" -#define MAX_INDEX_KEY_LEN 256// test only, change later +#define MAX_INDEX_KEY_LEN 256 // test only, change later -// ref index_cache.h:22 -#define CACHE_KEY_LEN(p) (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType)) +// ref index_cache.h:22 +#define CACHE_KEY_LEN(p) \ + (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType)) -static char* getIndexKey(const void *pData) { - return NULL; -} -static int32_t compareKey(const void *l, const void *r) { - char *lp = (char *)l; - char *rp = (char *)r; +static char* getIndexKey(const void* pData) { + return NULL; +} +static int32_t compareKey(const void* l, const void* r) { + char* lp = (char*)l; + char* rp = (char*)r; // skip total len, not compare - int32_t ll, rl; // len + int32_t ll, rl; // len memcpy(&ll, lp, sizeof(int32_t)); memcpy(&rl, rp, sizeof(int32_t)); - lp += sizeof(int32_t); + lp += sizeof(int32_t); rp += sizeof(int32_t); - + // compare field id - int16_t lf, rf; // field id + int16_t lf, rf; // field id memcpy(&lf, lp, sizeof(lf)); memcpy(&rf, rp, sizeof(rf)); - if (lf != rf) { - return lf < rf ? -1: 1; - } + if (lf != rf) { return lf < rf ? -1 : 1; } lp += sizeof(lf); rp += sizeof(rf); // compare field type - int8_t lft, rft; + int8_t lft, rft; memcpy(&lft, lp, sizeof(lft)); memcpy(&rft, rp, sizeof(rft)); lp += sizeof(lft); rp += sizeof(rft); assert(rft == rft); - - // skip value len + + // skip value len int32_t lfl, rfl; - memcpy(&lfl, lp, sizeof(lfl)); - memcpy(&rfl, rp, sizeof(rfl)); + memcpy(&lfl, lp, sizeof(lfl)); + memcpy(&rfl, rp, sizeof(rfl)); lp += sizeof(lfl); rp += sizeof(rfl); - - // compare value + + // compare value int32_t i, j; for (i = 0, j = 0; i < lfl && j < rfl; i++, j++) { - if (lp[i] == rp[j]) { continue; } - else { return lp[i] < rp[j] ? -1 : 1;} + if (lp[i] == rp[j]) { + continue; + } else { + return lp[i] < rp[j] ? -1 : 1; + } + } + if (i < lfl) { + return 1; + } else if (j < rfl) { + return -1; } - if (i < lfl) { return 1;} - else if (j < rfl) { return -1; } lp += lfl; - rp += rfl; + rp += rfl; - // skip uid + // skip uid uint64_t lu, ru; - memcpy(&lu, lp, sizeof(lu)); + memcpy(&lu, lp, sizeof(lu)); memcpy(&ru, rp, sizeof(ru)); lp += sizeof(lu); rp += sizeof(ru); - + // compare version, desc order int32_t lv, rv; memcpy(&lv, lp, sizeof(lv)); memcpy(&rv, rp, sizeof(rv)); - if (lv != rv) { - return lv > rv ? -1 : 1; - } + if (lv != rv) { return lv > rv ? -1 : 1; } + lp += sizeof(lv); rp += sizeof(rv); // not care item type - return 0; - -} -IndexCache *indexCacheCreate() { - IndexCache *cache = calloc(1, sizeof(IndexCache)); - cache->skiplist = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); + return 0; +} +IndexCache* indexCacheCreate() { + IndexCache* cache = calloc(1, sizeof(IndexCache)); + if (cache == NULL) { + indexError("failed to create index cache"); + return NULL; + } + cache->skiplist = + tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); return cache; - } -void indexCacheDestroy(void *cache) { - IndexCache *pCache = cache; - if (pCache == NULL) { return; } +void indexCacheDestroy(void* cache) { + IndexCache* pCache = cache; + if (pCache == NULL) { return; } tSkipListDestroy(pCache->skiplist); free(pCache); } -int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid) { - if (cache == NULL) { return -1;} - - IndexCache *pCache = cache; +int indexCachePut(void* cache, SIndexTerm* term, int16_t colId, int32_t version, uint64_t uid) { + if (cache == NULL) { return -1; } + IndexCache* pCache = cache; // encode data - int32_t total = CACHE_KEY_LEN(term); - - char *buf = calloc(1, total); - char *p = buf; - - memcpy(p, &total, sizeof(total)); - p += sizeof(total); + int32_t total = CACHE_KEY_LEN(term); - memcpy(p, &colId, sizeof(colId)); - p += sizeof(colId); + char* buf = calloc(1, total); + char* p = buf; - memcpy(p, &term->colType, sizeof(term->colType)); - p += sizeof(term->colType); - - memcpy(p, &term->nColVal, sizeof(term->nColVal)); - p += sizeof(term->nColVal); - memcpy(p, term->colVal, term->nColVal); - p += term->nColVal; + SERIALIZE_VAR_TO_BUF(p, total, int32_t); + SERIALIZE_VAR_TO_BUF(p, colId, int16_t); - memcpy(p, &version, sizeof(version)); - p += sizeof(version); + SERIALIZE_MEM_TO_BUF(p, term, colType); + SERIALIZE_MEM_TO_BUF(p, term, nColVal); + SERIALIZE_STR_MEM_TO_BUF(p, term, colVal, term->nColVal); - memcpy(p, &uid, sizeof(uid)); - p += sizeof(uid); + SERIALIZE_VAR_TO_BUF(p, version, int32_t); + SERIALIZE_VAR_TO_BUF(p, uid, uint64_t); - memcpy(p, &term->operType, sizeof(term->operType)); - p += sizeof(term->operType); + SERIALIZE_MEM_TO_BUF(p, term, operType); - tSkipListPut(pCache->skiplist, (void *)buf); + tSkipListPut(pCache->skiplist, (void*)buf); return 0; // encode end - } -int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { - IndexCache *pCache = cache; +int indexCacheDel(void* cache, int32_t fieldId, const char* fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { + IndexCache* pCache = cache; return 0; } -int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) { +int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t version, SArray* result, STermValueType* s) { if (cache == NULL) { return -1; } - IndexCache *pCache = cache; - SIndexTerm *term = query->term; - EIndexQueryType qtype = query->qType; - - int32_t keyLen = CACHE_KEY_LEN(term); + IndexCache* pCache = cache; + SIndexTerm* term = query->term; + EIndexQueryType qtype = query->qType; - char *buf = calloc(1, keyLen); + int32_t keyLen = CACHE_KEY_LEN(term); + + char* buf = calloc(1, keyLen); if (qtype == QUERY_TERM) { - + // } else if (qtype == QUERY_PREFIX) { - + // } else if (qtype == QUERY_SUFFIX) { - + // } else if (qtype == QUERY_REGEX) { - + // } - + return 0; } - diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 7aaa498864fff74dccccdb2446c190b24272fcb2..18514cd0d5498eba9a31d8d28b197557bdf23bd8 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -14,314 +14,303 @@ */ #include "index_fst.h" -#include "tcoding.h" -#include "tchecksum.h" -#include "indexInt.h" +#include "indexInt.h" #include "index_fst_automation.h" +#include "tchecksum.h" +#include "tcoding.h" - -static void fstPackDeltaIn(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { +static void fstPackDeltaIn(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr; - fstCountingWriterPackUintIn(wrt, deltaAddr, nBytes); + fstCountingWriterPackUintIn(wrt, deltaAddr, nBytes); } -static uint8_t fstPackDetla(FstCountingWriter *wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) { +static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) { uint8_t nBytes = packDeltaSize(nodeAddr, transAddr); - fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes); + fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes); return nBytes; } -FstUnFinishedNodes *fstUnFinishedNodesCreate() { - FstUnFinishedNodes *nodes = malloc(sizeof(FstUnFinishedNodes)); +FstUnFinishedNodes* fstUnFinishedNodesCreate() { + FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes)); if (nodes == NULL) { return NULL; } - nodes->stack = (SArray *)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); + nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); fstUnFinishedNodesPushEmpty(nodes, false); return nodes; } void unFinishedNodeDestroyElem(void* elem) { - FstBuilderNodeUnfinished *b = (FstBuilderNodeUnfinished*)elem; - fstBuilderNodeDestroy(b->node); - free(b->last); + FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem; + fstBuilderNodeDestroy(b->node); + free(b->last); b->last = NULL; -} -void fstUnFinishedNodesDestroy(FstUnFinishedNodes *nodes) { - if (nodes == NULL) { return; } +} +void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { + if (nodes == NULL) { return; } - taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); + taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); free(nodes); } -void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) { - FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); - node->isFinal = isFinal; +void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) { + FstBuilderNode* node = malloc(sizeof(FstBuilderNode)); + node->isFinal = isFinal; node->finalOutput = 0; - node->trans = taosArrayInit(16, sizeof(FstTransition)); + node->trans = taosArrayInit(16, sizeof(FstTransition)); - FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; + FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; taosArrayPush(nodes->stack, &un); - } -FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes) { +FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) { assert(taosArrayGetSize(nodes->stack) == 1); - FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); - assert(un->last == NULL); - return un->node; + FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); + assert(un->last == NULL); + return un->node; } -FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr) { - FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); +FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { + FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); fstBuilderNodeUnfinishedLastCompiled(un, addr); - //free(un->last); // TODO add func FstLastTransitionFree() - //un->last = NULL; - return un->node; + // free(un->last); // TODO add func FstLastTransitionFree() + // un->last = NULL; + return un->node; } -FstBuilderNode *fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes) { - FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); - assert(un->last == NULL); - return un->node; - +FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes) { + FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack); + assert(un->last == NULL); + return un->node; } -void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *nodes, Output out) { - FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, 0); - un->node->isFinal = true; +void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* nodes, Output out) { + FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, 0); + un->node->isFinal = true; un->node->finalOutput = out; - //un->node->trans = NULL; -} -void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr) { - size_t sz = taosArrayGetSize(nodes->stack) - 1; - FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz); + // un->node->trans = NULL; +} +void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr addr) { + FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, taosArrayGetSize(nodes->stack) - 1); fstBuilderNodeUnfinishedLastCompiled(un, addr); } -void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output out) { - FstSlice *s = &bs; - if (fstSliceIsEmpty(s)) { - return; - } - size_t sz = taosArrayGetSize(nodes->stack) - 1; - FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz); +void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) { + FstSlice* s = &bs; + if (fstSliceIsEmpty(s)) { return; } + size_t sz = taosArrayGetSize(nodes->stack) - 1; + FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); assert(un->last == NULL); - //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); - //trn->inp = s->data[s->start]; - //trn->out = out; - int32_t len = 0; - uint8_t *data = fstSliceData(s, &len); - un->last = fstLastTransitionCreate(data[0], out); + // FstLastTransition *trn = malloc(sizeof(FstLastTransition)); + // trn->inp = s->data[s->start]; + // trn->out = out; + int32_t len = 0; + uint8_t* data = fstSliceData(s, &len); + un->last = fstLastTransitionCreate(data[0], out); for (uint64_t i = 1; i < len; i++) { - FstBuilderNode *n = malloc(sizeof(FstBuilderNode)); - n->isFinal = false; + FstBuilderNode* n = malloc(sizeof(FstBuilderNode)); + n->isFinal = false; n->finalOutput = 0; - n->trans = taosArrayInit(16, sizeof(FstTransition)); - - //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); - //trn->inp = s->data[i]; - //trn->out = out; - FstLastTransition *trn = fstLastTransitionCreate(data[i], 0); - - FstBuilderNodeUnfinished un = {.node = n, .last = trn}; - taosArrayPush(nodes->stack, &un); + n->trans = taosArrayInit(16, sizeof(FstTransition)); + + // FstLastTransition *trn = malloc(sizeof(FstLastTransition)); + // trn->inp = s->data[i]; + // trn->out = out; + FstLastTransition* trn = fstLastTransitionCreate(data[i], 0); + + FstBuilderNodeUnfinished un = {.node = n, .last = trn}; + taosArrayPush(nodes->stack, &un); } - fstUnFinishedNodesPushEmpty(nodes, true); + fstUnFinishedNodesPushEmpty(nodes, true); } +uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes* node, FstSlice bs) { + FstSlice* s = &bs; -uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) { - FstSlice *s = &bs; - - size_t ssz = taosArrayGetSize(node->stack); // stack size + size_t ssz = taosArrayGetSize(node->stack); // stack size uint64_t count = 0; - int32_t lsz; // data len - uint8_t *data = fstSliceData(s, &lsz); + int32_t lsz; // data len + uint8_t* data = fstSliceData(s, &lsz); for (size_t i = 0; i < ssz && i < lsz; i++) { - FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); + FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i); if (un->last->inp == data[i]) { count++; } else { break; - } + } } return count; } -uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out) { - FstSlice *s = &bs; +uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstSlice bs, Output in, Output* out) { + FstSlice* s = &bs; - size_t lsz = (size_t)(s->end - s->start + 1); // data len - size_t ssz = taosArrayGetSize(node->stack); // stack size + size_t lsz = (size_t)(s->end - s->start + 1); // data len + size_t ssz = taosArrayGetSize(node->stack); // stack size *out = in; uint64_t i = 0; for (i = 0; i < lsz && i < ssz; i++) { - FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); + FstBuilderNodeUnfinished* un = taosArrayGet(node->stack, i); - FstLastTransition *t = un->last; - uint64_t addPrefix = 0; - uint8_t *data = fstSliceData(s, NULL); + FstLastTransition* t = un->last; + uint64_t addPrefix = 0; + uint8_t* data = fstSliceData(s, NULL); if (t && t->inp == data[i]) { - uint64_t commPrefix = MIN(t->out, *out); - uint64_t tAddPrefix = t->out - commPrefix; - (*out) = (*out) - commPrefix; + uint64_t commPrefix = MIN(t->out, *out); + uint64_t tAddPrefix = t->out - commPrefix; + (*out) = (*out) - commPrefix; t->out = commPrefix; - addPrefix = tAddPrefix; + addPrefix = tAddPrefix; } else { - break; + break; } if (addPrefix != 0) { if (i + 1 < ssz) { - FstBuilderNodeUnfinished *unf = taosArrayGet(node->stack, i + 1); - fstBuilderNodeUnfinishedAddOutputPrefix(unf, addPrefix); + FstBuilderNodeUnfinished* unf = taosArrayGet(node->stack, i + 1); + fstBuilderNodeUnfinishedAddOutputPrefix(unf, addPrefix); } } - } + } return i; -} - +} FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) { FstState fs = {.state = EmptyFinal, .val = 0}; - if (addr == EMPTY_ADDRESS) { - return fs; - } - - uint8_t *data = fstSliceData(slice, NULL); - uint8_t v = data[addr]; - uint8_t t = (v & 0b11000000) >> 6; + if (addr == EMPTY_ADDRESS) { return fs; } + + uint8_t* data = fstSliceData(slice, NULL); + uint8_t v = data[addr]; + uint8_t t = (v & 0b11000000) >> 6; if (t == 0b11) { fs.state = OneTransNext; } else if (t == 0b10) { - fs.state = OneTrans; + fs.state = OneTrans; } else { - fs.state = AnyTrans; + fs.state = AnyTrans; } fs.val = v; return fs; } -static FstState fstStateDict[] = { - {.state = OneTransNext, .val = 0b11000000}, - {.state = OneTrans, .val = 0b10000000}, - {.state = AnyTrans, .val = 0b00000000}, - {.state = EmptyFinal, .val = 0b00000000} -}; -// debug -static const char *fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; +static FstState fstStateDict[] = {{.state = OneTransNext, .val = 0b11000000}, + {.state = OneTrans, .val = 0b10000000}, + {.state = AnyTrans, .val = 0b00000000}, + {.state = EmptyFinal, .val = 0b00000000}}; +// debug +static const char* fstStateStr[] = {"ONE_TRANS_NEXT", "ONE_TRANS", "ANY_TRANS", "EMPTY_FINAL"}; -FstState fstStateCreate(State state){ +FstState fstStateCreate(State state) { uint8_t idx = (uint8_t)state; return fstStateDict[idx]; } -//compile -void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uint8_t inp) { - FstState s = fstStateCreate(OneTransNext); +// compile +void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp) { + FstState s = fstStateCreate(OneTransNext); fstStateSetCommInput(&s, inp); - bool null = false; + bool null = false; uint8_t v = fstStateCommInput(&s, &null); if (null) { // w->write_all(&[inp]) fstCountingWriterWrite(w, &inp, 1); - } + } fstCountingWriterWrite(w, &(s.val), 1); // w->write_all(&[s.val]) return; } -void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTransition* trn) { - Output out = trn->out; - uint8_t outPackSize = (out == 0 ? 0 : fstCountingWriterPackUint(w, out)); - uint8_t transPackSize = fstPackDetla(w, addr, trn->addr); +void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn) { + Output out = trn->out; + uint8_t outPackSize = (out == 0 ? 0 : fstCountingWriterPackUint(w, out)); + uint8_t transPackSize = fstPackDetla(w, addr, trn->addr); PackSizes packSizes = 0; FST_SET_OUTPUT_PACK_SIZE(packSizes, outPackSize); FST_SET_TRANSITION_PACK_SIZE(packSizes, transPackSize); - fstCountingWriterWrite(w, (char *)&packSizes, sizeof(packSizes)); + fstCountingWriterWrite(w, (char*)&packSizes, sizeof(packSizes)); - FstState st = fstStateCreate(OneTrans); - - fstStateSetCommInput(&st, trn->inp); - bool null = false; - uint8_t inp = fstStateCommInput(&st, &null); - if (null == true) { - fstCountingWriterWrite(w, (char *)&trn->inp, sizeof(trn->inp)); - } - fstCountingWriterWrite(w, (char *)(&(st.val)), sizeof(st.val)); - return ; + FstState st = fstStateCreate(OneTrans); + fstStateSetCommInput(&st, trn->inp); + bool null = false; + uint8_t inp = fstStateCommInput(&st, &null); + if (null == true) { fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); } + fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val)); + return; } -void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node) { - size_t sz = taosArrayGetSize(node->trans); +void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node) { + size_t sz = taosArrayGetSize(node->trans); assert(sz <= 256); uint8_t tSize = 0; - uint8_t oSize = packSize(node->finalOutput) ; - + uint8_t oSize = packSize(node->finalOutput); + // finalOutput.is_zero() - bool anyOuts = (node->finalOutput != 0) ; + bool anyOuts = (node->finalOutput != 0); for (size_t i = 0; i < sz; i++) { - FstTransition *t = taosArrayGet(node->trans, i); - tSize = MAX(tSize, packDeltaSize(addr, t->addr)); + FstTransition* t = taosArrayGet(node->trans, i); + tSize = MAX(tSize, packDeltaSize(addr, t->addr)); oSize = MAX(oSize, packSize(t->out)); - anyOuts = anyOuts || (t->out != 0); + anyOuts = anyOuts || (t->out != 0); } - PackSizes packSizes = 0; - if (anyOuts) { FST_SET_OUTPUT_PACK_SIZE(packSizes, oSize); } - else { FST_SET_OUTPUT_PACK_SIZE(packSizes, 0); } + PackSizes packSizes = 0; + if (anyOuts) { + FST_SET_OUTPUT_PACK_SIZE(packSizes, oSize); + } else { + FST_SET_OUTPUT_PACK_SIZE(packSizes, 0); + } FST_SET_TRANSITION_PACK_SIZE(packSizes, tSize); - + FstState st = fstStateCreate(AnyTrans); - fstStateSetFinalState(&st, node->isFinal); + fstStateSetFinalState(&st, node->isFinal); fstStateSetStateNtrans(&st, (uint8_t)sz); - + if (anyOuts) { - if (FST_BUILDER_NODE_IS_FINAL(node)) { - fstCountingWriterPackUintIn(w, node->finalOutput, oSize); - } + if (FST_BUILDER_NODE_IS_FINAL(node)) { fstCountingWriterPackUintIn(w, node->finalOutput, oSize); } for (int32_t i = sz - 1; i >= 0; i--) { - FstTransition *t = taosArrayGet(node->trans, i); + FstTransition* t = taosArrayGet(node->trans, i); fstCountingWriterPackUintIn(w, t->out, oSize); } - } + } for (int32_t i = sz - 1; i >= 0; i--) { - FstTransition *t = taosArrayGet(node->trans, i); - fstPackDeltaIn(w, addr, t->addr, tSize); + FstTransition* t = taosArrayGet(node->trans, i); + fstPackDeltaIn(w, addr, t->addr, tSize); } for (int32_t i = sz - 1; i >= 0; i--) { - FstTransition *t = taosArrayGet(node->trans, i); - fstCountingWriterWrite(w, (char *)&t->inp, 1); - //fstPackDeltaIn(w, addr, t->addr, tSize); + FstTransition* t = taosArrayGet(node->trans, i); + fstCountingWriterWrite(w, (char*)&t->inp, 1); + // fstPackDeltaIn(w, addr, t->addr, tSize); } if (sz > TRANS_INDEX_THRESHOLD) { // A value of 255 indicates that no transition exists for the byte // at that index. (Except when there are 256 transitions.) Namely, // any value greater than or equal to the number of transitions in // this node indicates an absent transition. - uint8_t *index = (uint8_t *)malloc(sizeof(uint8_t) * 256); + uint8_t* index = (uint8_t*)malloc(sizeof(uint8_t) * 256); memset(index, 255, sizeof(uint8_t) * 256); - ///for (uint8_t i = 0; i < 256; i++) { + /// for (uint8_t i = 0; i < 256; i++) { // index[i] = 255; ///} for (size_t i = 0; i < sz; i++) { - FstTransition *t = taosArrayGet(node->trans, i); + FstTransition* t = taosArrayGet(node->trans, i); index[t->inp] = i; - //fstPackDeltaIn(w, addr, t->addr, tSize); + // fstPackDeltaIn(w, addr, t->addr, tSize); } - fstCountingWriterWrite(w, (char *)index, 256); + fstCountingWriterWrite(w, (char*)index, 256); free(index); } - fstCountingWriterWrite(w, (char *)&packSizes, 1); + fstCountingWriterWrite(w, (char*)&packSizes, 1); bool null = false; fstStateStateNtrans(&st, &null); if (null == true) { - // 256 can't be represented in a u8, so we abuse the fact that - // the # of transitions can never be 1 here, since 1 is always - // encoded in the state byte. + // 256 can't be represented in a u8, so we abuse the fact that + // the # of transitions can never be 1 here, since 1 is always + // encoded in the state byte. uint8_t v = 1; - if (sz == 256) { fstCountingWriterWrite(w, (char *)&v, 1); } - else { fstCountingWriterWrite(w, (char *)&sz, 1); } + if (sz == 256) { + fstCountingWriterWrite(w, (char*)&v, 1); + } else { + fstCountingWriterWrite(w, (char*)&sz, 1); + } } - fstCountingWriterWrite(w, (char *)(&(st.val)), 1); + fstCountingWriterWrite(w, (char*)(&(st.val)), 1); return; } @@ -330,20 +319,20 @@ void fstStateSetCommInput(FstState* s, uint8_t inp) { assert(s->state == OneTransNext || s->state == OneTrans); uint8_t val; - COMMON_INDEX(inp, 0x111111, val); - s->val = (s->val & fstStateDict[s->state].val) | val; + COMMON_INDEX(inp, 0x111111, val); + s->val = (s->val & fstStateDict[s->state].val) | val; } // comm_input -uint8_t fstStateCommInput(FstState* s, bool *null) { +uint8_t fstStateCommInput(FstState* s, bool* null) { assert(s->state == OneTransNext || s->state == OneTrans); uint8_t v = s->val & 0b00111111; - if (v == 0) { - *null = true; + if (v == 0) { + *null = true; return v; - } - //v = 0 indicate that common_input is None - return v == 0 ? 0 : COMMON_INPUT(v); + } + // v = 0 indicate that common_input is None + return v == 0 ? 0 : COMMON_INPUT(v); } // input_len @@ -351,373 +340,320 @@ uint8_t fstStateCommInput(FstState* s, bool *null) { uint64_t fstStateInputLen(FstState* s) { assert(s->state == OneTransNext || s->state == OneTrans); bool null = false; - fstStateCommInput(s, &null); - return null ? 1 : 0 ; -} - -// end_addr -uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice *data) { + fstStateCommInput(s, &null); + return null ? 1 : 0; +} + +// end_addr +uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice* data) { assert(s->state == OneTransNext); return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s); } -uint64_t fstStateEndAddrForOneTrans(FstState *s, FstSlice *data, PackSizes sizes) { +uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes) { assert(s->state == OneTrans); - return FST_SLICE_LEN(data) - - 1 - - fstStateInputLen(s) - - 1 // pack size - - FST_GET_TRANSITION_PACK_SIZE(sizes) - - FST_GET_OUTPUT_PACK_SIZE(sizes); -} -uint64_t fstStateEndAddrForAnyTrans(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans) { - uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); - uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; - return FST_SLICE_LEN(date) - - 1 - - fstStateNtransLen(state) - - 1 //pack size - - fstStateTotalTransSize(state, version, sizes, nTrans) - - nTrans * oSizes // output values - - finalOsize; // final output -} -// input -uint8_t fstStateInput(FstState *s, FstNode *node) { + return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size + - FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes); +} +uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice* date, PackSizes sizes, uint64_t nTrans) { + uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); + uint8_t finalOsize = !fstStateIsFinalState(state) ? 0 : oSizes; + return FST_SLICE_LEN(date) - 1 - fstStateNtransLen(state) - 1 // pack size + - fstStateTotalTransSize(state, version, sizes, nTrans) - nTrans * oSizes // output values + - finalOsize; // final output +} +// input +uint8_t fstStateInput(FstState* s, FstNode* node) { assert(s->state == OneTransNext || s->state == OneTrans); - FstSlice *slice = &node->data; - bool null = false; - uint8_t inp = fstStateCommInput(s, &null); - uint8_t *data = fstSliceData(slice, NULL); + FstSlice* slice = &node->data; + bool null = false; + uint8_t inp = fstStateCommInput(s, &null); + uint8_t* data = fstSliceData(slice, NULL); return null == false ? inp : data[-1]; } -uint8_t fstStateInputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { +uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { assert(s->state == AnyTrans); - FstSlice *slice = &node->data; + FstSlice* slice = &node->data; - uint64_t at = node->start - - fstStateNtransLen(s) - - 1 // pack size - - fstStateTransIndexSize(s, node->version, node->nTrans) - - i - - 1; // the output size + uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size + - fstStateTransIndexSize(s, node->version, node->nTrans) - i - 1; // the output size - uint8_t *data = fstSliceData(slice, NULL); + uint8_t* data = fstSliceData(slice, NULL); return data[at]; } // trans_addr -CompiledAddr fstStateTransAddr(FstState *s, FstNode *node) { +CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) { assert(s->state == OneTransNext || s->state == OneTrans); - FstSlice *slice = &node->data; + FstSlice* slice = &node->data; if (s->state == OneTransNext) { - return (CompiledAddr)(node->end) - 1; + return (CompiledAddr)(node->end) - 1; } else { - PackSizes sizes = node->sizes; - uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(sizes); - uint64_t i = node->start - - fstStateInputLen(s) - - 1 // PackSizes - - tSizes; - - // refactor error logic - uint8_t *data = fstSliceData(slice, NULL); - return unpackDelta(data +i, tSizes, node->end); - } -} -CompiledAddr fstStateTransAddrForAnyTrans(FstState *s, FstNode *node, uint64_t i) { + PackSizes sizes = node->sizes; + uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(sizes); + uint64_t i = node->start - fstStateInputLen(s) - 1 // PackSizes + - tSizes; + + // refactor error logic + uint8_t* data = fstSliceData(slice, NULL); + return unpackDelta(data + i, tSizes, node->end); + } +} +CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i) { assert(s->state == AnyTrans); - FstSlice *slice = &node->data; - uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); - uint64_t at = node->start - - fstStateNtransLen(s) - - 1 - - fstStateTransIndexSize(s, node->version, node->nTrans) - - node->nTrans - - (i * tSizes) - - tSizes; - uint8_t *data = fstSliceData(slice, NULL); - return unpackDelta(data + at, tSizes, node->end); -} - -// sizes -PackSizes fstStateSizes(FstState *s, FstSlice *slice) { - assert(s->state == OneTrans || s->state == AnyTrans) ; - uint64_t i; + FstSlice* slice = &node->data; + uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); + uint64_t at = node->start - fstStateNtransLen(s) - 1 - fstStateTransIndexSize(s, node->version, node->nTrans) - node->nTrans - + (i * tSizes) - tSizes; + uint8_t* data = fstSliceData(slice, NULL); + return unpackDelta(data + at, tSizes, node->end); +} + +// sizes +PackSizes fstStateSizes(FstState* s, FstSlice* slice) { + assert(s->state == OneTrans || s->state == AnyTrans); + uint64_t i; if (s->state == OneTrans) { - i = FST_SLICE_LEN(slice) - 1 - fstStateInputLen(s) - 1; + i = FST_SLICE_LEN(slice) - 1 - fstStateInputLen(s) - 1; } else { i = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1; } - uint8_t *data = fstSliceData(slice, NULL); - return (PackSizes)(*(data +i)); + uint8_t* data = fstSliceData(slice, NULL); + return (PackSizes)(*(data + i)); } -// Output -Output fstStateOutput(FstState *s, FstNode *node) { - assert(s->state == OneTrans); - +// Output +Output fstStateOutput(FstState* s, FstNode* node) { + assert(s->state == OneTrans); + uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); - if (oSizes == 0) { - return 0; - } - FstSlice *slice = &node->data; - uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); - - uint64_t i = node->start - - fstStateInputLen(s) - - 1 - - tSizes - - oSizes; - uint8_t *data = fstSliceData(slice, NULL); + if (oSizes == 0) { return 0; } + FstSlice* slice = &node->data; + uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); + + uint64_t i = node->start - fstStateInputLen(s) - 1 - tSizes - oSizes; + uint8_t* data = fstSliceData(slice, NULL); return unpackUint64(data + i, oSizes); - } -Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { +Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) { assert(s->state == AnyTrans); - uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); - if (oSizes == 0) { - return 0; - } - FstSlice *slice = &node->data; - uint8_t *data = fstSliceData(slice, NULL); - uint64_t at = node->start - - fstStateNtransLen(s) - - 1 // pack size - - fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - - (i * oSizes) - - oSizes; + uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); + if (oSizes == 0) { return 0; } + FstSlice* slice = &node->data; + uint8_t* data = fstSliceData(slice, NULL); + uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size + - fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - (i * oSizes) - oSizes; return unpackUint64(data + at, oSizes); } // anyTrans specify function -void fstStateSetFinalState(FstState *s, bool yes) { - assert(s->state == AnyTrans); - if (yes) { s->val |= 0b01000000; } +void fstStateSetFinalState(FstState* s, bool yes) { + assert(s->state == AnyTrans); + if (yes) { s->val |= 0b01000000; } return; } -bool fstStateIsFinalState(FstState *s) { - assert(s->state == AnyTrans); - return (s->val & 0b01000000) == 0b01000000; -} +bool fstStateIsFinalState(FstState* s) { + assert(s->state == AnyTrans); + return (s->val & 0b01000000) == 0b01000000; +} -void fstStateSetStateNtrans(FstState *s, uint8_t n) { - assert(s->state == AnyTrans); - if (n <= 0b00111111) { - s->val = (s->val & 0b11000000) | n; - } +void fstStateSetStateNtrans(FstState* s, uint8_t n) { + assert(s->state == AnyTrans); + if (n <= 0b00111111) { s->val = (s->val & 0b11000000) | n; } return; } // state_ntrans -uint8_t fstStateStateNtrans(FstState *s, bool *null) { - assert(s->state == AnyTrans); +uint8_t fstStateStateNtrans(FstState* s, bool* null) { + assert(s->state == AnyTrans); *null = false; - uint8_t n = s->val & 0b00111111; + uint8_t n = s->val & 0b00111111; if (n == 0) { - *null = true; // None - } + *null = true; // None + } return n; } -uint64_t fstStateTotalTransSize(FstState *s, uint64_t version, PackSizes sizes, uint64_t nTrans) { - assert(s->state == AnyTrans); - uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans); +uint64_t fstStateTotalTransSize(FstState* s, uint64_t version, PackSizes sizes, uint64_t nTrans) { + assert(s->state == AnyTrans); + uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans); return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize; } -uint64_t fstStateTransIndexSize(FstState *s, uint64_t version, uint64_t nTrans) { - assert(s->state == AnyTrans); - return (version >= 2 &&nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0; +uint64_t fstStateTransIndexSize(FstState* s, uint64_t version, uint64_t nTrans) { + assert(s->state == AnyTrans); + return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0; } -uint64_t fstStateNtransLen(FstState *s) { +uint64_t fstStateNtransLen(FstState* s) { assert(s->state == AnyTrans); bool null = false; fstStateStateNtrans(s, &null); - return null == true ? 1 : 0; + return null == true ? 1 : 0; } -uint64_t fstStateNtrans(FstState *s, FstSlice *slice) { - bool null = false; +uint64_t fstStateNtrans(FstState* s, FstSlice* slice) { + bool null = false; uint8_t n = fstStateStateNtrans(s, &null); - if (null != true) { - return n; - } - int32_t len; - uint8_t *data = fstSliceData(slice, &len); + if (null != true) { return n; } + int32_t len; + uint8_t* data = fstSliceData(slice, &len); n = data[len - 2]; - //n = data[slice->end - 1]; // data[data.len() - 2] - return n == 1 ? 256: n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, then it is encoded in the state byte -} -Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, PackSizes sizes, uint64_t nTrans) { - uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); - if (oSizes == 0 || !fstStateIsFinalState(s)) { - return 0; - } - - uint64_t at = FST_SLICE_LEN(slice) - - 1 - - fstStateNtransLen(s) - - 1 // pack size - - fstStateTotalTransSize(s, version, sizes, nTrans) - - (nTrans * oSizes) - - oSizes; - uint8_t *data = fstSliceData(slice, NULL); - return unpackUint64(data + at, (uint8_t)oSizes); - -} -uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { + // n = data[slice->end - 1]; // data[data.len() - 2] + return n == 1 ? 256 : n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, + // then it is encoded in the state byte +} +Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) { + uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); + if (oSizes == 0 || !fstStateIsFinalState(s)) { return 0; } + + uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size + - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; + uint8_t* data = fstSliceData(slice, NULL); + return unpackUint64(data + at, (uint8_t)oSizes); +} +uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) { assert(s->state == AnyTrans); - FstSlice *slice = &node->data; + FstSlice* slice = &node->data; if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) { - uint64_t at = node->start - - fstStateNtransLen(s) - - 1 // pack size + uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size - fstStateTransIndexSize(s, node->version, node->nTrans); - int32_t dlen = 0; - uint8_t *data = fstSliceData(slice, &dlen); + int32_t dlen = 0; + uint8_t* data = fstSliceData(slice, &dlen); uint64_t i = data[at + b]; - //uint64_t i = slice->data[slice->start + at + b]; - if (i >= node->nTrans) { - *null = true; - } + // uint64_t i = slice->data[slice->start + at + b]; + if (i >= node->nTrans) { *null = true; } return i; } else { - uint64_t start = node->start - - fstStateNtransLen(s) - - 1 // pack size - - node->nTrans; - uint64_t end = start + node->nTrans; + uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size + - node->nTrans; + uint64_t end = start + node->nTrans; FstSlice t = fstSliceCopy(slice, start, end - 1); - int32_t len = 0; - uint8_t *data = fstSliceData(&t, &len); - int i = 0; - for(; i < len; i++) { - uint8_t v = data[i]; + int32_t len = 0; + uint8_t* data = fstSliceData(&t, &len); + int i = 0; + for (; i < len; i++) { + uint8_t v = data[i]; if (v == b) { fstSliceDestroy(&t); - return node->nTrans - i - 1; // bug + return node->nTrans - i - 1; // bug } - } + } if (i == len) { *null = true; } fstSliceDestroy(&t); - } + } } +// fst node function -// fst node function - -FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) { - FstNode *n = (FstNode *)malloc(sizeof(FstNode)); +FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { + FstNode* n = (FstNode*)malloc(sizeof(FstNode)); if (n == NULL) { return NULL; } - FstState st = fstStateCreateFrom(slice, addr); + FstState st = fstStateCreateFrom(slice, addr); if (st.state == EmptyFinal) { - n->data = fstSliceCreate(NULL, 0); - n->version = version; - n->state = st; - n->start = EMPTY_ADDRESS; - n->end = EMPTY_ADDRESS; - n->isFinal = true; - n->nTrans = 0; - n->sizes = 0; - n->finalOutput = 0; + n->data = fstSliceCreate(NULL, 0); + n->version = version; + n->state = st; + n->start = EMPTY_ADDRESS; + n->end = EMPTY_ADDRESS; + n->isFinal = true; + n->nTrans = 0; + n->sizes = 0; + n->finalOutput = 0; } else if (st.state == OneTransNext) { - n->data = fstSliceCopy(slice, 0, addr); - n->version = version; - n->state = st; - n->start = addr; - n->end = fstStateEndAddrForOneTransNext(&st, &n->data); //? s.end_addr(data); - n->isFinal = false; - n->sizes = 0; - n->nTrans = 1; - n->finalOutput = 0; + n->data = fstSliceCopy(slice, 0, addr); + n->version = version; + n->state = st; + n->start = addr; + n->end = fstStateEndAddrForOneTransNext(&st, &n->data); //? s.end_addr(data); + n->isFinal = false; + n->sizes = 0; + n->nTrans = 1; + n->finalOutput = 0; } else if (st.state == OneTrans) { - FstSlice data = fstSliceCopy(slice, 0, addr); - PackSizes sz = fstStateSizes(&st, &data); - n->data = data; - n->version = version; - n->state = st; - n->start = addr; - n->end = fstStateEndAddrForOneTrans(&st, &data, sz); // s.end_addr(data, sz); - n->isFinal = false; - n->nTrans = 1; - n->sizes = sz; - n->finalOutput = 0; + FstSlice data = fstSliceCopy(slice, 0, addr); + PackSizes sz = fstStateSizes(&st, &data); + n->data = data; + n->version = version; + n->state = st; + n->start = addr; + n->end = fstStateEndAddrForOneTrans(&st, &data, sz); // s.end_addr(data, sz); + n->isFinal = false; + n->nTrans = 1; + n->sizes = sz; + n->finalOutput = 0; } else { - FstSlice data = fstSliceCopy(slice, 0, addr); - uint64_t sz = fstStateSizes(&st, &data); // s.sizes(data) - uint32_t nTrans = fstStateNtrans(&st, &data); // s.ntrans(data) - n->data = data; - n->version = version; - n->state = st; - n->start = addr; - n->end = fstStateEndAddrForAnyTrans(&st, version, &data, sz, nTrans); // s.end_addr(version, data, sz, ntrans); - n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); - n->nTrans = nTrans; - n->sizes = sz; - n->finalOutput = fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); + FstSlice data = fstSliceCopy(slice, 0, addr); + uint64_t sz = fstStateSizes(&st, &data); // s.sizes(data) + uint32_t nTrans = fstStateNtrans(&st, &data); // s.ntrans(data) + n->data = data; + n->version = version; + n->state = st; + n->start = addr; + n->end = fstStateEndAddrForAnyTrans(&st, version, &data, sz, nTrans); // s.end_addr(version, data, sz, ntrans); + n->isFinal = fstStateIsFinalState(&st); // s.is_final_state(); + n->nTrans = nTrans; + n->sizes = sz; + n->finalOutput = fstStateFinalOutput(&st, version, &data, sz, nTrans); // s.final_output(version, data, sz, ntrans); } - return n; + return n; } // debug state transition -static const char *fstNodeState(FstNode *node) { - FstState *st = &node->state; - return fstStateStr[st->state]; +static const char* fstNodeState(FstNode* node) { + FstState* st = &node->state; + return fstStateStr[st->state]; } - -void fstNodeDestroy(FstNode *node) { - fstSliceDestroy(&node->data); +void fstNodeDestroy(FstNode* node) { + fstSliceDestroy(&node->data); free(node); } -FstTransitions* fstNodeTransitions(FstNode *node) { - FstTransitions *t = malloc(sizeof(FstTransitions)); - if (NULL == t) { - return NULL; - } +FstTransitions* fstNodeTransitions(FstNode* node) { + FstTransitions* t = malloc(sizeof(FstTransitions)); + if (NULL == t) { return NULL; } FstRange range = {.start = 0, .end = FST_NODE_LEN(node)}; t->range = range; - t->node = node; - return t; -} - -// Returns the transition at index `i`. -bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *trn) { - bool s = true; - FstState *st = &node->state; + t->node = node; + return t; +} + +// Returns the transition at index `i`. +bool fstNodeGetTransitionAt(FstNode* node, uint64_t i, FstTransition* trn) { + bool s = true; + FstState* st = &node->state; if (st->state == OneTransNext) { - trn->inp = fstStateInput(st, node); - trn->out = 0; - trn->addr = fstStateTransAddr(st, node); + trn->inp = fstStateInput(st, node); + trn->out = 0; + trn->addr = fstStateTransAddr(st, node); } else if (st->state == OneTrans) { - trn->inp = fstStateInput(st, node); - trn->out = fstStateOutput(st, node); - trn->addr = fstStateTransAddr(st, node); + trn->inp = fstStateInput(st, node); + trn->out = fstStateOutput(st, node); + trn->addr = fstStateTransAddr(st, node); } else if (st->state == AnyTrans) { - trn->inp = fstStateInputForAnyTrans(st, node, i); - trn->out = fstStateOutputForAnyTrans(st, node, i); - trn->addr = fstStateTransAddrForAnyTrans(st, node, i); + trn->inp = fstStateInputForAnyTrans(st, node, i); + trn->out = fstStateOutputForAnyTrans(st, node, i); + trn->addr = fstStateTransAddrForAnyTrans(st, node, i); } else { s = false; } return s; -} +} // Returns the transition address of the `i`th transition -bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res) { - bool s = true; - FstState *st = &node->state; +bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res) { + bool s = true; + FstState* st = &node->state; if (st->state == OneTransNext) { assert(i == 0); fstStateTransAddr(st, node); } else if (st->state == OneTrans) { - assert(i == 0); + assert(i == 0); fstStateTransAddr(st, node); } else if (st->state == AnyTrans) { fstStateTransAddrForAnyTrans(st, node, i); - } else if (FST_STATE_EMPTY_FINAL(node)){ + } else if (FST_STATE_EMPTY_FINAL(node)) { s = false; } else { assert(0); @@ -726,132 +662,137 @@ bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res) { } // Finds the `i`th transition corresponding to the given input byte. -// If no transition for this byte exists, then `false` is returned. -bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res) { - bool s = true; - FstState *st = &node->state; +// If no transition for this byte exists, then `false` is returned. +bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res) { + bool s = true; + FstState* st = &node->state; if (st->state == OneTransNext) { - if (fstStateInput(st,node) == b) { *res = 0; } - else { s = false; } } - else if (st->state == OneTrans) { - if (fstStateInput(st, node) == b) { *res = 0 ;} - else { s = false; } + if (fstStateInput(st, node) == b) { + *res = 0; + } else { + s = false; + } + } else if (st->state == OneTrans) { + if (fstStateInput(st, node) == b) { + *res = 0; + } else { + s = false; + } } else if (st->state == AnyTrans) { - bool null = false; - uint64_t out = fstStateFindInput(st, node, b, &null); - if (null == false) { *res = out; } - else { s = false;} + bool null = false; + uint64_t out = fstStateFindInput(st, node, b, &null); + if (null == false) { + *res = out; + } else { + s = false; + } } return s; -} +} -bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode) { - size_t sz = taosArrayGetSize(builderNode->trans); +bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode) { + size_t sz = taosArrayGetSize(builderNode->trans); assert(sz < 256); if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) { - return true; + return true; } else if (sz != 1 || builderNode->isFinal) { - fstStateCompileForAnyTrans(w, addr, builderNode); + fstStateCompileForAnyTrans(w, addr, builderNode); // AnyTrans->Compile(w, addr, node); } else { - FstTransition *tran = taosArrayGet(builderNode->trans, 0); + FstTransition* tran = taosArrayGet(builderNode->trans, 0); if (tran->addr == lastAddr && tran->out == 0) { - fstStateCompileForOneTransNext(w, addr, tran->inp); - //OneTransNext::compile(w, lastAddr, tran->inp); - return true; + fstStateCompileForOneTransNext(w, addr, tran->inp); + // OneTransNext::compile(w, lastAddr, tran->inp); + return true; } else { - fstStateCompileForOneTrans(w, addr, tran); - //OneTrans::Compile(w, lastAddr, *tran); - return true; - } - } - return true; -} - -bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { - return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b); + fstStateCompileForOneTrans(w, addr, tran); + // OneTrans::Compile(w, lastAddr, *tran); + return true; + } + } + return true; } +bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { + return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b); +} - -FstBuilder *fstBuilderCreate(void *w, FstType ty) { - FstBuilder *b = malloc(sizeof(FstBuilder)); +FstBuilder* fstBuilderCreate(void* w, FstType ty) { + FstBuilder* b = malloc(sizeof(FstBuilder)); if (NULL == b) { return b; } - - b->wrt = fstCountingWriterCreate(w, false); - b->unfinished = fstUnFinishedNodesCreate(); - b->registry = fstRegistryCreate(10000, 2) ; - b->last = fstSliceCreate(NULL, 0); - b->lastAddr = NONE_ADDRESS; - b->len = 0; - - char buf64[8] = {0}; - void *pBuf64 = buf64; - taosEncodeFixedU64(&pBuf64, VERSION); + b->wrt = fstCountingWriterCreate(w); + b->unfinished = fstUnFinishedNodesCreate(); + b->registry = fstRegistryCreate(10000, 2); + b->last = fstSliceCreate(NULL, 0); + b->lastAddr = NONE_ADDRESS; + b->len = 0; + + char buf64[8] = {0}; + void* pBuf64 = buf64; + taosEncodeFixedU64(&pBuf64, VERSION); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); - - memset(buf64, 0, sizeof(buf64)); + + memset(buf64, 0, sizeof(buf64)); pBuf64 = buf64; - taosEncodeFixedU64(&pBuf64, ty); + taosEncodeFixedU64(&pBuf64, ty); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); return b; } -void fstBuilderDestroy(FstBuilder *b) { +void fstBuilderDestroy(FstBuilder* b) { if (b == NULL) { return; } - fstCountingWriterDestroy(b->wrt); - fstUnFinishedNodesDestroy(b->unfinished); + fstCountingWriterDestroy(b->wrt); + fstUnFinishedNodesDestroy(b->unfinished); fstRegistryDestroy(b->registry); fstSliceDestroy(&b->last); free(b); } - -bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in) { - OrderType t = fstBuilderCheckLastKey(b, bs, true); +bool fstBuilderInsert(FstBuilder* b, FstSlice bs, Output in) { + OrderType t = fstBuilderCheckLastKey(b, bs, true); if (t == Ordered) { // add log info - fstBuilderInsertOutput(b, bs, in); - return true; - } - indexInfo("key must be ordered"); + fstBuilderInsertOutput(b, bs, in); + return true; + } + indexInfo("fst write key must be ordered"); return false; } -void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { - FstSlice *s = &bs; - if (fstSliceIsEmpty(s)) { - b->len = 1; - fstUnFinishedNodesSetRootOutput(b->unfinished, in); - return; - } - //if (in != 0) { //if let Some(in) = in - // prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); - //} else { - // prefixLen = fstUnFinishedNodesFindCommPrefix(b->unfinished, bs); - // out = 0; - //} - Output out; - uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); - - if (prefixLen == FST_SLICE_LEN(s)) { - assert(out == 0); - return; - } - - b->len += 1; - fstBuilderCompileFrom(b, prefixLen); - - FstSlice sub = fstSliceCopy(s, prefixLen, s->end); - fstUnFinishedNodesAddSuffix(b->unfinished, sub, out); - fstSliceDestroy(&sub); - return; - } - -OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { - FstSlice *input = &bs; +void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) { + FstSlice* s = &bs; + if (fstSliceIsEmpty(s)) { + b->len = 1; + fstUnFinishedNodesSetRootOutput(b->unfinished, in); + return; + } + // if (in != 0) { //if let Some(in) = in + // prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); + //} else { + // prefixLen = fstUnFinishedNodesFindCommPrefix(b->unfinished, bs); + // out = 0; + //} + Output out; + uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); + + if (prefixLen == FST_SLICE_LEN(s)) { + assert(out == 0); + return; + } + + b->len += 1; + fstBuilderCompileFrom(b, prefixLen); + + FstSlice sub = fstSliceCopy(s, prefixLen, s->end); + fstUnFinishedNodesAddSuffix(b->unfinished, sub, out); + fstSliceDestroy(&sub); + return; +} + +OrderType fstBuilderCheckLastKey(FstBuilder* b, FstSlice bs, bool ckDup) { + FstSlice* input = &bs; if (fstSliceIsEmpty(&b->last)) { fstSliceDestroy(&b->last); // deep copy or not @@ -859,20 +800,20 @@ OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { } else { int comp = fstSliceCompare(&b->last, &bs); if (comp == 0 && ckDup) { - return DuplicateKey; + return DuplicateKey; } else if (comp == 1) { return OutOfOrdered; } // deep copy or not fstSliceDestroy(&b->last); - b->last = fstSliceDeepCopy(&bs, input->start, input->end); - } + b->last = fstSliceDeepCopy(&bs, input->start, input->end); + } return Ordered; -} -void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { +} +void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) { CompiledAddr addr = NONE_ADDRESS; while (istate + 1 < FST_UNFINISHED_NODES_LEN(b->unfinished)) { - FstBuilderNode *bn = NULL; + FstBuilderNode* bn = NULL; if (addr == NONE_ADDRESS) { bn = fstUnFinishedNodesPopEmpty(b->unfinished); } else { @@ -881,77 +822,71 @@ void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { addr = fstBuilderCompile(b, bn); fstBuilderNodeDestroy(bn); - assert(addr != NONE_ADDRESS); - //fstBuilderNodeDestroy(n); + assert(addr != NONE_ADDRESS); + // fstBuilderNodeDestroy(n); } fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); - return; + return; } -CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn) { - if (FST_BUILDER_NODE_IS_FINAL(bn) - && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) - && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) { - return EMPTY_ADDRESS; +CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) { + if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) { + return EMPTY_ADDRESS; } - FstRegistryEntry *entry = fstRegistryGetEntry(b->registry, bn); - if (entry->state == FOUND) { + FstRegistryEntry* entry = fstRegistryGetEntry(b->registry, bn); + if (entry->state == FOUND) { CompiledAddr ret = entry->addr; fstRegistryEntryDestroy(entry); return ret; - } + } CompiledAddr startAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt)); - fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); - b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); - if (entry->state == NOTFOUND) { - FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); - } + fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); + b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); + if (entry->state == NOTFOUND) { FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); } fstRegistryEntryDestroy(entry); - - return b->lastAddr; + + return b->lastAddr; } -void* fstBuilderInsertInner(FstBuilder *b) { - fstBuilderCompileFrom(b, 0); - FstBuilderNode *rootNode = fstUnFinishedNodesPopRoot(b->unfinished); - CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); +void* fstBuilderInsertInner(FstBuilder* b) { + fstBuilderCompileFrom(b, 0); + FstBuilderNode* rootNode = fstUnFinishedNodesPopRoot(b->unfinished); + CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); fstBuilderNodeDestroy(rootNode); - char buf64[8] = {0}; + char buf64[8] = {0}; + + void* pBuf64 = buf64; + taosEncodeFixedU64(&pBuf64, b->len); + fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); - void *pBuf64 = buf64; - taosEncodeFixedU64(&pBuf64, b->len); - fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); - pBuf64 = buf64; - taosEncodeFixedU64(&pBuf64, rootAddr); - fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); + taosEncodeFixedU64(&pBuf64, rootAddr); + fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); - char buf32[4] = {0}; - void *pBuf32 = buf32; + char buf32[4] = {0}; + void* pBuf32 = buf32; uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt); - taosEncodeFixedU32(&pBuf32, sum); - fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32)); - + taosEncodeFixedU32(&pBuf32, sum); + fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32)); + fstCountingWriterFlush(b->wrt); - //fstCountingWriterDestroy(b->wrt); - //b->wrt = NULL; + // fstCountingWriterDestroy(b->wrt); + // b->wrt = NULL; return b->wrt; } -void fstBuilderFinish(FstBuilder *b) { +void fstBuilderFinish(FstBuilder* b) { fstBuilderInsertInner(b); } - - -FstSlice fstNodeAsSlice(FstNode *node) { - FstSlice *slice = &node->data; - FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1); - return s; +FstSlice fstNodeAsSlice(FstNode* node) { + FstSlice* slice = &node->data; + FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1); + return s; } -FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out) { - FstLastTransition *trn = malloc(sizeof(FstLastTransition)); +FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { + FstLastTransition* trn = malloc(sizeof(FstLastTransition)); if (trn == NULL) { return NULL; } trn->inp = inp; @@ -959,194 +894,176 @@ FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out) { return trn; } -void fstLastTransitionDestroy(FstLastTransition *trn) { +void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); } -void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *unNode, CompiledAddr addr) { - FstLastTransition *trn = unNode->last; - if (trn == NULL) { return; } - FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; - taosArrayPush(unNode->node->trans, &t); - fstLastTransitionDestroy(trn); +void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { + FstLastTransition* trn = unNode->last; + if (trn == NULL) { return; } + FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; + taosArrayPush(unNode->node->trans, &t); + fstLastTransitionDestroy(trn); unNode->last = NULL; return; } -void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *unNode, Output out) { - if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { - unNode->node->finalOutput += out; - } +void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) { + if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { unNode->node->finalOutput += out; } size_t sz = taosArrayGetSize(unNode->node->trans); for (size_t i = 0; i < sz; i++) { - FstTransition *trn = taosArrayGet(unNode->node->trans, i); + FstTransition* trn = taosArrayGet(unNode->node->trans, i); trn->out += out; } - if (unNode->last) { - unNode->last->out += out; - } + if (unNode->last) { unNode->last->out += out; } return; } -Fst* fstCreate(FstSlice *slice) { +Fst* fstCreate(FstSlice* slice) { int32_t slen; - char *buf = fstSliceData(slice, &slen); - if (slen < 36) { - return NULL; - } + char* buf = fstSliceData(slice, &slen); + if (slen < 36) { return NULL; } uint64_t len = slen; - uint64_t skip = 0; + uint64_t skip = 0; - uint64_t version; - taosDecodeFixedU64(buf, &version); - skip += sizeof(version); - if (version == 0 || version > VERSION) { - return NULL; - } + uint64_t version; + taosDecodeFixedU64(buf, &version); + skip += sizeof(version); + if (version == 0 || version > VERSION) { return NULL; } uint64_t type; taosDecodeFixedU64(buf + skip, &type); - skip += sizeof(type); + skip += sizeof(type); uint32_t checkSum = 0; len -= sizeof(checkSum); - taosDecodeFixedU32(buf + len, &checkSum); + taosDecodeFixedU32(buf + len, &checkSum); CompiledAddr rootAddr; - len -= sizeof(rootAddr); - taosDecodeFixedU64(buf + len, &rootAddr); + len -= sizeof(rootAddr); + taosDecodeFixedU64(buf + len, &rootAddr); - uint64_t fstLen; - len -= sizeof(fstLen); + uint64_t fstLen; + len -= sizeof(fstLen); taosDecodeFixedU64(buf + len, &fstLen); - //TODO(validate root addr) - Fst *fst= (Fst *)calloc(1, sizeof(Fst)); - if (fst == NULL) { return NULL; } - - fst->meta = (FstMeta *)malloc(sizeof(FstMeta)); - if (NULL == fst->meta) { - goto FST_CREAT_FAILED; - } + // TODO(validate root addr) + Fst* fst = (Fst*)calloc(1, sizeof(Fst)); + if (fst == NULL) { return NULL; } + + fst->meta = (FstMeta*)malloc(sizeof(FstMeta)); + if (NULL == fst->meta) { goto FST_CREAT_FAILED; } - fst->meta->version = version; - fst->meta->rootAddr = rootAddr; - fst->meta->ty = type; - fst->meta->len = fstLen; + fst->meta->version = version; + fst->meta->rootAddr = rootAddr; + fst->meta->ty = type; + fst->meta->len = fstLen; fst->meta->checkSum = checkSum; - FstSlice *s = calloc(1, sizeof(FstSlice)); - *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice)); - fst->data = s; - + FstSlice* s = calloc(1, sizeof(FstSlice)); + *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice)); + fst->data = s; + return fst; -FST_CREAT_FAILED: - free(fst->meta); +FST_CREAT_FAILED: + free(fst->meta); free(fst); - } -void fstDestroy(Fst *fst) { - if (fst) { - free(fst->meta); +void fstDestroy(Fst* fst) { + if (fst) { + free(fst->meta); fstSliceDestroy(fst->data); free(fst->data); - } - free(fst); + } + free(fst); } -bool fstGet(Fst *fst, FstSlice *b, Output *out) { - FstNode *root = fstGetRoot(fst); - Output tOut = 0; - int32_t len; - uint8_t *data = fstSliceData(b, &len); +bool fstGet(Fst* fst, FstSlice* b, Output* out) { + FstNode* root = fstGetRoot(fst); + Output tOut = 0; + int32_t len; + uint8_t* data = fstSliceData(b, &len); - SArray *nodes = (SArray *)taosArrayInit(len, sizeof(FstNode *)); + SArray* nodes = (SArray*)taosArrayInit(len, sizeof(FstNode*)); taosArrayPush(nodes, &root); for (uint32_t i = 0; i < len; i++) { uint8_t inp = data[i]; Output res = 0; - if (false == fstNodeFindInput(root, inp, &res)) { - return false; - } + if (false == fstNodeFindInput(root, inp, &res)) { return false; } - FstTransition trn; + FstTransition trn; fstNodeGetTransitionAt(root, res, &trn); - tOut += trn.out; + tOut += trn.out; root = fstGetNode(fst, trn.addr); taosArrayPush(nodes, &root); } if (!FST_NODE_IS_FINAL(root)) { return false; } else { - tOut = tOut + FST_NODE_FINAL_OUTPUT(root); + tOut = tOut + FST_NODE_FINAL_OUTPUT(root); } for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { - FstNode **node = (FstNode **)taosArrayGet(nodes, i); - fstNodeDestroy(*node); + FstNode** node = (FstNode**)taosArrayGet(nodes, i); + fstNodeDestroy(*node); } taosArrayDestroy(nodes); fst->root = NULL; *out = tOut; - - return true; + + return true; } -FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx) { +FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) { return fstStreamBuilderCreate(fst, ctx); } -StreamWithState* streamBuilderIntoStream(FstStreamBuilder *sb) { - if (sb == NULL) { return NULL; } +StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) { + if (sb == NULL) { return NULL; } return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); } -FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx) { +FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { return fstStreamBuilderCreate(fst, ctx); } -FstNode *fstGetRoot(Fst *fst) { - if (fst->root != NULL) { - return fst->root; - } - CompiledAddr rAddr = fstGetRootAddr(fst); - fst->root = fstGetNode(fst, rAddr); +FstNode* fstGetRoot(Fst* fst) { + if (fst->root != NULL) { return fst->root; } + CompiledAddr rAddr = fstGetRootAddr(fst); + fst->root = fstGetNode(fst, rAddr); return fst->root; } -FstNode* fstGetNode(Fst *fst, CompiledAddr addr) { - return fstNodeCreate(fst->meta->version, addr, fst->data); - +FstNode* fstGetNode(Fst* fst, CompiledAddr addr) { + return fstNodeCreate(fst->meta->version, addr, fst->data); } -FstType fstGetType(Fst *fst) { +FstType fstGetType(Fst* fst) { return fst->meta->ty; } -CompiledAddr fstGetRootAddr(Fst *fst) { +CompiledAddr fstGetRootAddr(Fst* fst) { return fst->meta->rootAddr; -} +} -Output fstEmptyFinalOutput(Fst *fst, bool *null) { - Output res = 0; - FstNode *node = fstGetRoot(fst); +Output fstEmptyFinalOutput(Fst* fst, bool* null) { + Output res = 0; + FstNode* node = fstGetRoot(fst); if (FST_NODE_IS_FINAL(node)) { *null = false; - res = FST_NODE_FINAL_OUTPUT(node); + res = FST_NODE_FINAL_OUTPUT(node); } else { *null = true; } return res; } -bool fstVerify(Fst *fst) { +bool fstVerify(Fst* fst) { uint32_t checkSum = fst->meta->checkSum; - int32_t len; - uint8_t *data = fstSliceData(fst->data, &len); - TSCKSUM initSum = 0; - if (!taosCheckChecksumWhole(data, len)) { - return false; - } + int32_t len; + uint8_t* data = fstSliceData(fst->data, &len); + TSCKSUM initSum = 0; + if (!taosCheckChecksumWhole(data, len)) { return false; } return true; } // data bound function -FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data) { - FstBoundWithData *b = calloc(1, sizeof(FstBoundWithData)); +FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) { + FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData)); if (b == NULL) { return NULL; } if (data != NULL) { @@ -1156,11 +1073,10 @@ FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data) { } b->type = type; - return b; + return b; } - -bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) { +bool fstBoundWithDataExceededBy(FstBoundWithData* bound, FstSlice* slice) { int comp = fstSliceCompare(slice, &bound->data); if (bound->type == Included) { return comp > 0 ? true : false; @@ -1170,65 +1086,61 @@ bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice) { return true; } } -bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) { +bool fstBoundWithDataIsEmpty(FstBoundWithData* bound) { if (bound->type == Unbounded) { return true; - } else { - return fstSliceIsEmpty(&bound->data); - } + } else { + return fstSliceIsEmpty(&bound->data); + } } - -bool fstBoundWithDataIsIncluded(FstBoundWithData *bound) { - return bound->type == Excluded? false : true; +bool fstBoundWithDataIsIncluded(FstBoundWithData* bound) { + return bound->type == Excluded ? false : true; } -void fstBoundDestroy(FstBoundWithData *bound) { +void fstBoundDestroy(FstBoundWithData* bound) { free(bound); } -StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) { - StreamWithState *sws = calloc(1, sizeof(StreamWithState)); - if (sws == NULL) { return NULL; } +StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, FstBoundWithData* max) { + StreamWithState* sws = calloc(1, sizeof(StreamWithState)); + if (sws == NULL) { return NULL; } + + sws->fst = fst; + sws->aut = automation; + sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t)); - sws->fst = fst; - sws->aut = automation; - sws->inp = (SArray *)taosArrayInit(256, sizeof(uint8_t)); - sws->emptyOutput.null = false; - sws->emptyOutput.out = 0; + sws->emptyOutput.out = 0; - sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); - sws->endAt = max; + sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); + sws->endAt = max; streamWithStateSeekMin(sws, min); return sws; } -void streamWithStateDestroy(StreamWithState *sws) { +void streamWithStateDestroy(StreamWithState* sws) { if (sws == NULL) { return; } taosArrayDestroy(sws->inp); taosArrayDestroyEx(sws->stack, streamStateDestroy); - free(sws); + free(sws); } -bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { - - AutomationCtx *aut = sws->aut; +bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) { + AutomationCtx* aut = sws->aut; if (fstBoundWithDataIsEmpty(min)) { - if (fstBoundWithDataIsIncluded(min)) { - sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); - } - StreamState s = {.node = fstGetRoot(sws->fst), - .trans = 0, - .out = {.null = false, .out = 0}, - .autState = automFuncs[aut->type].start(aut)}; // auto.start callback + if (fstBoundWithDataIsIncluded(min)) { sws->emptyOutput.out = fstEmptyFinalOutput(sws->fst, &(sws->emptyOutput.null)); } + StreamState s = {.node = fstGetRoot(sws->fst), + .trans = 0, + .out = {.null = false, .out = 0}, + .autState = automFuncs[aut->type].start(aut)}; // auto.start callback taosArrayPush(sws->stack, &s); return true; - } - FstSlice *key = NULL; - bool inclusize = false;; + } + FstSlice* key = NULL; + bool inclusize = false; if (min->type == Included) { key = &min->data; @@ -1239,204 +1151,186 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { return false; } - FstNode *node = fstGetRoot(sws->fst); - Output out = 0; - //void* autState = sws->aut->start(); + FstNode* node = fstGetRoot(sws->fst); + Output out = 0; + // void* autState = sws->aut->start(); void* autState = automFuncs[aut->type].start(aut); - int32_t len; - uint8_t *data = fstSliceData(key, &len); + int32_t len; + uint8_t* data = fstSliceData(key, &len); for (uint32_t i = 0; i < len; i++) { - uint8_t b = data[i]; + uint8_t b = data[i]; uint64_t res = 0; - bool null = fstNodeFindInput(node, b, &res); + bool null = fstNodeFindInput(node, b, &res); if (null == false) { FstTransition trn; - fstNodeGetTransitionAt(node, res, &trn); - void *preState = autState; + fstNodeGetTransitionAt(node, res, &trn); + void* preState = autState; // autState = sws->aut->accept(preState, b); autState = automFuncs[aut->type].accept(aut, preState, b); taosArrayPush(sws->inp, &b); - StreamState s = {.node = node, - .trans = res + 1, - .out = {.null = false, .out = out}, - .autState = preState}; + StreamState s = {.node = node, .trans = res + 1, .out = {.null = false, .out = out}, .autState = preState}; taosArrayPush(sws->stack, &s); out += trn.out; - node = fstGetNode(sws->fst, trn.addr); + node = fstGetNode(sws->fst, trn.addr); fstNodeDestroy(node); } else { - // This is a little tricky. We're in this case if the // given bound is not a prefix of any key in the FST. // Since this is a minimum bound, we need to find the // first transition in this node that proceeds the current - // input byte. - FstTransitions *trans = fstNodeTransitions(node); - uint64_t i = 0; + // input byte. + FstTransitions* trans = fstNodeTransitions(node); + uint64_t i = 0; for (i = trans->range.start; i < trans->range.end; i++) { FstTransition trn; - if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { - break; - } + if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { break; } } - - StreamState s = {.node = node, - .trans = i, - .out = {.null = false, .out = out}, - .autState = autState}; - taosArrayPush(sws->stack, &s); - return true; + + StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; + taosArrayPush(sws->stack, &s); + return true; } } - uint32_t sz = taosArrayGetSize(sws->stack); + uint32_t sz = taosArrayGetSize(sws->stack); if (sz != 0) { - StreamState *s = taosArrayGet(sws->stack, sz - 1); + StreamState* s = taosArrayGet(sws->stack, sz - 1); if (inclusize) { s->trans -= 1; taosArrayPop(sws->inp); } else { - FstNode *n = s->node; - uint64_t trans = s->trans; - FstTransition trn; + FstNode* n = s->node; + uint64_t trans = s->trans; + FstTransition trn; fstNodeGetTransitionAt(n, trans - 1, &trn); - StreamState s = {.node = fstGetNode(sws->fst, trn.addr), - .trans= 0, - .out = {.null = false, .out = out}, - .autState = autState}; + StreamState s = {.node = fstGetNode(sws->fst, trn.addr), .trans = 0, .out = {.null = false, .out = out}, .autState = autState}; taosArrayPush(sws->stack, &s); - return true; - } - return false; - } -} - -StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback) { - AutomationCtx *aut = sws->aut; - FstOutput output = sws->emptyOutput; + return true; + } + return false; + } +} + +StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback) { + AutomationCtx* aut = sws->aut; + FstOutput output = sws->emptyOutput; if (output.null == false) { - FstSlice emptySlice = fstSliceCreate(NULL, 0); + FstSlice emptySlice = fstSliceCreate(NULL, 0); if (fstBoundWithDataExceededBy(sws->endAt, &emptySlice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); - sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); return NULL; } - void *start = automFuncs[aut->type].start(aut); + void* start = automFuncs[aut->type].start(aut); if (automFuncs[aut->type].isMatch(aut, start)) { FstSlice s = fstSliceCreate(NULL, 0); return swsResultCreate(&s, output, callback(start)); } } - SArray *nodes = taosArrayInit(8, sizeof(FstNode *)); + SArray* nodes = taosArrayInit(8, sizeof(FstNode*)); while (taosArrayGetSize(sws->stack) > 0) { - StreamState *p = (StreamState *)taosArrayPop(sws->stack); + StreamState* p = (StreamState*)taosArrayPop(sws->stack); if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { - if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { - taosArrayPop(sws->inp); - } - streamStateDestroy(p); + if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); } + streamStateDestroy(p); continue; } - FstTransition trn; + FstTransition trn; fstNodeGetTransitionAt(p->node, p->trans, &trn); - Output out = p->out.out + trn.out; - void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); - void* tState = callback(nextState); - bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); - FstNode *nextNode = fstGetNode(sws->fst, trn.addr); - taosArrayPush(nodes, &nextNode); - taosArrayPush(sws->inp, &(trn.inp)); + Output out = p->out.out + trn.out; + void* nextState = automFuncs[aut->type].accept(aut, p->autState, trn.inp); + void* tState = callback(nextState); + bool isMatch = automFuncs[aut->type].isMatch(aut, nextState); + FstNode* nextNode = fstGetNode(sws->fst, trn.addr); + taosArrayPush(nodes, &nextNode); + taosArrayPush(sws->inp, &(trn.inp)); if (FST_NODE_IS_FINAL(nextNode)) { - //void *eofState = sws->aut->acceptEof(nextState); - void *eofState = automFuncs[aut->type].acceptEof(aut, nextState); - if (eofState != NULL) { - isMatch = automFuncs[aut->type].isMatch(aut, eofState); - } - } - StreamState s1 = { .node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; + // void *eofState = sws->aut->acceptEof(nextState); + void* eofState = automFuncs[aut->type].acceptEof(aut, nextState); + if (eofState != NULL) { isMatch = automFuncs[aut->type].isMatch(aut, eofState); } + } + StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; taosArrayPush(sws->stack, &s1); StreamState s2 = {.node = nextNode, .trans = 0, .out = {.null = false, .out = out}, .autState = nextState}; taosArrayPush(sws->stack, &s2); - - size_t isz = taosArrayGetSize(sws->inp); - uint8_t *buf = (uint8_t *)malloc(isz * sizeof(uint8_t)); + size_t isz = taosArrayGetSize(sws->inp); + uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t)); for (uint32_t i = 0; i < isz; i++) { - buf[i] = *(uint8_t *)taosArrayGet(sws->inp, i); + buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); - sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); free(buf); fstSliceDestroy(&slice); return NULL; } if (FST_NODE_IS_FINAL(nextNode) && isMatch) { - FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; - StreamWithStateResult *result = swsResultCreate(&slice, fOutput, tState); + FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; + StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); free(buf); fstSliceDestroy(&slice); - return result; + return result; } free(buf); fstSliceDestroy(&slice); } for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { - FstNode** node = (FstNode **)taosArrayGet(nodes, i); + FstNode** node = (FstNode**)taosArrayGet(nodes, i); fstNodeDestroy(*node); } taosArrayDestroy(nodes); - return NULL; - + return NULL; } -StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state) { - StreamWithStateResult *result = calloc(1, sizeof(StreamWithStateResult)); +StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) { + StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult)); if (result == NULL) { return NULL; } - - result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); - result->out = fOut; - result->state = state; + + result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); + result->out = fOut; + result->state = state; return result; } -void swsResultDestroy(StreamWithStateResult *result) { +void swsResultDestroy(StreamWithStateResult* result) { if (NULL == result) { return; } - + fstSliceDestroy(&result->data); - startWithStateValueDestroy(result->state); + startWithStateValueDestroy(result->state); free(result); } -void streamStateDestroy(void *s) { +void streamStateDestroy(void* s) { if (NULL == s) { return; } - StreamState *ss = (StreamState *)s; + StreamState* ss = (StreamState*)s; fstNodeDestroy(ss->node); - //free(s->autoState); + // free(s->autoState); } -FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut) { - FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder)); +FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { + FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder)); if (NULL == b) { return NULL; } b->fst = fst; b->aut = aut; b->min = fstBoundStateCreate(Unbounded, NULL); - b->max = fstBoundStateCreate(Unbounded, NULL); + b->max = fstBoundStateCreate(Unbounded, NULL); return b; } -void fstStreamBuilderDestroy(FstStreamBuilder *b) { +void fstStreamBuilderDestroy(FstStreamBuilder* b) { fstSliceDestroy(&b->min->data); - tfree(b->min); + tfree(b->min); fstSliceDestroy(&b->max->data); tfree(b->max); free(b); } -FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) { +FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) { if (b == NULL) { return NULL; } if (type == GE) { @@ -1458,9 +1352,3 @@ FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, Rang } return b; } - - - - - - diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 07ad45079b172cc94fe90e5b84826156f098e857..eb4c479c9406d4be367df458235c47f7b87ff9f8 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -15,166 +15,149 @@ #include "index_fst_automation.h" - -StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val) { - StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); +StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { + StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); if (nsv == NULL) { return NULL; } nsv->kind = kind; nsv->type = ty; if (ty == FST_INT) { - nsv->val = *(int *)val; + nsv->val = *(int*)val; } else if (ty == FST_CHAR) { - size_t len = strlen((char *)val); - nsv->ptr = (char *)calloc(1, len + 1); + size_t len = strlen((char*)val); + nsv->ptr = (char*)calloc(1, len + 1); memcpy(nsv->ptr, val, len); } else if (ty == FST_ARRAY) { - //TODO, - //nsv->arr = taosArrayFromList() + // TODO, + // nsv->arr = taosArrayFromList() } return nsv; } -void startWithStateValueDestroy(void *val) { - StartWithStateValue *sv = (StartWithStateValue *)val; +void startWithStateValueDestroy(void* val) { + StartWithStateValue* sv = (StartWithStateValue*)val; if (sv == NULL) { return; } if (sv->type == FST_INT) { - // + // } else if (sv->type == FST_CHAR) { free(sv->ptr); } else if (sv->type == FST_ARRAY) { taosArrayDestroy(sv->arr); } - free(sv); + free(sv); } -StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv) { - StartWithStateValue *nsv = calloc(1, sizeof(StartWithStateValue)); +StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { + StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); if (nsv == NULL) { return NULL; } nsv->kind = sv->kind; - nsv->type= sv->type; + nsv->type = sv->type; if (nsv->type == FST_INT) { nsv->val = sv->val; } else if (nsv->type == FST_CHAR) { size_t len = strlen(sv->ptr); - nsv->ptr = (char *)calloc(1, len + 1); + nsv->ptr = (char*)calloc(1, len + 1); memcpy(nsv->ptr, sv->ptr, len); } else if (nsv->type == FST_ARRAY) { + // } return nsv; } - // prefix query, impl later -static void* prefixStart(AutomationCtx *ctx) { - StartWithStateValue *data = (StartWithStateValue *)(ctx->stdata); - return startWithStateValueDump(data); +static void* prefixStart(AutomationCtx* ctx) { + StartWithStateValue* data = (StartWithStateValue*)(ctx->stdata); + return startWithStateValueDump(data); }; -static bool prefixIsMatch(AutomationCtx *ctx, void *sv) { - StartWithStateValue* ssv = (StartWithStateValue *)sv; - return ssv->val == strlen(ctx->data); -} -static bool prefixCanMatch(AutomationCtx *ctx, void *sv) { - StartWithStateValue* ssv = (StartWithStateValue *)sv; - return ssv->val >= 0; -} -static bool prefixWillAlwaysMatch(AutomationCtx *ctx, void *state) { +static bool prefixIsMatch(AutomationCtx* ctx, void* sv) { + StartWithStateValue* ssv = (StartWithStateValue*)sv; + return ssv->val == strlen(ctx->data); +} +static bool prefixCanMatch(AutomationCtx* ctx, void* sv) { + StartWithStateValue* ssv = (StartWithStateValue*)sv; + return ssv->val >= 0; +} +static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } -static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { - StartWithStateValue* ssv = (StartWithStateValue *)state; - if (ssv == NULL || ctx == NULL) {return NULL;} +static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { + StartWithStateValue* ssv = (StartWithStateValue*)state; + if (ssv == NULL || ctx == NULL) { return NULL; } - char *data = ctx->data; - if (ssv->kind == Done) { - return startWithStateValueCreate(Done, FST_INT, &ssv->val); - } + char* data = ctx->data; + if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); } if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { - int val = ssv->val + 1; - StartWithStateValue *nsv = startWithStateValueCreate(Running, FST_INT, &val); + int val = ssv->val + 1; + StartWithStateValue* nsv = startWithStateValueCreate(Running, FST_INT, &val); if (prefixIsMatch(ctx, nsv)) { nsv->kind = Done; } else { nsv->kind = Running; - } + } return nsv; - } + } return NULL; } -static void* prefixAcceptEof(AutomationCtx *ctx, void *state) { +static void* prefixAcceptEof(AutomationCtx* ctx, void* state) { return NULL; } // pattern query, impl later -static void* patternStart(AutomationCtx *ctx) { +static void* patternStart(AutomationCtx* ctx) { return NULL; } -static bool patternIsMatch(AutomationCtx *ctx, void *data) { +static bool patternIsMatch(AutomationCtx* ctx, void* data) { return true; -} -static bool patternCanMatch(AutomationCtx *ctx, void *data) { +} +static bool patternCanMatch(AutomationCtx* ctx, void* data) { return true; -} -static bool patternWillAlwaysMatch(AutomationCtx *ctx, void *state) { +} +static bool patternWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } -static void* patternAccept(AutomationCtx *ctx, void *state, uint8_t byte) { +static void* patternAccept(AutomationCtx* ctx, void* state, uint8_t byte) { return NULL; } -static void* patternAcceptEof(AutomationCtx *ctx, void *state) { +static void* patternAcceptEof(AutomationCtx* ctx, void* state) { return NULL; } -AutomationFunc automFuncs[] = {{ - prefixStart, - prefixIsMatch, - prefixCanMatch, - prefixWillAlwaysMatch, - prefixAccept, - prefixAcceptEof - }, - { - patternStart, - patternIsMatch, - patternCanMatch, - patternWillAlwaysMatch, - patternAccept, - patternAcceptEof - } - // add more search type +AutomationFunc automFuncs[] = { + {prefixStart, prefixIsMatch, prefixCanMatch, prefixWillAlwaysMatch, prefixAccept, prefixAcceptEof}, + {patternStart, patternIsMatch, patternCanMatch, patternWillAlwaysMatch, patternAccept, patternAcceptEof} + // add more search type }; -AutomationCtx* automCtxCreate(void *data,AutomationType atype) { - AutomationCtx *ctx = calloc(1, sizeof(AutomationCtx)); +AutomationCtx* automCtxCreate(void* data, AutomationType atype) { + AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx)); if (ctx == NULL) { return NULL; } - StartWithStateValue *sv = NULL; + StartWithStateValue* sv = NULL; if (atype == AUTOMATION_PREFIX) { int val = 0; sv = startWithStateValueCreate(Running, FST_INT, &val); - ctx->stdata = (void *)sv; + ctx->stdata = (void*)sv; } else if (atype == AUTMMATION_MATCH) { - } else { // add more search type } - char* src = (char *)data; + char* src = (char*)data; size_t len = strlen(src); - char* dst = (char *)malloc(len * sizeof(char) + 1); + char* dst = (char*)malloc(len * sizeof(char) + 1); memcpy(dst, src, len); dst[len] = 0; - - ctx->data = dst; - ctx->type = atype; - ctx->stdata = (void *)sv; - return ctx; -} -void automCtxDestroy(AutomationCtx *ctx) { + + ctx->data = dst; + ctx->type = atype; + ctx->stdata = (void*)sv; + return ctx; +} +void automCtxDestroy(AutomationCtx* ctx) { startWithStateValueDestroy(ctx->stdata); free(ctx->data); free(ctx); diff --git a/source/libs/index/src/index_fst_common.c b/source/libs/index/src/index_fst_common.c index 97fb88d60eb44ea684bb949b8f304535059074e7..2d7586e8d939c8ef6dc80e998e97f3cf0b08f921 100644 --- a/source/libs/index/src/index_fst_common.c +++ b/source/libs/index/src/index_fst_common.c @@ -12,296 +12,282 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - -#include "tutil.h" +#include "index_fst_common.h" const uint8_t COMMON_INPUTS[] = { - 84, // '\x00' - 85, // '\x01' - 86, // '\x02' - 87, // '\x03' - 88, // '\x04' - 89, // '\x05' - 90, // '\x06' - 91, // '\x07' - 92, // '\x08' - 93, // '\t' - 94, // '\n' - 95, // '\x0b' - 96, // '\x0c' - 97, // '\r' - 98, // '\x0e' - 99, // '\x0f' - 100, // '\x10' - 101, // '\x11' - 102, // '\x12' - 103, // '\x13' - 104, // '\x14' - 105, // '\x15' - 106, // '\x16' - 107, // '\x17' - 108, // '\x18' - 109, // '\x19' - 110, // '\x1a' - 111, // '\x1b' - 112, // '\x1c' - 113, // '\x1d' - 114, // '\x1e' - 115, // '\x1f' - 116, // ' ' - 80, // '!' - 117, // '"' - 118, // '#' - 79, // '$' - 39, // '%' - 30, // '&' - 81, // "'" - 75, // '(' - 74, // ')' - 82, // '*' - 57, // '+' - 66, // ',' - 16, // '-' - 12, // '.' - 2, // '/' - 19, // '0' - 20, // '1' - 21, // '2' - 27, // '3' - 32, // '4' - 29, // '5' - 35, // '6' - 36, // '7' - 37, // '8' - 34, // '9' - 24, // ':' - 73, // ';' - 119, // '<' - 23, // '=' - 120, // '>' - 40, // '?' - 83, // '@' - 44, // 'A' - 48, // 'B' - 42, // 'C' - 43, // 'D' - 49, // 'E' - 46, // 'F' - 62, // 'G' - 61, // 'H' - 47, // 'I' - 69, // 'J' - 68, // 'K' - 58, // 'L' - 56, // 'M' - 55, // 'N' - 59, // 'O' - 51, // 'P' - 72, // 'Q' - 54, // 'R' - 45, // 'S' - 52, // 'T' - 64, // 'U' - 65, // 'V' - 63, // 'W' - 71, // 'X' - 67, // 'Y' - 70, // 'Z' - 77, // '[' - 121, // '\\' - 78, // ']' - 122, // '^' - 31, // '_' - 123, // '`' - 4, // 'a' - 25, // 'b' - 9, // 'c' - 17, // 'd' - 1, // 'e' - 26, // 'f' - 22, // 'g' - 13, // 'h' - 7, // 'i' - 50, // 'j' - 38, // 'k' - 14, // 'l' - 15, // 'm' - 10, // 'n' - 3, // 'o' - 8, // 'p' - 60, // 'q' - 6, // 'r' - 5, // 's' - 0, // 't' - 18, // 'u' - 33, // 'v' - 11, // 'w' - 41, // 'x' - 28, // 'y' - 53, // 'z' - 124, // '{' - 125, // '|' - 126, // '}' - 76, // '~' - 127, // '\x7f' - 128, // '\x80' - 129, // '\x81' - 130, // '\x82' - 131, // '\x83' - 132, // '\x84' - 133, // '\x85' - 134, // '\x86' - 135, // '\x87' - 136, // '\x88' - 137, // '\x89' - 138, // '\x8a' - 139, // '\x8b' - 140, // '\x8c' - 141, // '\x8d' - 142, // '\x8e' - 143, // '\x8f' - 144, // '\x90' - 145, // '\x91' - 146, // '\x92' - 147, // '\x93' - 148, // '\x94' - 149, // '\x95' - 150, // '\x96' - 151, // '\x97' - 152, // '\x98' - 153, // '\x99' - 154, // '\x9a' - 155, // '\x9b' - 156, // '\x9c' - 157, // '\x9d' - 158, // '\x9e' - 159, // '\x9f' - 160, // '\xa0' - 161, // '¡' - 162, // '¢' - 163, // '£' - 164, // '¤' - 165, // '¥' - 166, // '¦' - 167, // '§' - 168, // '¨' - 169, // '©' - 170, // 'ª' - 171, // '«' - 172, // '¬' - 173, // '\xad' - 174, // '®' - 175, // '¯' - 176, // '°' - 177, // '±' - 178, // '²' - 179, // '³' - 180, // '´' - 181, // 'µ' - 182, // '¶' - 183, // '·' - 184, // '¸' - 185, // '¹' - 186, // 'º' - 187, // '»' - 188, // '¼' - 189, // '½' - 190, // '¾' - 191, // '¿' - 192, // 'À' - 193, // 'Á' - 194, // 'Â' - 195, // 'Ã' - 196, // 'Ä' - 197, // 'Å' - 198, // 'Æ' - 199, // 'Ç' - 200, // 'È' - 201, // 'É' - 202, // 'Ê' - 203, // 'Ë' - 204, // 'Ì' - 205, // 'Í' - 206, // 'Î' - 207, // 'Ï' - 208, // 'Ð' - 209, // 'Ñ' - 210, // 'Ò' - 211, // 'Ó' - 212, // 'Ô' - 213, // 'Õ' - 214, // 'Ö' - 215, // '×' - 216, // 'Ø' - 217, // 'Ù' - 218, // 'Ú' - 219, // 'Û' - 220, // 'Ü' - 221, // 'Ý' - 222, // 'Þ' - 223, // 'ß' - 224, // 'à' - 225, // 'á' - 226, // 'â' - 227, // 'ã' - 228, // 'ä' - 229, // 'å' - 230, // 'æ' - 231, // 'ç' - 232, // 'è' - 233, // 'é' - 234, // 'ê' - 235, // 'ë' - 236, // 'ì' - 237, // 'í' - 238, // 'î' - 239, // 'ï' - 240, // 'ð' - 241, // 'ñ' - 242, // 'ò' - 243, // 'ó' - 244, // 'ô' - 245, // 'õ' - 246, // 'ö' - 247, // '÷' - 248, // 'ø' - 249, // 'ù' - 250, // 'ú' - 251, // 'û' - 252, // 'ü' - 253, // 'ý' - 254, // 'þ' - 255, // 'ÿ' + 84, // '\x00' + 85, // '\x01' + 86, // '\x02' + 87, // '\x03' + 88, // '\x04' + 89, // '\x05' + 90, // '\x06' + 91, // '\x07' + 92, // '\x08' + 93, // '\t' + 94, // '\n' + 95, // '\x0b' + 96, // '\x0c' + 97, // '\r' + 98, // '\x0e' + 99, // '\x0f' + 100, // '\x10' + 101, // '\x11' + 102, // '\x12' + 103, // '\x13' + 104, // '\x14' + 105, // '\x15' + 106, // '\x16' + 107, // '\x17' + 108, // '\x18' + 109, // '\x19' + 110, // '\x1a' + 111, // '\x1b' + 112, // '\x1c' + 113, // '\x1d' + 114, // '\x1e' + 115, // '\x1f' + 116, // ' ' + 80, // '!' + 117, // '"' + 118, // '#' + 79, // '$' + 39, // '%' + 30, // '&' + 81, // "'" + 75, // '(' + 74, // ')' + 82, // '*' + 57, // '+' + 66, // ',' + 16, // '-' + 12, // '.' + 2, // '/' + 19, // '0' + 20, // '1' + 21, // '2' + 27, // '3' + 32, // '4' + 29, // '5' + 35, // '6' + 36, // '7' + 37, // '8' + 34, // '9' + 24, // ':' + 73, // ';' + 119, // '<' + 23, // '=' + 120, // '>' + 40, // '?' + 83, // '@' + 44, // 'A' + 48, // 'B' + 42, // 'C' + 43, // 'D' + 49, // 'E' + 46, // 'F' + 62, // 'G' + 61, // 'H' + 47, // 'I' + 69, // 'J' + 68, // 'K' + 58, // 'L' + 56, // 'M' + 55, // 'N' + 59, // 'O' + 51, // 'P' + 72, // 'Q' + 54, // 'R' + 45, // 'S' + 52, // 'T' + 64, // 'U' + 65, // 'V' + 63, // 'W' + 71, // 'X' + 67, // 'Y' + 70, // 'Z' + 77, // '[' + 121, // '\\' + 78, // ']' + 122, // '^' + 31, // '_' + 123, // '`' + 4, // 'a' + 25, // 'b' + 9, // 'c' + 17, // 'd' + 1, // 'e' + 26, // 'f' + 22, // 'g' + 13, // 'h' + 7, // 'i' + 50, // 'j' + 38, // 'k' + 14, // 'l' + 15, // 'm' + 10, // 'n' + 3, // 'o' + 8, // 'p' + 60, // 'q' + 6, // 'r' + 5, // 's' + 0, // 't' + 18, // 'u' + 33, // 'v' + 11, // 'w' + 41, // 'x' + 28, // 'y' + 53, // 'z' + 124, // '{' + 125, // '|' + 126, // '}' + 76, // '~' + 127, // '\x7f' + 128, // '\x80' + 129, // '\x81' + 130, // '\x82' + 131, // '\x83' + 132, // '\x84' + 133, // '\x85' + 134, // '\x86' + 135, // '\x87' + 136, // '\x88' + 137, // '\x89' + 138, // '\x8a' + 139, // '\x8b' + 140, // '\x8c' + 141, // '\x8d' + 142, // '\x8e' + 143, // '\x8f' + 144, // '\x90' + 145, // '\x91' + 146, // '\x92' + 147, // '\x93' + 148, // '\x94' + 149, // '\x95' + 150, // '\x96' + 151, // '\x97' + 152, // '\x98' + 153, // '\x99' + 154, // '\x9a' + 155, // '\x9b' + 156, // '\x9c' + 157, // '\x9d' + 158, // '\x9e' + 159, // '\x9f' + 160, // '\xa0' + 161, // '¡' + 162, // '¢' + 163, // '£' + 164, // '¤' + 165, // '¥' + 166, // '¦' + 167, // '§' + 168, // '¨' + 169, // '©' + 170, // 'ª' + 171, // '«' + 172, // '¬' + 173, // '\xad' + 174, // '®' + 175, // '¯' + 176, // '°' + 177, // '±' + 178, // '²' + 179, // '³' + 180, // '´' + 181, // 'µ' + 182, // '¶' + 183, // '·' + 184, // '¸' + 185, // '¹' + 186, // 'º' + 187, // '»' + 188, // '¼' + 189, // '½' + 190, // '¾' + 191, // '¿' + 192, // 'À' + 193, // 'Á' + 194, // 'Â' + 195, // 'Ã' + 196, // 'Ä' + 197, // 'Å' + 198, // 'Æ' + 199, // 'Ç' + 200, // 'È' + 201, // 'É' + 202, // 'Ê' + 203, // 'Ë' + 204, // 'Ì' + 205, // 'Í' + 206, // 'Î' + 207, // 'Ï' + 208, // 'Ð' + 209, // 'Ñ' + 210, // 'Ò' + 211, // 'Ó' + 212, // 'Ô' + 213, // 'Õ' + 214, // 'Ö' + 215, // '×' + 216, // 'Ø' + 217, // 'Ù' + 218, // 'Ú' + 219, // 'Û' + 220, // 'Ü' + 221, // 'Ý' + 222, // 'Þ' + 223, // 'ß' + 224, // 'à' + 225, // 'á' + 226, // 'â' + 227, // 'ã' + 228, // 'ä' + 229, // 'å' + 230, // 'æ' + 231, // 'ç' + 232, // 'è' + 233, // 'é' + 234, // 'ê' + 235, // 'ë' + 236, // 'ì' + 237, // 'í' + 238, // 'î' + 239, // 'ï' + 240, // 'ð' + 241, // 'ñ' + 242, // 'ò' + 243, // 'ó' + 244, // 'ô' + 245, // 'õ' + 246, // 'ö' + 247, // '÷' + 248, // 'ø' + 249, // 'ù' + 250, // 'ú' + 251, // 'û' + 252, // 'ü' + 253, // 'ý' + 254, // 'þ' + 255, // 'ÿ' }; const char COMMON_INPUTS_INV[] = { - 't', 'e', '/', 'o', 'a', 's', 'r', 'i', 'p', 'c', 'n', 'w', - '.', 'h', 'l', 'm', '-', 'd', 'u', '0', '1', '2', 'g', '=', - ':', 'b', 'f', '3', 'y', '5', '&', '_', '4', 'v', '9', '6', - '7', '8', 'k', '%', '?', 'x', 'C', 'D', 'A', 'S', 'F', 'I', - 'B', 'E', 'j', 'P', 'T', 'z', 'R', 'N', 'M', '+', 'L', 'O', - 'q', 'H', 'G', 'W', 'U', 'V', ',', 'Y', 'K', 'J', 'Z', 'X', - 'Q', ';', ')', '(', '~', '[', ']', '$', '!', '\'', '*', '@', - '\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', - '\x08', '\t', '\n', '\x0b', '\x0c', '\r', '\x0e', '\x0f', '\x10', - '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', '\x18', - '\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f', ' ', '"', - '#', '<', '>', '\\', '^', '`', '{', '|', '}','\x7f','\x80', - '\x81', '\x82', '\x83', '\x84', '\x85', '\x86', '\x87', '\x88', - '\x89', '\x8a', '\x8b', '\x8c', '\x8d', '\x8e', '\x8f', '\x90', - '\x91', '\x92', '\x93', '\x94', '\x95', '\x96', '\x97', '\x98', - '\x99', '\x9a', '\x9b', '\x9c', '\x9d', '\x9e', '\x9f', '\xa0', - '\xa1', '\xa2', '\xa3', '\xa4', '\xa5', '\xa6', '\xa7', '\xa8', - '\xa9', '\xaa', '\xab', '\xac', '\xad', '\xae', '\xaf', '\xb0', - '\xb1', '\xb2', '\xb3', '\xb4', '\xb5', '\xb6', '\xb7', '\xb8', - '\xb9', '\xba', '\xbb', '\xbc', '\xbd', '\xbe', '\xbf', '\xc0', - '\xc1', '\xc2', '\xc3', '\xc4', '\xc5', '\xc6', '\xc7', '\xc8', - '\xc9', '\xca', '\xcb', '\xcc', '\xcd', '\xce', '\xcf', '\xd0', - '\xd1', '\xd2', '\xd3', '\xd4', '\xd5', '\xd6', '\xd7', '\xd8', - '\xd9', '\xda', '\xdb', '\xdc', '\xdd', '\xde', '\xdf', '\xe0', - '\xe1', '\xe2', '\xe3', '\xe4', '\xe5', '\xe6', '\xe7', '\xe8', - '\xe9', '\xea', '\xeb', '\xec', '\xed', '\xee', '\xef', '\xf0', - '\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', '\xf8', - '\xf9', '\xfa', '\xfb', '\xfc', '\xfd', '\xfe', '\xff', + 't', 'e', '/', 'o', 'a', 's', 'r', 'i', 'p', 'c', 'n', 'w', '.', 'h', 'l', 'm', + '-', 'd', 'u', '0', '1', '2', 'g', '=', ':', 'b', 'f', '3', 'y', '5', '&', '_', + '4', 'v', '9', '6', '7', '8', 'k', '%', '?', 'x', 'C', 'D', 'A', 'S', 'F', 'I', + 'B', 'E', 'j', 'P', 'T', 'z', 'R', 'N', 'M', '+', 'L', 'O', 'q', 'H', 'G', 'W', + 'U', 'V', ',', 'Y', 'K', 'J', 'Z', 'X', 'Q', ';', ')', '(', '~', '[', ']', '$', + '!', '\'', '*', '@', '\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', '\x08', '\t', '\n', '\x0b', + '\x0c', '\r', '\x0e', '\x0f', '\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', '\x18', '\x19', '\x1a', '\x1b', + '\x1c', '\x1d', '\x1e', '\x1f', ' ', '"', '#', '<', '>', '\\', '^', '`', '{', '|', '}', '\x7f', + '\x80', '\x81', '\x82', '\x83', '\x84', '\x85', '\x86', '\x87', '\x88', '\x89', '\x8a', '\x8b', '\x8c', '\x8d', '\x8e', '\x8f', + '\x90', '\x91', '\x92', '\x93', '\x94', '\x95', '\x96', '\x97', '\x98', '\x99', '\x9a', '\x9b', '\x9c', '\x9d', '\x9e', '\x9f', + '\xa0', '\xa1', '\xa2', '\xa3', '\xa4', '\xa5', '\xa6', '\xa7', '\xa8', '\xa9', '\xaa', '\xab', '\xac', '\xad', '\xae', '\xaf', + '\xb0', '\xb1', '\xb2', '\xb3', '\xb4', '\xb5', '\xb6', '\xb7', '\xb8', '\xb9', '\xba', '\xbb', '\xbc', '\xbd', '\xbe', '\xbf', + '\xc0', '\xc1', '\xc2', '\xc3', '\xc4', '\xc5', '\xc6', '\xc7', '\xc8', '\xc9', '\xca', '\xcb', '\xcc', '\xcd', '\xce', '\xcf', + '\xd0', '\xd1', '\xd2', '\xd3', '\xd4', '\xd5', '\xd6', '\xd7', '\xd8', '\xd9', '\xda', '\xdb', '\xdc', '\xdd', '\xde', '\xdf', + '\xe0', '\xe1', '\xe2', '\xe3', '\xe4', '\xe5', '\xe6', '\xe7', '\xe8', '\xe9', '\xea', '\xeb', '\xec', '\xed', '\xee', '\xef', + '\xf0', '\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', '\xf8', '\xf9', '\xfa', '\xfb', '\xfc', '\xfd', '\xfe', '\xff', }; - diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 9ec346cebc026cba7d03e1cf39890a6c262d1a9a..962973bb5cd6fa47c0635ac5a4e5d8fa708dbfa5 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -12,132 +12,146 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "tutil.h" +#include "index_fst_counting_writer.h" #include "indexInt.h" #include "index_fst_util.h" -#include "index_fst_counting_writer.h" +#include "tutil.h" -static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { - if (ctx->offset + len > ctx->limit) { - return -1; - } +static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) { + if (ctx->offset + len > ctx->limit) { return -1; } if (ctx->type == TFile) { - assert(len == tfWrite(ctx->fd, buf, len)); + assert(len == tfWrite(ctx->file.fd, buf, len)); } else { - memcpy(ctx->mem + ctx->offset, buf, len); - } + memcpy(ctx->mem.buf + ctx->offset, buf, len); + } ctx->offset += len; return len; } -static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { - int nRead = 0; +static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { + int nRead = 0; if (ctx->type == TFile) { - nRead = tfRead(ctx->fd, buf, len); + nRead = tfRead(ctx->file.fd, buf, len); } else { - memcpy(buf, ctx->mem + ctx->offset, len); + memcpy(buf, ctx->mem.buf + ctx->offset, len); } ctx->offset += nRead; return nRead; -} -static int writeCtxDoFlush(WriterCtx *ctx) { +} +static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) { + int nRead = 0; if (ctx->type == TFile) { - //tfFsync(ctx->fd); - //tfFlush(ctx->fd); + tfLseek(ctx->file.fd, offset, 0); + nRead = tfRead(ctx->file.fd, buf, len); + } else { + // refactor later + assert(0); + } + return nRead; +} +static int writeCtxDoFlush(WriterCtx* ctx) { + if (ctx->type == TFile) { + // tfFsync(ctx->fd); + // tfFlush(ctx->file.fd); } else { // do nothing } return 1; } -WriterCtx* writerCtxCreate(WriterType type, bool readOnly) { - WriterCtx *ctx = calloc(1, sizeof(WriterCtx)); +WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity) { + WriterCtx* ctx = calloc(1, sizeof(WriterCtx)); if (ctx == NULL) { return NULL; } ctx->type = type; if (ctx->type == TFile) { - tfInit(); // ugly code, refactor later + ctx->file.readOnly = readOnly; if (readOnly == false) { - ctx->fd = tfOpenCreateWriteAppend(tmpFile); + ctx->file.fd = tfOpenCreateWriteAppend(tmpFile); } else { - ctx->fd = tfOpenReadWrite(tmpFile); - } - if (ctx->fd < 0) { - indexError("open file error %d", errno); + ctx->file.fd = tfOpenReadWrite(tmpFile); + } + if (ctx->file.fd < 0) { + indexError("open file error %d", errno); + goto END; } } else if (ctx->type == TMemory) { - ctx->mem = calloc(1, DefaultMem * sizeof(uint8_t)); - } + ctx->mem.buf = calloc(1, sizeof(char) * capacity); + ctx->mem.capa = capacity; + } ctx->write = writeCtxDoWrite; - ctx->read = writeCtxDoRead; + ctx->read = writeCtxDoRead; ctx->flush = writeCtxDoFlush; + ctx->readFrom = writeCtxDoReadFrom; ctx->offset = 0; - ctx->limit = DefaultMem; + ctx->limit = capacity; return ctx; +END: + if (ctx->type == TMemory) { free(ctx->mem.buf); } + free(ctx); } -void writerCtxDestroy(WriterCtx *ctx) { +void writerCtxDestroy(WriterCtx* ctx) { if (ctx->type == TMemory) { - free(ctx->mem); + free(ctx->mem.buf); } else { - tfClose(ctx->fd); - tfCleanup(); + tfClose(ctx->file.fd); } free(ctx); } - -FstCountingWriter *fstCountingWriterCreate(void *wrt, bool readOnly) { - FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); +FstCountingWriter* fstCountingWriterCreate(void* wrt) { + FstCountingWriter* cw = calloc(1, sizeof(FstCountingWriter)); if (cw == NULL) { return NULL; } - - cw->wrt = (void *)(writerCtxCreate(TFile, readOnly)); - return cw; + + cw->wrt = wrt; + //(void *)(writerCtxCreate(TFile, readOnly)); + return cw; } -void fstCountingWriterDestroy(FstCountingWriter *cw) { - // free wrt object: close fd or free mem +void fstCountingWriterDestroy(FstCountingWriter* cw) { + // free wrt object: close fd or free mem fstCountingWriterFlush(cw); - writerCtxDestroy((WriterCtx *)(cw->wrt)); + // writerCtxDestroy((WriterCtx *)(cw->wrt)); free(cw); } -int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t len) { - if (write == NULL) { return 0; } - // update checksum +int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len) { + if (write == NULL) { return 0; } + // update checksum // write data to file/socket or mem - WriterCtx *ctx = write->wrt; + WriterCtx* ctx = write->wrt; - int nWrite = ctx->write(ctx, buf, len); + int nWrite = ctx->write(ctx, buf, len); assert(nWrite == len); write->count += len; - return len; -} -int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len) { + return len; +} +int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) { if (write == NULL) { return 0; } - WriterCtx *ctx = write->wrt; - int nRead = ctx->read(ctx, buf, len); - //assert(nRead == len); - return nRead; -} + WriterCtx* ctx = write->wrt; + int nRead = ctx->read(ctx, buf, len); + // assert(nRead == len); + return nRead; +} -uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { +uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) { return 0; } -int fstCountingWriterFlush(FstCountingWriter *write) { - WriterCtx *ctx = write->wrt; +int fstCountingWriterFlush(FstCountingWriter* write) { + WriterCtx* ctx = write->wrt; ctx->flush(ctx); - //write->wtr->flush + // write->wtr->flush return 1; } -void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes) { +void fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes) { assert(1 <= nBytes && nBytes <= 8); - uint8_t *buf = calloc(8, sizeof(uint8_t)); + uint8_t* buf = calloc(8, sizeof(uint8_t)); for (uint8_t i = 0; i < nBytes; i++) { - buf[i] = (uint8_t)n; + buf[i] = (uint8_t)n; n = n >> 8; } fstCountingWriterWrite(writer, buf, nBytes); @@ -145,10 +159,8 @@ void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t return; } -uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n) { +uint8_t fstCountingWriterPackUint(FstCountingWriter* writer, uint64_t n) { uint8_t nBytes = packSize(n); fstCountingWriterPackUintIn(writer, n, nBytes); - return nBytes; -} - - + return nBytes; +} diff --git a/source/libs/index/src/index_fst_node.c b/source/libs/index/src/index_fst_node.c index 5abe8ad5a04af47f68d562705f3c2c569c3f6a0b..5cdf6adb31a421e38fb22d30fe34599f1489924c 100644 --- a/source/libs/index/src/index_fst_node.c +++ b/source/libs/index/src/index_fst_node.c @@ -14,102 +14,91 @@ */ #include "index_fst_node.h" -FstBuilderNode *fstBuilderNodeDefault() { - FstBuilderNode *bn = malloc(sizeof(FstBuilderNode)); - bn->isFinal = false; - bn->finalOutput = 0; - bn->trans = taosArrayInit(16, sizeof(FstTransition)); +FstBuilderNode* fstBuilderNodeDefault() { + FstBuilderNode* bn = malloc(sizeof(FstBuilderNode)); + bn->isFinal = false; + bn->finalOutput = 0; + bn->trans = taosArrayInit(16, sizeof(FstTransition)); return bn; } -void fstBuilderNodeDestroy(FstBuilderNode *node) { +void fstBuilderNodeDestroy(FstBuilderNode* node) { if (node == NULL) { return; } taosArrayDestroy(node->trans); free(node); -} +} -bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) { +bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2) { if (n1 == n2) { return true; } - if (n1 == NULL || n2 == NULL ) { - return false; - } + if (n1 == NULL || n2 == NULL) { return false; } - if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) { - return false; - } - size_t s1 = n1->trans? taosArrayGetSize(n1->trans): 0; - size_t s2 = n2->trans? taosArrayGetSize(n2->trans): 0; - if (s1 != s2) { - return false; - } + if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) { return false; } + size_t s1 = n1->trans ? taosArrayGetSize(n1->trans) : 0; + size_t s2 = n2->trans ? taosArrayGetSize(n2->trans) : 0; + if (s1 != s2) { return false; } for (size_t i = 0; i < s1; i++) { - FstTransition *t1 = taosArrayGet(n1->trans, i); - FstTransition *t2 = taosArrayGet(n2->trans, i); - if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { - return false; - } + FstTransition* t1 = taosArrayGet(n1->trans, i); + FstTransition* t2 = taosArrayGet(n2->trans, i); + if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { return false; } } - + return true; } -FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) { - FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); +FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src) { + FstBuilderNode* node = malloc(sizeof(FstBuilderNode)); if (node == NULL) { return NULL; } - // - size_t sz = taosArrayGetSize(src->trans); - SArray *trans = taosArrayInit(sz, sizeof(FstTransition)); + // + size_t sz = taosArrayGetSize(src->trans); + SArray* trans = taosArrayInit(sz, sizeof(FstTransition)); for (size_t i = 0; i < sz; i++) { - FstTransition *tran = taosArrayGet(src->trans, i); - taosArrayPush(trans, tran); + FstTransition* tran = taosArrayGet(src->trans, i); + taosArrayPush(trans, tran); } - node->trans = trans; + node->trans = trans; node->isFinal = src->isFinal; node->finalOutput = src->finalOutput; return node; - } -// not destroy src, User's bussiness -void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) { - if (dst == NULL || src == NULL) { return; } +// not destroy src, User's bussiness +void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) { + if (dst == NULL || src == NULL) { return; } - dst->isFinal = src->isFinal; + dst->isFinal = src->isFinal; dst->finalOutput = src->finalOutput; - //release free avoid mem leak - taosArrayDestroy(dst->trans); + // release free avoid mem leak + taosArrayDestroy(dst->trans); size_t sz = taosArrayGetSize(src->trans); - dst->trans = taosArrayInit(sz, sizeof(FstTransition)); + dst->trans = taosArrayInit(sz, sizeof(FstTransition)); for (size_t i = 0; i < sz; i++) { - FstTransition *trn = taosArrayGet(src->trans, i); + FstTransition* trn = taosArrayGet(src->trans, i); taosArrayPush(dst->trans, trn); - } + } } +// bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr +// startAddr) { -//bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { - - //size_t sz = taosArrayGetSize(b->trans); - //assert(sz < 256); - //if (FST_BUILDER_NODE_IS_FINAL(b) - // && FST_BUILDER_NODE_TRANS_ISEMPTY(b) - // && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(b)) { - // return true; - //} else if (sz != 1 || b->isFinal) { - // // AnyTrans->Compile(w, addr, node); - //} else { - // FstTransition *tran = taosArrayGet(b->trans, 0); - // if (tran->addr == lastAddr && tran->out == 0) { - // //OneTransNext::compile(w, lastAddr, tran->inp); - // return true; - // } else { - // //OneTrans::Compile(w, lastAddr, *tran); - // return true; - // } - //} - //return true; -//} - - +// size_t sz = taosArrayGetSize(b->trans); +// assert(sz < 256); +// if (FST_BUILDER_NODE_IS_FINAL(b) +// && FST_BUILDER_NODE_TRANS_ISEMPTY(b) +// && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(b)) { +// return true; +//} else if (sz != 1 || b->isFinal) { +// // AnyTrans->Compile(w, addr, node); +//} else { +// FstTransition *tran = taosArrayGet(b->trans, 0); +// if (tran->addr == lastAddr && tran->out == 0) { +// //OneTransNext::compile(w, lastAddr, tran->inp); +// return true; +// } else { +// //OneTrans::Compile(w, lastAddr, *tran); +// return true; +// } +//} +// return true; +//} diff --git a/source/libs/index/src/index_fst_registry.c b/source/libs/index/src/index_fst_registry.c index 8fb0dbfcaa2f86d40b270b197e273802e95ddfe7..07afc52ecf12bf515e32afca7fc50a44e0a422b3 100644 --- a/source/libs/index/src/index_fst_registry.c +++ b/source/libs/index/src/index_fst_registry.c @@ -15,33 +15,31 @@ #include "index_fst_registry.h" - -uint64_t fstRegistryHash(FstRegistry *registry, FstBuilderNode *bNode) { - //TODO(yihaoDeng): refactor later +uint64_t fstRegistryHash(FstRegistry* registry, FstBuilderNode* bNode) { + // TODO(yihaoDeng): refactor later const uint64_t FNV_PRIME = 1099511628211; - uint64_t h = 14695981039346656037u; + uint64_t h = 14695981039346656037u; - h = (h ^ (uint64_t)bNode->isFinal) * FNV_PRIME; + h = (h ^ (uint64_t)bNode->isFinal) * FNV_PRIME; h = (h ^ (bNode)->finalOutput) * FNV_PRIME; - uint32_t sz = (uint32_t)taosArrayGetSize(bNode->trans); + uint32_t sz = (uint32_t)taosArrayGetSize(bNode->trans); for (uint32_t i = 0; i < sz; i++) { - FstTransition *trn = taosArrayGet(bNode->trans, i); - h = (h ^ (uint64_t)(trn->inp)) * FNV_PRIME; - h = (h ^ (uint64_t)(trn->out)) * FNV_PRIME; - h = (h ^ (uint64_t)(trn->addr))* FNV_PRIME; - } - return h %(registry->tableSize); - + FstTransition* trn = taosArrayGet(bNode->trans, i); + h = (h ^ (uint64_t)(trn->inp)) * FNV_PRIME; + h = (h ^ (uint64_t)(trn->out)) * FNV_PRIME; + h = (h ^ (uint64_t)(trn->addr)) * FNV_PRIME; + } + return h % (registry->tableSize); } -static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) { +static void fstRegistryCellSwap(SArray* arr, uint32_t a, uint32_t b) { size_t sz = taosArrayGetSize(arr); if (a >= sz || b >= sz) { return; } - FstRegistryCell *cell1 = (FstRegistryCell *)taosArrayGet(arr, a); - FstRegistryCell *cell2 = (FstRegistryCell *)taosArrayGet(arr, b); + FstRegistryCell* cell1 = (FstRegistryCell*)taosArrayGet(arr, a); + FstRegistryCell* cell2 = (FstRegistryCell*)taosArrayGet(arr, b); - FstRegistryCell t = {.addr = cell1->addr, .node = cell1->node}; + FstRegistryCell t = {.addr = cell1->addr, .node = cell1->node}; cell1->addr = cell2->addr; cell1->node = cell2->node; @@ -51,125 +49,121 @@ static void fstRegistryCellSwap(SArray *arr, uint32_t a, uint32_t b) { return; } -static void fstRegistryCellPromote(SArray *arr, uint32_t start, uint32_t end) { - size_t sz = taosArrayGetSize(arr); - if (start >= sz && end >= sz) {return; } - +static void fstRegistryCellPromote(SArray* arr, uint32_t start, uint32_t end) { + size_t sz = taosArrayGetSize(arr); + if (start >= sz && end >= sz) { return; } + assert(start >= end); int32_t s = (int32_t)start; int32_t e = (int32_t)end; - while(s > e) { + while (s > e) { fstRegistryCellSwap(arr, s - 1, s); s -= 1; } } FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize) { - FstRegistry *registry = malloc(sizeof(FstRegistry)); - if (registry == NULL) { return NULL ;} - - uint64_t nCells = tableSize * mruSize; - SArray* tb = (SArray *)taosArrayInit(nCells, sizeof(FstRegistryCell)); - if (NULL == tb) { - free(registry); - return NULL; + FstRegistry* registry = malloc(sizeof(FstRegistry)); + if (registry == NULL) { return NULL; } + + uint64_t nCells = tableSize * mruSize; + SArray* tb = (SArray*)taosArrayInit(nCells, sizeof(FstRegistryCell)); + if (NULL == tb) { + free(registry); + return NULL; } for (uint64_t i = 0; i < nCells; i++) { - FstRegistryCell cell = {.addr = NONE_ADDRESS, .node = fstBuilderNodeDefault()}; - taosArrayPush(tb, &cell); + FstRegistryCell cell = {.addr = NONE_ADDRESS, .node = fstBuilderNodeDefault()}; + taosArrayPush(tb, &cell); } - - registry->table = tb; - registry->tableSize = tableSize; - registry->mruSize = mruSize; - return registry; + + registry->table = tb; + registry->tableSize = tableSize; + registry->mruSize = mruSize; + return registry; } -void fstRegistryDestroy(FstRegistry *registry) { +void fstRegistryDestroy(FstRegistry* registry) { if (registry == NULL) { return; } - SArray *tb = registry->table; - size_t sz = taosArrayGetSize(tb); + SArray* tb = registry->table; + size_t sz = taosArrayGetSize(tb); for (size_t i = 0; i < sz; i++) { - FstRegistryCell *cell = taosArrayGet(tb, i); - fstBuilderNodeDestroy(cell->node); + FstRegistryCell* cell = taosArrayGet(tb, i); + fstBuilderNodeDestroy(cell->node); } taosArrayDestroy(tb); free(registry); } -FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode) { - if (taosArrayGetSize(registry->table) <= 0) { - return NULL; - } +FstRegistryEntry* fstRegistryGetEntry(FstRegistry* registry, FstBuilderNode* bNode) { + if (taosArrayGetSize(registry->table) <= 0) { return NULL; } uint64_t bucket = fstRegistryHash(registry, bNode); - uint64_t start = registry->mruSize * bucket; - uint64_t end = start + registry->mruSize; - - FstRegistryEntry *entry = malloc(sizeof(FstRegistryEntry)); + uint64_t start = registry->mruSize * bucket; + uint64_t end = start + registry->mruSize; + + FstRegistryEntry* entry = malloc(sizeof(FstRegistryEntry)); if (end - start == 1) { - FstRegistryCell *cell = taosArrayGet(registry->table, start); - //cell->isNode && + FstRegistryCell* cell = taosArrayGet(registry->table, start); + // cell->isNode && if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { - entry->state = FOUND; - entry->addr = cell->addr ; - return entry; + entry->state = FOUND; + entry->addr = cell->addr; + return entry; } else { - fstBuilderNodeCloneFrom(cell->node, bNode); - entry->state = NOTFOUND; - entry->cell = cell; // copy or not + fstBuilderNodeCloneFrom(cell->node, bNode); + entry->state = NOTFOUND; + entry->cell = cell; // copy or not } } else if (end - start == 2) { - FstRegistryCell *cell1 = taosArrayGet(registry->table, start); + FstRegistryCell* cell1 = taosArrayGet(registry->table, start); if (cell1->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell1->node, bNode)) { - entry->state = FOUND; - entry->addr = cell1->addr; + entry->state = FOUND; + entry->addr = cell1->addr; return entry; - } - FstRegistryCell *cell2 = taosArrayGet(registry->table, start + 1); + } + FstRegistryCell* cell2 = taosArrayGet(registry->table, start + 1); if (cell2->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell2->node, bNode)) { - entry->state = FOUND; - entry->addr = cell2->addr; + entry->state = FOUND; + entry->addr = cell2->addr; // must swap here - fstRegistryCellSwap(registry->table, start, start + 1); - return entry; + fstRegistryCellSwap(registry->table, start, start + 1); + return entry; } - //clone from bNode, refactor later + // clone from bNode, refactor later fstBuilderNodeCloneFrom(cell2->node, bNode); fstRegistryCellSwap(registry->table, start, start + 1); - FstRegistryCell *cCell = taosArrayGet(registry->table, start); - entry->state = NOTFOUND; - entry->cell = cCell; + FstRegistryCell* cCell = taosArrayGet(registry->table, start); + entry->state = NOTFOUND; + entry->cell = cCell; } else { - uint32_t i = start; + uint32_t i = start; for (; i < end; i++) { - FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, i); + FstRegistryCell* cell = (FstRegistryCell*)taosArrayGet(registry->table, i); if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { - entry->state = FOUND; - entry->addr = cell->addr; + entry->state = FOUND; + entry->addr = cell->addr; fstRegistryCellPromote(registry->table, i, start); break; } - } + } if (i >= end) { - uint64_t last = end - 1; - FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, last); - //clone from bNode, refactor later - fstBuilderNodeCloneFrom(cell->node, bNode); + uint64_t last = end - 1; + FstRegistryCell* cell = (FstRegistryCell*)taosArrayGet(registry->table, last); + // clone from bNode, refactor later + fstBuilderNodeCloneFrom(cell->node, bNode); fstRegistryCellPromote(registry->table, last, start); - FstRegistryCell *cCell = taosArrayGet(registry->table, start); - entry->state = NOTFOUND; - entry->cell = cCell; + FstRegistryCell* cCell = taosArrayGet(registry->table, start); + entry->state = NOTFOUND; + entry->cell = cCell; } - } + } return entry; } -void fstRegistryEntryDestroy(FstRegistryEntry *entry) { - free(entry); +void fstRegistryEntryDestroy(FstRegistryEntry* entry) { + free(entry); } - - diff --git a/source/libs/index/src/index_fst_util.c b/source/libs/index/src/index_fst_util.c index c933c6d23b2eed040bcf9fd276d783c13b7ad025..da1e177a18b3563dd2388e536cc301bb7b8117d0 100644 --- a/source/libs/index/src/index_fst_util.c +++ b/source/libs/index/src/index_fst_util.c @@ -15,37 +15,32 @@ #include "index_fst_util.h" #include "index_fst_common.h" - - -//A sentinel value used to indicate an empty final state -const CompiledAddr EMPTY_ADDRESS = 0; +// A sentinel value used to indicate an empty final state +const CompiledAddr EMPTY_ADDRESS = 0; /// A sentinel value used to indicate an invalid state. -const CompiledAddr NONE_ADDRESS = 1; +const CompiledAddr NONE_ADDRESS = 1; // This version number is written to every finite state transducer created by // this crate. When a finite state transducer is read, its version number is // checked against this value. -const uint64_t VERSION = 3; -// The threshold (in number of transitions) at which an index is created for -// a node's transitions. This speeds up lookup time at the expense of FST size +const uint64_t VERSION = 3; +// The threshold (in number of transitions) at which an index is created for +// a node's transitions. This speeds up lookup time at the expense of FST size const uint64_t TRANS_INDEX_THRESHOLD = 32; - -//uint8_t commonInput(uint8_t idx) { +// uint8_t commonInput(uint8_t idx) { // if (idx == 0) { return -1; } // else { -// return COMMON_INPUTS_INV[idx - 1]; +// return COMMON_INPUTS_INV[idx - 1]; // } -//} +//} // -//uint8_t commonIdx(uint8_t v, uint8_t max) { +// uint8_t commonIdx(uint8_t v, uint8_t max) { // uint8_t v = ((uint16_t)tCOMMON_INPUTS[v] + 1)%256; // return v > max ? 0: v; //} - - uint8_t packSize(uint64_t n) { if (n < (1u << 8)) { return 1; @@ -66,22 +61,23 @@ uint8_t packSize(uint64_t n) { } } -uint64_t unpackUint64(uint8_t *ch, uint8_t sz) { +uint64_t unpackUint64(uint8_t* ch, uint8_t sz) { uint64_t n = 0; for (uint8_t i = 0; i < sz; i++) { + // n = n | (ch[i] << (8 * i)); } - return n; + return n; } uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr) { if (transAddr == EMPTY_ADDRESS) { - return packSize(EMPTY_ADDRESS); + return packSize(EMPTY_ADDRESS); } else { return packSize(nodeAddr - transAddr); - } -} -CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) { - uint64_t delta = unpackUint64(data, len); + } +} +CompiledAddr unpackDelta(char* data, uint64_t len, uint64_t nodeAddr) { + uint64_t delta = unpackUint64(data, len); // delta_add = u64_to_usize if (delta == EMPTY_ADDRESS) { return EMPTY_ADDRESS; @@ -93,99 +89,101 @@ CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) { // fst slice func // -FstSlice fstSliceCreate(uint8_t *data, uint64_t len) { - FstString *str = (FstString *)malloc(sizeof(FstString)); - str->ref = 1; - str->len = len; +FstSlice fstSliceCreate(uint8_t* data, uint64_t len) { + FstString* str = (FstString*)malloc(sizeof(FstString)); + str->ref = 1; + str->len = len; str->data = malloc(len * sizeof(uint8_t)); memcpy(str->data, data, len); - + FstSlice s = {.str = str, .start = 0, .end = len - 1}; return s; -} +} // just shallow copy -FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) { - FstString *str = s->str; +FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end) { + FstString* str = s->str; str->ref++; - //uint8_t *buf = fstSliceData(s, &alen); - //start = buf + start - (buf - s->start); - //end = buf + end - (buf - s->start); + // uint8_t *buf = fstSliceData(s, &alen); + // start = buf + start - (buf - s->start); + // end = buf + end - (buf - s->start); FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start}; return t; } -FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { - - int32_t tlen = end - start + 1; - int32_t slen; - uint8_t *data = fstSliceData(s, &slen); +FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) { + int32_t tlen = end - start + 1; + int32_t slen; + uint8_t* data = fstSliceData(s, &slen); assert(tlen <= slen); - uint8_t *buf = malloc(sizeof(uint8_t) * tlen); + uint8_t* buf = malloc(sizeof(uint8_t) * tlen); memcpy(buf, data + start, tlen); - - FstString *str = malloc(sizeof(FstString)); + + FstString* str = malloc(sizeof(FstString)); str->data = buf; - str->len = tlen; - str->ref = 1; + str->len = tlen; + str->ref = 1; FstSlice ans; - ans.str = str; - ans.start = 0; - ans.end = tlen - 1; - return ans; + ans.str = str; + ans.start = 0; + ans.end = tlen - 1; + return ans; } -bool fstSliceIsEmpty(FstSlice *s) { +bool fstSliceIsEmpty(FstSlice* s) { return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; } -uint8_t *fstSliceData(FstSlice *s, int32_t *size) { - FstString *str = s->str; - if (size != NULL) { - *size = s->end - s->start + 1; - } - return str->data + s->start; +uint8_t* fstSliceData(FstSlice* s, int32_t* size) { + FstString* str = s->str; + if (size != NULL) { *size = s->end - s->start + 1; } + return str->data + s->start; } -void fstSliceDestroy(FstSlice *s) { - FstString *str = s->str; +void fstSliceDestroy(FstSlice* s) { + FstString* str = s->str; str->ref--; if (str->ref <= 0) { free(str->data); free(str); - s->str = NULL; + s->str = NULL; } } -int fstSliceCompare(FstSlice *a, FstSlice *b) { - int32_t alen, blen; - uint8_t *aBuf = fstSliceData(a, &alen); - uint8_t *bBuf = fstSliceData(b, &blen); +int fstSliceCompare(FstSlice* a, FstSlice* b) { + int32_t alen, blen; + uint8_t* aBuf = fstSliceData(a, &alen); + uint8_t* bBuf = fstSliceData(b, &blen); uint32_t i, j; for (i = 0, j = 0; i < alen && j < blen; i++, j++) { uint8_t x = aBuf[i]; uint8_t y = bBuf[j]; - if (x == y) { continue;} - else if (x < y) { return -1; } - else { return 1; }; + if (x == y) { + continue; + } else if (x < y) { + return -1; + } else { + return 1; + }; + } + if (i < alen) { + return 1; + } else if (j < blen) { + return -1; + } else { + return 0; } - if (i < alen) { return 1; } - else if (j < blen) { return -1; } - else { return 0; } -} +} -//FstStack* fstStackCreate(size_t elemSize, StackFreeElem freeFn) { +// FstStack* fstStackCreate(size_t elemSize, StackFreeElem freeFn) { // FstStack *s = calloc(1, sizeof(FstStack)); // if (s == NULL) { return NULL; } -// s-> -// s->freeFn -// +// s-> +// s->freeFn +// //} -//void *fstStackPush(FstStack *s, void *elem); -//void *fstStackTop(FstStack *s); -//size_t fstStackLen(FstStack *s); -//void *fstStackGetAt(FstStack *s, size_t i); -//void fstStackDestory(FstStack *); - - - +// void *fstStackPush(FstStack *s, void *elem); +// void *fstStackTop(FstStack *s); +// size_t fstStackLen(FstStack *s); +// void *fstStackGetAt(FstStack *s, size_t i); +// void fstStackDestory(FstStack *); diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index a1bba56391ce445899fe1fc23b8e422c8cf3cd8f..ecacdcd13c463ef4ffc689534d13737d69f15e1b 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -13,19 +13,449 @@ * along with this program. If not, see . */ +//#include +//#include #include "index_tfile.h" +#include "index.h" +#include "index_fst.h" +#include "index_fst_counting_writer.h" +#include "index_util.h" +#include "taosdef.h" +#include "tcompare.h" -IndexTFile *indexTFileCreate() { - IndexTFile *tfile = calloc(1, sizeof(IndexTFile)); +#define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t)) + +typedef struct TFileValue { + char* colVal; // null terminated + SArray* tableId; + int32_t offset; +} TFileValue; + +static int tfileValueCompare(const void* a, const void* b, const void* param); +static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds); + +static int tfileWriteHeader(TFileWriter* writer); +static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset); +static int tfileWriteData(TFileWriter* write, TFileValue* tval); + +static int tfileReadLoadHeader(TFileReader* reader); +static int tfileReadLoadFst(TFileReader* reader); +static int tfileReadLoadTableIds(TFileReader* reader, int32_t offset, SArray* result); +static void tfileReadRef(TFileReader* reader); +static void tfileReadUnRef(TFileReader* reader); + +static int tfileGetFileList(const char* path, SArray* result); +static int tfileRmExpireFile(SArray* result); +static void tfileDestroyFileName(void* elem); +static int tfileCompare(const void* a, const void* b); +static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version); +static void tfileSerialCacheKey(TFileCacheKey* key, char* buf); + +TFileCache* tfileCacheCreate(const char* path) { + TFileCache* tcache = calloc(1, sizeof(TFileCache)); + if (tcache == NULL) { return NULL; } + + tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + tcache->capacity = 64; + + SArray* files = taosArrayInit(4, sizeof(void*)); + tfileGetFileList(path, files); + taosArraySort(files, tfileCompare); + tfileRmExpireFile(files); + + uint64_t suid; + int32_t colId, version; + for (size_t i = 0; i < taosArrayGetSize(files); i++) { + char* file = taosArrayGetP(files, i); + if (0 != tfileParseFileName(file, &suid, (int*)&colId, (int*)&version)) { + indexInfo("try parse invalid file: %s, skip it", file); + continue; + } + + WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 64); + if (wc == NULL) { + indexError("failed to open index: %s", file); + goto End; + } + + TFileReader* reader = tfileReaderCreate(wc); + if (0 != tfileReadLoadHeader(reader)) { + tfileReaderDestroy(reader); + indexError("failed to load index header, index file: %s", file); + goto End; + } + + if (0 != tfileReadLoadFst(reader)) { + tfileReaderDestroy(reader); + indexError("failed to load index fst, index file: %s", file); + goto End; + } + tfileReadRef(reader); + // loader fst and validate it + TFileHeader* header = &reader->header; + TFileCacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType}; + + char buf[128] = {0}; + tfileSerialCacheKey(&key, buf); + taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); + } + taosArrayDestroyEx(files, tfileDestroyFileName); + return tcache; +End: + tfileCacheDestroy(tcache); + taosArrayDestroyEx(files, tfileDestroyFileName); + return NULL; +} +void tfileCacheDestroy(TFileCache* tcache) { + if (tcache == NULL) { return; } + + // free table cache + TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); + while (reader) { + TFileReader* p = *reader; + indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, p->header.colType); + + tfileReadUnRef(p); + reader = taosHashIterate(tcache->tableCache, reader); + } + taosHashCleanup(tcache->tableCache); + free(tcache); +} + +TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) { + char buf[128] = {0}; + tfileSerialCacheKey(key, buf); + + TFileReader* reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); + tfileReadRef(reader); + + return reader; +} +void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) { + char buf[128] = {0}; + tfileSerialCacheKey(key, buf); + // remove last version index reader + TFileReader** p = taosHashGet(tcache->tableCache, buf, strlen(buf)); + if (*p != NULL) { + TFileReader* oldReader = *p; + taosHashRemove(tcache->tableCache, buf, strlen(buf)); + tfileReadUnRef(oldReader); + } + + tfileReadRef(reader); + taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); + return; +} + +TFileReader* tfileReaderCreate(WriterCtx* ctx) { + TFileReader* reader = calloc(1, sizeof(TFileReader)); + if (reader == NULL) { return NULL; } + + // T_REF_INC(reader); + reader->ctx = ctx; + return reader; +} +void tfileReaderDestroy(TFileReader* reader) { + if (reader == NULL) { return; } + // T_REF_INC(reader); + fstDestroy(reader->fst); + writerCtxDestroy(reader->ctx); + free(reader); +} + +int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result) { + SIndexTerm* term = query->term; + EIndexQueryType qtype = query->qType; + + int ret = -1; + // refactor to callback later + if (qtype == QUERY_TERM) { + uint64_t offset; + FstSlice key = fstSliceCreate(term->colVal, term->nColVal); + if (fstGet(reader->fst, &key, &offset)) { + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex", term->suid, term->colName, term->colVal); + ret = tfileReadLoadTableIds(reader, offset, result); + } else { + indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName, term->colVal); + } + fstSliceDestroy(&key); + } else if (qtype == QUERY_PREFIX) { + // handle later + // + } else { + // handle later + } + tfileReadUnRef(reader); + return ret; +} + +TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { + // char pathBuf[128] = {0}; + // sprintf(pathBuf, "%s/% " PRIu64 "-%d-%d.tindex", path, suid, colId, version); + // TFileHeader header = {.suid = suid, .version = version, .colName = {0}, colType = colType}; + // memcpy(header.colName, ); + + // char buf[TFILE_HADER_PRE_SIZE]; + // int len = TFILE_HADER_PRE_SIZE; + // if (len != ctx->write(ctx, buf, len)) { + // indexError("index: %" PRIu64 " failed to write header info", header->suid); + // return NULL; + //} + TFileWriter* tw = calloc(1, sizeof(TFileWriter)); + if (tw == NULL) { + indexError("index: %" PRIu64 " failed to alloc TFilerWriter", header->suid); + return NULL; + } + tw->ctx = ctx; + tw->header = *header; + tfileWriteHeader(tw); + return tw; +} + +int tfileWriterPut(TFileWriter* tw, void* data) { + // sort by coltype and write to tindex + __compar_fn_t fn = getComparFunc(tw->header.colType, 0); + taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn); + + int32_t bufLimit = 4096, offset = 0; + char* buf = calloc(1, sizeof(bufLimit)); + char* p = buf; + int32_t sz = taosArrayGetSize((SArray*)data); + int32_t fstOffset = tw->offset; + + // ugly code, refactor later + for (size_t i = 0; i < sz; i++) { + TFileValue* v = taosArrayGetP((SArray*)data, i); + + int32_t tbsz = taosArrayGetSize(v->tableId); + fstOffset += TF_TABLE_TATOAL_SIZE(tbsz); + } + tfileWriteFstOffset(tw, fstOffset); + + for (size_t i = 0; i < sz; i++) { + TFileValue* v = taosArrayGetP((SArray*)data, i); + + int32_t tbsz = taosArrayGetSize(v->tableId); + // check buf has enough space or not + int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz); + if (offset + ttsz > bufLimit) { + // batch write + tw->ctx->write(tw->ctx, buf, offset); + offset = 0; + memset(buf, 0, bufLimit); + p = buf; + } + + tfileSerialTableIdsToBuf(p, v->tableId); + offset += ttsz; + p = buf + offset; + // set up value offset + v->offset = tw->offset; + tw->offset += ttsz; + } + if (offset != 0) { + // write reversed data in buf to tindex + tw->ctx->write(tw->ctx, buf, offset); + } + tfree(buf); + + // write fst + for (size_t i = 0; i < sz; i++) { + // TODO, fst batch write later + TFileValue* v = taosArrayGetP((SArray*)data, i); + if (tfileWriteData(tw, v) == 0) { + // + } + } + return 0; +} +void tfileWriterDestroy(TFileWriter* tw) { + if (tw == NULL) { return; } + + writerCtxDestroy(tw->ctx); + free(tw); +} + +IndexTFile* indexTFileCreate(const char* path) { + IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); + if (tfile == NULL) { return NULL; } + + tfile->cache = tfileCacheCreate(path); return tfile; } -void IndexTFileDestroy(IndexTFile *tfile) { - free(tfile); +void IndexTFileDestroy(IndexTFile* tfile) { + free(tfile); +} + +int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { + int ret = -1; + if (tfile == NULL) { return ret; } + IndexTFile* pTfile = (IndexTFile*)tfile; + + SIndexTerm* term = query->term; + TFileCacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; + TFileReader* reader = tfileCacheGet(pTfile->cache, &key); + + return tfileReaderSearch(reader, query, result); +} +int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) { + // TFileWriterOpt wOpt = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName, .version = + // 1}; + + return 0; +} + +static int tfileValueCompare(const void* a, const void* b, const void* param) { + __compar_fn_t fn = *(__compar_fn_t*)param; + + TFileValue* av = (TFileValue*)a; + TFileValue* bv = (TFileValue*)b; + + return fn(av->colVal, bv->colVal); +} +static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { + int sz = taosArrayGetSize(ids); + SERIALIZE_VAR_TO_BUF(buf, sz, int32_t); + for (size_t i = 0; i < sz; i++) { + uint64_t* v = taosArrayGet(ids, i); + SERIALIZE_VAR_TO_BUF(buf, *v, uint64_t); + } +} + +static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) { + int32_t fstOffset = offset + sizeof(tw->header.fstOffset); + tw->header.fstOffset = fstOffset; + if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; } + return 0; +} +static int tfileWriteHeader(TFileWriter* writer) { + char buf[TFILE_HEADER_NO_FST] = {0}; + + TFileHeader* header = &writer->header; + memcpy(buf, (char*)header, sizeof(buf)); + + int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); + if (sizeof(buf) != nwrite) { return -1; } + writer->offset = nwrite; + return 0; +} +static int tfileWriteData(TFileWriter* write, TFileValue* tval) { + TFileHeader* header = &write->header; + uint8_t colType = header->colType; + if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { + FstSlice key = fstSliceCreate((uint8_t*)(tval->colVal), (size_t)strlen(tval->colVal)); + if (fstBuilderInsert(write->fb, key, tval->offset)) { + fstSliceDestroy(&key); + return 0; + } + fstSliceDestroy(&key); + return -1; + } else { + // handle other type later + } + return 0; +} +static int tfileReadLoadHeader(TFileReader* reader) { + // TODO simple tfile header later + char buf[TFILE_HEADER_SIZE] = {0}; + + int64_t nread = reader->ctx->read(reader->ctx, buf, sizeof(buf)); + assert(nread == sizeof(buf)); + memcpy(&reader->header, buf, sizeof(buf)); + return 0; +} +static int tfileReadLoadFst(TFileReader* reader) { + // current load fst into memory, refactor it later + static int FST_MAX_SIZE = 16 * 1024; + + char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); + if (buf == NULL) { return -1; } + + WriterCtx* ctx = reader->ctx; + int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); + // we assuse fst size less than FST_MAX_SIZE + assert(nread > 0 && nread < FST_MAX_SIZE); + + FstSlice st = fstSliceCreate((uint8_t*)buf, nread); + reader->fst = fstCreate(&st); + free(buf); + fstSliceDestroy(&st); + + return reader->fst == NULL ? 0 : -1; } -int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) { - IndexTFile *ptfile = (IndexTFile *)tfile; +static int tfileReadLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) { + int32_t nid; + WriterCtx* ctx = reader->ctx; + + int32_t nread = ctx->readFrom(ctx, (char*)&nid, sizeof(nid), offset); + assert(sizeof(nid) == nread); + + int32_t total = sizeof(uint64_t) * nid; + char* buf = calloc(1, total); + if (buf == NULL) { return -1; } + + nread = ctx->read(ctx, buf, total); + assert(total == nread); + + for (int32_t i = 0; i < nid; i++) { + taosArrayPush(result, (uint64_t*)buf + i); + } + free(buf); return 0; } +static void tfileReadRef(TFileReader* reader) { + int ref = T_REF_INC(reader); + UNUSED(ref); +} +static void tfileReadUnRef(TFileReader* reader) { + int ref = T_REF_DEC(reader); + if (ref == 0) { tfileReaderDestroy(reader); } +} +static int tfileGetFileList(const char* path, SArray* result) { + DIR* dir = opendir(path); + if (NULL == dir) { return -1; } + struct dirent* entry; + while ((entry = readdir(dir)) != NULL) { + size_t len = strlen(entry->d_name); + char* buf = calloc(1, len + 1); + memcpy(buf, entry->d_name, len); + taosArrayPush(result, &buf); + } + closedir(dir); + return 0; +} +static int tfileRmExpireFile(SArray* result) { + // TODO(yihao): remove expire tindex after restart + return 0; +} +static void tfileDestroyFileName(void* elem) { + char* p = *(char**)elem; + free(p); +} +static int tfileCompare(const void* a, const void* b) { + const char* aName = *(char**)a; + const char* bName = *(char**)b; + + size_t aLen = strlen(aName); + size_t bLen = strlen(bName); + + return strncmp(aName, bName, aLen > bLen ? aLen : bLen); +} +// tfile name suid-colId-version.tindex +static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version) { + if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) { + // read suid & colid & version success + return 0; + } + return -1; +} +static void tfileSerialCacheKey(TFileCacheKey* key, char* buf) { + SERIALIZE_MEM_TO_BUF(buf, key, suid); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_MEM_TO_BUF(buf, key, colType); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); +} diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 9dff2e9ea0ac901390cde63feee01f50c3179fe6..85d76bb716a887bf7e7b07d54afde9ae67f431de 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -26,6 +26,7 @@ class FstWriter { public: FstWriter() { + _wc = writerCtxCreate(TFile, "/tmp/tindex", false, 0); _b = fstBuilderCreate(NULL, 0); } bool Put(const std::string &key, uint64_t val) { @@ -37,15 +38,19 @@ class FstWriter { ~FstWriter() { fstBuilderFinish(_b); fstBuilderDestroy(_b); + + writerCtxDestroy(_wc); } private: FstBuilder *_b; + WriterCtx *_wc; }; class FstReadMemory { public: FstReadMemory(size_t size) { - _w = fstCountingWriterCreate(NULL, true); + _wc = writerCtxCreate(TFile, "/tmp/tindex", true, 0); + _w = fstCountingWriterCreate(_wc); _size = size; memset((void *)&_s, 0, sizeof(_s)); } @@ -94,12 +99,14 @@ class FstReadMemory { fstCountingWriterDestroy(_w); fstDestroy(_fst); fstSliceDestroy(&_s); + writerCtxDestroy(_wc); } private: FstCountingWriter *_w; Fst *_fst; FstSlice _s; + WriterCtx *_wc; size_t _size; }; diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index de7cdd58b85227c3a4dbb1742a0789fe01413802..400b533bf794bdbcfe66438de9fc8538e87d5acf 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -4,8 +4,11 @@ #include "parserInt.h" #include "taosmsg.h" -SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen); +SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); +SCreateAcctMsg* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); +SDropUserMsg* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen); SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen); +SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); #endif // TDENGINE_ASTTOMSG_H diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/dataBlockMgt.h index 7d2e6e3aec336c2e1158501c7197ebdf790fa79b..69dad4d9f4b792ed573628f05ddb95f301f8cc08 100644 --- a/source/libs/parser/inc/dataBlockMgt.h +++ b/source/libs/parser/inc/dataBlockMgt.h @@ -78,7 +78,7 @@ typedef struct { typedef struct STableDataBlocks { int8_t tsSource; // where does the UNIX timestamp come from, server or client bool ordered; // if current rows are ordered or not - int64_t vgId; // virtual group id + int32_t vgId; // virtual group id int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending int32_t numOfTables; // number of tables in current submit block int32_t rowSize; // row size for current table @@ -170,10 +170,11 @@ int32_t schemaIdxCompar(const void *lhs, const void *rhs); int32_t boundIdxCompar(const void *lhs, const void *rhs); void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols); void destroyBoundColumnInfo(SParsedDataColInfo* pColList); +void destroyBlockArrayList(SArray* pDataBlockList); int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen); int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList); -int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap); +int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, SArray** pVgDataBlocks); #endif // TDENGINE_DATABLOCKMGT_H diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index 4f506e631f5099f0b31a589c07f46aeff0cf3683..93df023fbaedb0de568249557dd887e3250f9b4a 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -38,6 +38,14 @@ typedef struct SMsgBuf { char *buf; } SMsgBuf; +// create table operation type +enum TSQL_CREATE_TABLE_TYPE { + TSQL_CREATE_TABLE = 0x1, + TSQL_CREATE_STABLE = 0x2, + TSQL_CREATE_CTABLE = 0x3, + TSQL_CREATE_STREAM = 0x4, +}; + void clearTableMetaInfo(STableMetaInfo* pTableMetaInfo); void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t id); @@ -60,7 +68,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQ * @param type * @return */ -int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen); +int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen); /** * Evaluate the numeric and timestamp arithmetic expression in the WHERE clause. diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index c588a34a40771391fd39fe9688277aa4640cb0fc..3d8729e72d57eb9dd8f8b0bbcd46a128ed865766 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -67,6 +67,8 @@ int32_t getExprFunctionId(SExprInfo *pExprInfo); STableMeta* tableMetaDup(const STableMeta* pTableMeta); bool isDclSqlStatement(SSqlInfo* pSqlInfo); +bool isDdlSqlStatement(SSqlInfo* pSqlInfo); +bool isDqlSqlStatement(SSqlInfo* pSqlInfo); #ifdef __cplusplus } diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index dda30c56fc6c9d647659f1fddcf2bcb69ce25c20..b6a6b73ccc258a5a8d5c12a1961d8f22fabba411 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -13,9 +13,10 @@ * along with this program. If not, see . */ -#include "taos.h" -#include "os.h" #include "astGenerator.h" +#include +#include "os.h" +#include "taos.h" #include "tmsgtype.h" SArray *tListItemAppend(SArray *pList, SVariant *pVar, uint8_t sortOrder) { diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 1b46faececee5dabb589d8fdae2bccc0ddf0aca1..a9272f39df9cbbfece0ea7d1fe4d97107a272bea 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -1,7 +1,8 @@ +#include #include "parserInt.h" #include "parserUtil.h" -SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen) { +SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen) { SCreateUserMsg* pMsg = (SCreateUserMsg*)calloc(1, sizeof(SCreateUserMsg)); if (pMsg == NULL) { // tscError("0x%" PRIx64 " failed to malloc for query msg", id); @@ -20,6 +21,68 @@ SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgB strncpy(pMsg->pass, pUser->passwd.z, pUser->passwd.n); } + *outputLen = sizeof(SUserInfo); + return pMsg; +} + +SCreateAcctMsg* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen) { + SCreateAcctMsg* pMsg = (SCreateAcctMsg*)calloc(1, sizeof(SCreateAcctMsg)); + if (pMsg == NULL) { + // tscError("0x%" PRIx64 " failed to malloc for query msg", id); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + SCreateAcctMsg *pCreateMsg = (SCreateAcctMsg *) calloc(1, sizeof(SCreateAcctMsg)); + + SToken *pName = &pInfo->pMiscInfo->user.user; + SToken *pPwd = &pInfo->pMiscInfo->user.passwd; + + strncpy(pCreateMsg->user, pName->z, pName->n); + strncpy(pCreateMsg->pass, pPwd->z, pPwd->n); + + SCreateAcctInfo *pAcctOpt = &pInfo->pMiscInfo->acctOpt; + + pCreateMsg->maxUsers = htonl(pAcctOpt->maxUsers); + pCreateMsg->maxDbs = htonl(pAcctOpt->maxDbs); + pCreateMsg->maxTimeSeries = htonl(pAcctOpt->maxTimeSeries); + pCreateMsg->maxStreams = htonl(pAcctOpt->maxStreams); +// pCreateMsg->maxPointsPerSecond = htonl(pAcctOpt->maxPointsPerSecond); + pCreateMsg->maxStorage = htobe64(pAcctOpt->maxStorage); +// pCreateMsg->maxQueryTime = htobe64(pAcctOpt->maxQueryTime); +// pCreateMsg->maxConnections = htonl(pAcctOpt->maxConnections); + + if (pAcctOpt->stat.n == 0) { + pCreateMsg->accessState = -1; + } else { + if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { + pCreateMsg->accessState = TSDB_VN_READ_ACCCESS; + } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { + pCreateMsg->accessState = TSDB_VN_WRITE_ACCCESS; + } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { + pCreateMsg->accessState = TSDB_VN_ALL_ACCCESS; + } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { + pCreateMsg->accessState = 0; + } + } + + *outputLen = sizeof(SCreateAcctMsg); + return pMsg; +} +SDropUserMsg* buildDropUserMsg(SSqlInfo* pInfo, int32_t *msgLen, int64_t id, char* msgBuf, int32_t msgBufLen) { + SToken* pName = taosArrayGet(pInfo->pMiscInfo->a, 0); + if (pName->n >= TSDB_USER_LEN) { + return NULL; + } + + + SDropUserMsg* pMsg = calloc(1, sizeof(SDropUserMsg)); + if (pMsg == NULL) { + return NULL; + } + + strncpy(pMsg->user, pName->z, pName->n); + *msgLen = sizeof(SDropUserMsg); return pMsg; } @@ -89,7 +152,7 @@ static int32_t setTimePrecision(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreate pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default - SToken* pToken = &pCreateDbInfo->precision; + SToken* pToken = (SToken*) &pCreateDbInfo->precision; if (pToken->n > 0) { pToken->n = strdequote(pToken->z); @@ -116,8 +179,8 @@ static void doSetDbOptions(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb) { pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); pMsg->commitTime = htonl((int32_t)pCreateDb->commitTime); - pMsg->minRowsPerFileBlock = htonl(pCreateDb->minRowsPerBlock); - pMsg->maxRowsPerFileBlock = htonl(pCreateDb->maxRowsPerBlock); + pMsg->minRows = htonl(pCreateDb->minRowsPerBlock); + pMsg->maxRows = htonl(pCreateDb->maxRowsPerBlock); pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod); pMsg->compression = pCreateDb->compressionLevel; pMsg->walLevel = (char)pCreateDb->walLevel; @@ -141,7 +204,6 @@ int32_t setDbOptions(SCreateDbMsg* pCreateDbMsg, const SCreateDbInfo* pCreateDbS // todo configurable pCreateDbMsg->numOfVgroups = htonl(2); - return TSDB_CODE_SUCCESS; } @@ -158,3 +220,126 @@ SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32 return pCreateMsg; } + +int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { + const char* msg1 = "name too long"; + const char* msg2 = "acctId too long"; + + int32_t code = TSDB_CODE_SUCCESS; + char* p = strnchr(pTableName->z, TS_PATH_DELIMITER[0], pTableName->n, false); + + if (p != NULL) { // db has been specified in sql string so we ignore current db path + code = tNameSetAcctId(pName, pParseCtx->acctId); + if (code != 0) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, pTableName->n); + + code = tNameFromString(pName, name, T_NAME_DB|T_NAME_TABLE); + if (code != 0) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + } else { // get current DB name first, and then set it into path + if (pTableName->n >= TSDB_TABLE_NAME_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + tNameSetDbName(pName, pParseCtx->acctId, pParseCtx->db, strlen(pParseCtx->db)); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, pTableName->n); + + code = tNameFromString(pName, name, T_NAME_TABLE); + if (code != 0) { + code = buildInvalidOperationMsg(pMsgBuf, msg1); + } + } + + return code; +} + +SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { + SSchema* pSchema; + + int32_t numOfCols = (int32_t) taosArrayGetSize(pCreateTableSql->colInfo.pColumns); + int32_t numOfTags = (int32_t) taosArrayGetSize(pCreateTableSql->colInfo.pTagColumns); + + SCreateStbMsg* pCreateTableMsg = (SCreateStbMsg*)calloc(1, sizeof(SCreateStbMsg) + (numOfCols + numOfTags) * sizeof(SSchema)); + + char* pMsg = NULL; + int8_t type = pCreateTableSql->type; + if (type == TSQL_CREATE_TABLE) { // create by using super table, tags value +#if 0 + SArray* list = pInfo->pCreateTableInfo->childTableInfo; + + int32_t numOfTables = (int32_t)taosArrayGetSize(list); + pCreateTableMsg->numOfTables = htonl(numOfTables); + + pMsg = (char*)pCreateMsg; + for (int32_t i = 0; i < numOfTables; ++i) { + SCreateTableMsg* pCreate = (SCreateTableMsg*)pMsg; + + pCreate->numOfColumns = htons(pCmd->numOfCols); + pCreate->numOfTags = htons(pCmd->count); + pMsg += sizeof(SCreateTableMsg); + + SCreatedTableInfo* p = taosArrayGet(list, i); + strcpy(pCreate->tableName, p->fullname); + pCreate->igExists = (p->igExist) ? 1 : 0; + + // use dbinfo from table id without modifying current db info + pMsg = serializeTagData(&p->tagdata, pMsg); + + int32_t len = (int32_t)(pMsg - (char*)pCreate); + pCreate->len = htonl(len); + } +#endif + } else { // create (super) table + SName n = {0}; + int32_t code = createSName(&n, &pCreateTableSql->name, pParseCtx, pMsgBuf); + if (code != 0) { + return NULL; + } + + code = tNameExtractFullName(&n, pCreateTableMsg->name); + if (code != 0) { + buildInvalidOperationMsg(pMsgBuf, "invalid table name or database not specified"); + return NULL; + } + + pCreateTableMsg->igExists = pCreateTableSql->existCheck ? 1 : 0; + + pCreateTableMsg->numOfColumns = htonl(numOfCols); + pCreateTableMsg->numOfTags = htonl(numOfTags); + + pSchema = (SSchema*) pCreateTableMsg->pSchema; + for (int i = 0; i < numOfCols; ++i) { + TAOS_FIELD* pField = taosArrayGet(pCreateTableSql->colInfo.pColumns, i); + + pSchema->type = pField->type; + pSchema->bytes = htonl(pField->bytes); + strcpy(pSchema->name, pField->name); + + pSchema++; + } + + for(int32_t i = 0; i < numOfTags; ++i) { + TAOS_FIELD* pField = taosArrayGet(pCreateTableSql->colInfo.pTagColumns, i); + + pSchema->type = pField->type; + pSchema->bytes = htonl(pField->bytes); + strcpy(pSchema->name, pField->name); + + pSchema++; + } + + pMsg = (char*)pSchema; + } + + int32_t msgLen = (int32_t)(pMsg - (char*)pCreateTableMsg); + *len = msgLen; + + return pCreateTableMsg; +} diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 135774cd3b6921130c183efabf1dcacc12c51120..33efeb469d7ad50fd51eebeacbe62a3e1689d58c 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4171,7 +4171,144 @@ static int32_t doCheckDbOptions(SCreateDbMsg* pCreate, SMsgBuf* pMsgBuf) { return TSDB_CODE_SUCCESS; } -int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen) { +/* is contained in pFieldList or not */ +static bool has(SArray* pFieldList, int32_t startIndex, const char* name) { + size_t numOfCols = taosArrayGetSize(pFieldList); + for (int32_t j = startIndex; j < numOfCols; ++j) { + TAOS_FIELD* field = taosArrayGet(pFieldList, j); + if (strncasecmp(name, field->name, sizeof(field->name) - 1) == 0) return true; + } + + return false; +} + +static int32_t validateTableColumns(SArray* pFieldList, int32_t maxRowLength, int32_t maxColumns, SMsgBuf* pMsgBuf) { + const char* msg2 = "row length exceeds max length"; + const char* msg3 = "duplicated column names"; + const char* msg4 = "invalid data type"; + const char* msg5 = "invalid binary/nchar column length"; + const char* msg6 = "invalid column name"; + const char* msg7 = "too many columns"; + const char* msg8 = "illegal number of columns"; + + size_t numOfCols = taosArrayGetSize(pFieldList); + if (numOfCols > maxColumns) { + return buildInvalidOperationMsg(pMsgBuf, msg7); + } + + int32_t rowLen = 0; + for (int32_t i = 0; i < numOfCols; ++i) { + TAOS_FIELD* pField = taosArrayGet(pFieldList, i); + if (!isValidDataType(pField->type)) { + return buildInvalidOperationMsg(pMsgBuf, msg4); + } + + if (pField->bytes == 0) { + return buildInvalidOperationMsg(pMsgBuf, msg5); + } + + if ((pField->type == TSDB_DATA_TYPE_BINARY && (pField->bytes <= 0 || pField->bytes > TSDB_MAX_BINARY_LEN)) || + (pField->type == TSDB_DATA_TYPE_NCHAR && (pField->bytes <= 0 || pField->bytes > TSDB_MAX_NCHAR_LEN))) { + return buildInvalidOperationMsg(pMsgBuf, msg5); + } + + SToken nameToken = {.z = pField->name, .n = strlen(pField->name), .type = TK_ID}; + if (parserValidateNameToken(&nameToken) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg6); + } + + // field name must be unique + if (has(pFieldList, i + 1, pField->name) == true) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + rowLen += pField->bytes; + } + + // max row length must be less than TSDB_MAX_BYTES_PER_ROW + if (rowLen > maxRowLength) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t validateTableColumnInfo(SArray* pFieldList, SMsgBuf* pMsgBuf) { + assert(pFieldList != NULL); + + const char* msg1 = "first column must be timestamp"; + const char* msg2 = "row length exceeds max length"; + const char* msg3 = "duplicated column names"; + const char* msg4 = "invalid data type"; + const char* msg5 = "invalid binary/nchar column length"; + const char* msg6 = "invalid column name"; + const char* msg7 = "too many columns"; + const char* msg8 = "illegal number of columns"; + + // first column must be timestamp + TAOS_FIELD* pField = taosArrayGet(pFieldList, 0); + if (pField->type != TSDB_DATA_TYPE_TIMESTAMP) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + // number of fields no less than 2 + size_t numOfCols = taosArrayGetSize(pFieldList); + if (numOfCols <= 1) { + return buildInvalidOperationMsg(pMsgBuf, msg8); + } + + return validateTableColumns(pFieldList, TSDB_MAX_BYTES_PER_ROW, TSDB_MAX_COLUMNS, pMsgBuf); +} + +static int32_t validateTagParams(SArray* pTagsList, SArray* pFieldList, SMsgBuf* pMsgBuf) { + assert(pTagsList != NULL); + + const char* msg1 = "invalid number of tag columns"; + const char* msg3 = "duplicated column names"; + + // number of fields at least 1 + size_t numOfTags = taosArrayGetSize(pTagsList); + if (numOfTags < 1) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + // field name must be unique + for (int32_t i = 0; i < numOfTags; ++i) { + TAOS_FIELD* p = taosArrayGet(pTagsList, i); + if (has(pFieldList, 0, p->name) == true) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + } + + return validateTableColumns(pFieldList, TSDB_MAX_TAGS_LEN, TSDB_MAX_TAGS, pMsgBuf); +} + +int32_t doCheckForCreateTable(SSqlInfo* pInfo, SMsgBuf* pMsgBuf) { + const char* msg1 = "invalid table name"; + + SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; + + SArray* pFieldList = pCreateTable->colInfo.pColumns; + SArray* pTagList = pCreateTable->colInfo.pTagColumns; + assert(pFieldList != NULL); + + // if sql specifies db, use it, otherwise use default db + SToken* pzTableName = &(pCreateTable->name); + + bool dbIncluded = false; + if (parserValidateNameToken(pzTableName) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + if (validateTableColumnInfo(pFieldList, pMsgBuf) != TSDB_CODE_SUCCESS || + (pTagList != NULL && validateTagParams(pTagList, pFieldList, pMsgBuf) != TSDB_CODE_SUCCESS)) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen) { int32_t code = 0; SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; @@ -4224,7 +4361,49 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, in } } - *output = buildUserManipulationMsg(pInfo, id, msgBuf, msgBufLen); + *output = buildUserManipulationMsg(pInfo, outputLen, pCtx->requestId, msgBuf, msgBufLen); + break; + } + + case TSDB_SQL_CREATE_ACCT: + case TSDB_SQL_ALTER_ACCT: { + const char* msg1 = "invalid state option, available options[no, r, w, all]"; + const char* msg2 = "invalid user/account name"; + const char* msg3 = "name too long"; + + SToken* pName = &pInfo->pMiscInfo->user.user; + SToken* pPwd = &pInfo->pMiscInfo->user.passwd; + + if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + if (pName->n >= TSDB_USER_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + if (parserValidateNameToken(pName) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; + if (pAcctOpt->stat.n > 0) { + if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { + } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { + } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { + } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { + } else { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + } + + *output = buildAcctManipulationMsg(pInfo, outputLen, pCtx->requestId, msgBuf, msgBufLen); + break; + } + + case TSDB_SQL_DROP_ACCT: + case TSDB_SQL_DROP_USER: { + *output = buildDropUserMsg(pInfo, outputLen, pCtx->requestId, msgBuf, msgBufLen); break; } @@ -4233,6 +4412,28 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, in break; } + case TSDB_SQL_USE_DB: { + const char* msg = "invalid db name"; + + SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); + if (parserValidateNameToken(pToken) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + SName n = {0}; + int32_t ret = tNameSetDbName(&n, pCtx->acctId, pToken->z, pToken->n); + if (ret != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + SUseDbMsg *pUseDbMsg = (SUseDbMsg *) calloc(1, sizeof(SUseDbMsg)); + tNameExtractFullName(&n, pUseDbMsg->db); + + *output = pUseDbMsg; + *outputLen = sizeof(SUseDbMsg); + break; + } + case TSDB_SQL_ALTER_DB: case TSDB_SQL_CREATE_DB: { const char* msg1 = "invalid db name"; @@ -4262,6 +4463,26 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, in break; } + case TSDB_SQL_CREATE_TABLE: { + SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; + + if (pCreateTable->type == TSQL_CREATE_TABLE || pCreateTable->type == TSQL_CREATE_STABLE) { + if ((code = doCheckForCreateTable(pInfo, pMsgBuf)) != TSDB_CODE_SUCCESS) { + return code; + } + *output = buildCreateTableMsg(pCreateTable, outputLen, pCtx, pMsgBuf); + } else if (pCreateTable->type == TSQL_CREATE_CTABLE) { + // if ((code = doCheckForCreateFromStable(pSql, pInfo)) != TSDB_CODE_SUCCESS) { + // return code; + // } + + } else if (pCreateTable->type == TSQL_CREATE_STREAM) { + // if ((code = doCheckForStream(pSql, pInfo)) != TSDB_CODE_SUCCESS) { + // return code; + } + + break; + } default: break; } diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c index 3138f0c1d8c8fe0bdca6b835e1c0afe9bb0fe407..2f3d5aab77ecaea9bf30ab2113f6636bcee7e231 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/dataBlockMgt.c @@ -123,7 +123,6 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->nAllocSize = dataBuf->headerSize * 2; } - //dataBuf->pData = calloc(1, dataBuf->nAllocSize); dataBuf->pData = malloc(dataBuf->nAllocSize); if (dataBuf->pData == NULL) { tfree(dataBuf); @@ -145,8 +144,6 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->tsSource = -1; dataBuf->vgId = dataBuf->pTableMeta->vgId; - // tNameAssign(&dataBuf->tableName, name); - assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); *dataBlocks = dataBuf; @@ -269,19 +266,17 @@ void destroyDataBlock(STableDataBlocks* pDataBlock) { tfree(pDataBlock); } -void* destroyBlockArrayList(SArray* pDataBlockList) { +void destroyBlockArrayList(SArray* pDataBlockList) { if (pDataBlockList == NULL) { - return NULL; + return; } size_t size = taosArrayGetSize(pDataBlockList); for (int32_t i = 0; i < size; i++) { - void* d = taosArrayGetP(pDataBlockList, i); - destroyDataBlock(d); + destroyDataBlock(taosArrayGetP(pDataBlockList, i)); } taosArrayDestroy(pDataBlockList); - return NULL; } // data block is disordered, sort it in ascending order @@ -298,6 +293,7 @@ void sortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) { int32_t i = 0; int32_t j = 1; + // delete rows with timestamp conflicts while (j < pBlocks->numOfRows) { TSKEY ti = *(TSKEY *)(pBlockData + dataBuf->rowSize * i); TSKEY tj = *(TSKEY *)(pBlockData + dataBuf->rowSize * j); @@ -430,7 +426,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB char* p = pTableDataBlock->pData + sizeof(SSubmitBlk); pBlock->dataLen = 0; - int32_t numOfRows = htons(pBlock->numOfRows); + int32_t numOfRows = pBlock->numOfRows; if (isRawPayload) { for (int32_t i = 0; i < numOfRows; ++i) { @@ -467,18 +463,10 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB } } - int32_t len = pBlock->dataLen + pBlock->schemaLen; - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->schemaLen = htonl(pBlock->schemaLen); - - return len; -} - -static void extractTableNameList(SHashObj* pHashObj, bool freeBlockMap) { - // todo + return pBlock->dataLen + pBlock->schemaLen; } -int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap) { +int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, SArray** pVgDataBlocks) { const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); int code = 0; bool isRawPayload = IS_RAW_PAYLOAD(payloadType); @@ -537,24 +525,13 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t (isRawPayload ? (pOneTableBlock->rowSize + expandSize) : getExtendedRowSize(pOneTableBlock)) + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); - pBlocks->tid = htonl(pBlocks->tid); - pBlocks->uid = htobe64(pBlocks->uid); - pBlocks->sversion = htonl(pBlocks->sversion); - pBlocks->numOfRows = htons(pBlocks->numOfRows); - pBlocks->schemaLen = 0; - // erase the empty space reserved for binary data int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, schemaAttached, isRawPayload); assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); assert(dataBuf->size <= dataBuf->nAllocSize); - - // the length does not include the SSubmitBlk structure - pBlocks->dataLen = htonl(finalLen); dataBuf->numOfTables += 1; - - pBlocks->numOfRows = 0; } p = taosHashIterate(pHashObj, p); @@ -565,12 +542,10 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t pOneTableBlock = *p; } - extractTableNameList(pHashObj, freeBlockMap); - // free the table data blocks; taosHashCleanup(pVnodeDataBlockHashList); tfree(blkKeyInfo.pKeyTuple); - + *pVgDataBlocks = pVnodeDataBlockList; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 877ecb2c85457c9aff9de9454306c9fffc559bf8..68aa7554833c34feee541d4c20a9ebd6b1e41873 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -31,33 +31,16 @@ pSql += index; \ } while (0) -#define CHECK_CODE(expr) \ - do { \ - int32_t code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - terrno = code; \ - return terrno; \ - } \ - } while (0) - -#define CHECK_CODE_1(expr, destroy) \ +#define NEXT_TOKEN_KEEP_SQL(pSql, sToken, index) \ do { \ - int32_t code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - (void)destroy; \ - terrno = code; \ - return terrno; \ - } \ + sToken = tStrGetToken(pSql, &index, false); \ } while (0) -#define CHECK_CODE_2(expr, destroy1, destroy2) \ +#define CHECK_CODE(expr) \ do { \ int32_t code = expr; \ if (TSDB_CODE_SUCCESS != code) { \ - (void)destroy1; \ - (void)destroy2; \ - terrno = code; \ - return terrno; \ + return code; \ } \ } while (0) @@ -67,12 +50,16 @@ enum { }; typedef struct SInsertParseContext { - SParseContext* pComCxt; - const char* pSql; - SMsgBuf msg; - struct SCatalog* pCatalog; - STableMeta* pTableMeta; - SHashObj* pTableBlockHashObj; // data block for each table. need release + SParseContext* pComCxt; // input + const char* pSql; // input + SMsgBuf msg; // input + STableMeta* pTableMeta; // each table + SParsedDataColInfo tags; // each table + SKVRowBuilder tagsBuilder; // each table + SHashObj* pVgroupsHashObj; // global + SHashObj* pTableBlockHashObj; // global + SArray* pTableDataBlocks; // global + SArray* pVgDataBlocks; // global int32_t totalNum; SInsertStmtInfo* pOutput; } SInsertParseContext; @@ -151,14 +138,6 @@ static int32_t toInt64(const char* z, int16_t type, int32_t n, int64_t* value, b return ret; } -static int32_t createInsertStmtInfo(SInsertStmtInfo **pInsertInfo) { - *pInsertInfo = calloc(1, sizeof(SQueryStmtInfo)); - if (NULL == *pInsertInfo) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - return TSDB_CODE_SUCCESS; -} - static int32_t skipInsertInto(SInsertParseContext* pCxt) { SToken sToken; NEXT_TOKEN(pCxt->pSql, sToken); @@ -179,14 +158,14 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD char* p = strnchr(pStname->z, TS_PATH_DELIMITER[0], pStname->n, false); if (NULL != p) { // db.table - strcpy(fullDbName, pCxt->pComCxt->pAcctId); - fullDbName[strlen(pCxt->pComCxt->pAcctId)] = TS_PATH_DELIMITER[0]; - strncpy(fullDbName, pStname->z, p - pStname->z); + int32_t n = sprintf(fullDbName, "%d.", pCxt->pComCxt->ctx.acctId); + strncpy(fullDbName + n, pStname->z, p - pStname->z); strncpy(tableName, p + 1, pStname->n - (p - pStname->z) - 1); } else { - snprintf(fullDbName, TSDB_FULL_DB_NAME_LEN, "%s.%s", pCxt->pComCxt->pAcctId, pCxt->pComCxt->pDbname); + snprintf(fullDbName, TSDB_FULL_DB_NAME_LEN, "%d.%s", pCxt->pComCxt->ctx.acctId, pCxt->pComCxt->ctx.db); strncpy(tableName, pStname->z, pStname->n); } + return TSDB_CODE_SUCCESS; } @@ -194,11 +173,13 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { char fullDbName[TSDB_FULL_DB_NAME_LEN] = {0}; char tableName[TSDB_TABLE_NAME_LEN] = {0}; CHECK_CODE(buildName(pCxt, pTname, fullDbName, tableName)); - CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &pCxt->pTableMeta)); + CHECK_CODE(catalogGetTableMeta(pCxt->pComCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &pCxt->pTableMeta)); + SVgroupInfo vg; + CHECK_CODE(catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &vg)); + CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); return TSDB_CODE_SUCCESS; } -// todo speedup by using hash list static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) { while (start < end) { if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { @@ -209,6 +190,51 @@ static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pS return -1; } +static void buildMsgHeader(SVgDataBlocks* blocks) { + SMsgDesc* desc = (SMsgDesc*)blocks->pData; + desc->numOfVnodes = htonl(1); + SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); + submit->header.vgId = htonl(blocks->vg.vgId); + submit->header.contLen = htonl(blocks->size - sizeof(SMsgDesc)); + submit->length = submit->header.contLen; + submit->numOfBlocks = htonl(blocks->numOfTables); + SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); + int32_t numOfBlocks = blocks->numOfTables; + while (numOfBlocks--) { + int32_t dataLen = blk->dataLen; + blk->uid = htobe64(blk->uid); + blk->tid = htonl(blk->tid); + blk->padding = htonl(blk->padding); + blk->sversion = htonl(blk->sversion); + blk->dataLen = htonl(blk->dataLen); + blk->schemaLen = htonl(blk->schemaLen); + blk->numOfRows = htons(blk->numOfRows); + blk = (SSubmitBlk*)(blk->data + dataLen); + } +} + +static int32_t buildOutput(SInsertParseContext* pCxt) { + size_t numOfVg = taosArrayGetSize(pCxt->pVgDataBlocks); + pCxt->pOutput->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); + if (NULL == pCxt->pOutput->pDataBlocks) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + for (size_t i = 0; i < numOfVg; ++i) { + STableDataBlocks* src = taosArrayGetP(pCxt->pVgDataBlocks, i); + SVgDataBlocks* dst = calloc(1, sizeof(SVgDataBlocks)); + if (NULL == dst) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + taosHashGetClone(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); + dst->numOfTables = src->numOfTables; + dst->size = src->size; + SWAP(dst->pData, src->pData, char*); + buildMsgHeader(dst); + taosArrayPush(pCxt->pOutput->pDataBlocks, &dst); + } + return TSDB_CODE_SUCCESS; +} + static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) { // once the data block is disordered, we do NOT keep previous timestamp any more if (!pDataBlocks->ordered) { @@ -219,16 +245,14 @@ static int32_t checkTimestamp(STableDataBlocks *pDataBlocks, const char *start) if (k == INT64_MIN) { if (pDataBlocks->tsSource == TSDB_USE_CLI_TS) { - return -1; - } else if (pDataBlocks->tsSource == -1) { - pDataBlocks->tsSource = TSDB_USE_SERVER_TS; + return TSDB_CODE_FAILED; // client time/server time can not be mixed } + pDataBlocks->tsSource = TSDB_USE_SERVER_TS; } else { if (pDataBlocks->tsSource == TSDB_USE_SERVER_TS) { - return -1; // client time/server time can not be mixed - } else if (pDataBlocks->tsSource == -1) { - pDataBlocks->tsSource = TSDB_USE_CLI_TS; + return TSDB_CODE_FAILED; // client time/server time can not be mixed } + pDataBlocks->tsSource = TSDB_USE_CLI_TS; } if (k <= pDataBlocks->prevTS && (pDataBlocks->tsSource == TSDB_USE_CLI_TS)) { @@ -525,7 +549,7 @@ static FORCE_INLINE int32_t parseOneValue(SInsertParseContext* pCxt, SToken* pTo } case TSDB_DATA_TYPE_BINARY: { // too long values will return invalid sql, not be truncated automatically - if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { // todo refactor + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { return buildSyntaxErrMsg(&pCxt->msg, "string data overflow", pToken->z); } return func(pToken->z, pToken->n, param); @@ -608,26 +632,22 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* } // pSql -> tag1_value, ...) -static int32_t parseTagsClause(SInsertParseContext* pCxt, SParsedDataColInfo* pSpd, SSchema* pTagsSchema, uint8_t precision) { - SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { - destroyBoundColumnInfo(pSpd); +static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, uint8_t precision) { + if (tdInitKVRowBuilder(&pCxt->tagsBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SKvParam param = {.builder = &kvRowBuilder}; + SKvParam param = {.builder = &pCxt->tagsBuilder}; SToken sToken; char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" - for (int i = 0; i < pSpd->numOfBound; ++i) { + for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN(pCxt->pSql, sToken); - SSchema* pSchema = &pTagsSchema[pSpd->boundedColumns[i]]; + SSchema* pSchema = &pTagsSchema[pCxt->tags.boundedColumns[i]]; param.schema = pSchema; - CHECK_CODE_2(parseOneValue(pCxt, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m), tdDestroyKVRowBuilder(&kvRowBuilder), destroyBoundColumnInfo(pSpd)); + CHECK_CODE(parseOneValue(pCxt, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m)); } - destroyBoundColumnInfo(pSpd); - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - tdDestroyKVRowBuilder(&kvRowBuilder); + SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder); if (NULL == row) { return buildInvalidOperationMsg(&pCxt->msg, "tag value expected"); } @@ -650,13 +670,12 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) } SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); - SParsedDataColInfo spd = {0}; - setBoundColumnInfo(&spd, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); + setBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); // pSql -> [(tag1_name, ...)] TAGS (tag1_value, ...) NEXT_TOKEN(pCxt->pSql, sToken); if (TK_LP == sToken.type) { - CHECK_CODE_1(parseBoundColumns(pCxt, &spd, pTagsSchema), destroyBoundColumnInfo(&spd)); + CHECK_CODE(parseBoundColumns(pCxt, &pCxt->tags, pTagsSchema)); NEXT_TOKEN(pCxt->pSql, sToken); } @@ -668,7 +687,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TK_LP != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, "( is expected", sToken.z); } - CHECK_CODE(parseTagsClause(pCxt, &spd, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision)); + CHECK_CODE(parseTagsClause(pCxt, pTagsSchema, getTableInfo(pCxt->pTableMeta).precision)); return TSDB_CODE_SUCCESS; } @@ -732,10 +751,12 @@ static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlo char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" SToken sToken; while (1) { - NEXT_TOKEN(pCxt->pSql, sToken); + int32_t index = 0; + NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index); if (TK_LP != sToken.type) { break; } + pCxt->pSql += index; if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { int32_t tSize; @@ -779,13 +800,30 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da return TSDB_CODE_SUCCESS; } +static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { + tfree(pCxt->pTableMeta); + destroyBoundColumnInfo(&pCxt->tags); + tdDestroyKVRowBuilder(&pCxt->tagsBuilder); +} + +static void destroyInsertParseContext(SInsertParseContext* pCxt) { + destroyInsertParseContextForTable(pCxt); + taosHashCleanup(pCxt->pVgroupsHashObj); + taosHashCleanup(pCxt->pTableBlockHashObj); + destroyBlockArrayList(pCxt->pTableDataBlocks); + destroyBlockArrayList(pCxt->pVgDataBlocks); +} + // tb_name // [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] // [(field1_name, ...)] // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; static int32_t parseInsertBody(SInsertParseContext* pCxt) { + // for each table while (1) { + destroyInsertParseContextForTable(pCxt); + SToken sToken; // pSql -> tb_name ... NEXT_TOKEN(pCxt->pSql, sToken); @@ -815,7 +853,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (TK_LP == sToken.type) { // pSql -> field1_name, ...) - CHECK_CODE_1(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta)), destroyBoundColumnInfo(&dataBuf->boundColumnInfo)); + CHECK_CODE(parseBoundColumns(pCxt, &dataBuf->boundColumnInfo, getTableColumnSchema(pCxt->pTableMeta))); NEXT_TOKEN(pCxt->pSql, sToken); } @@ -842,9 +880,9 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { } // merge according to vgId if (!TSDB_QUERY_HAS_TYPE(pCxt->pOutput->insertType, TSDB_QUERY_TYPE_STMT_INSERT) && taosHashGetSize(pCxt->pTableBlockHashObj) > 0) { - CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->schemaAttache, pCxt->pOutput->payloadType, true)); + CHECK_CODE(mergeTableDataBlocks(pCxt->pTableBlockHashObj, pCxt->pOutput->schemaAttache, pCxt->pOutput->payloadType, &pCxt->pVgDataBlocks)); } - return TSDB_CODE_SUCCESS; + return buildOutput(pCxt); } // INSERT INTO @@ -854,26 +892,31 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { - CHECK_CODE(createInsertStmtInfo(pInfo)); - SInsertParseContext context = { .pComCxt = pContext, .pSql = pContext->pSql, .msg = {.buf = pContext->pMsg, .len = pContext->msgLen}, - .pCatalog = NULL, .pTableMeta = NULL, + .pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false), .pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false), .totalNum = 0, - .pOutput = *pInfo + .pOutput = calloc(1, sizeof(SInsertStmtInfo)) }; - if (NULL == context.pTableBlockHashObj) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pOutput) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; } - CHECK_CODE(catalogGetHandle(pContext->pClusterId, &context.pCatalog)); - CHECK_CODE(skipInsertInto(&context)); - CHECK_CODE(parseInsertBody(&context)); + *pInfo = context.pOutput; + context.pOutput->schemaAttache = pContext->schemaAttached; + context.pOutput->payloadType = PAYLOAD_TYPE_KV; - return TSDB_CODE_SUCCESS; + int32_t code = skipInsertInto(&context); + if (TSDB_CODE_SUCCESS == code) { + code = parseInsertBody(&context); + } + destroyInsertParseContext(&context); + terrno = code; + return (TSDB_CODE_SUCCESS == code ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED); } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index e0ac7c295bc82d5fe9dff60519dd9ede7efe7447..3490580f155f2f583c487852f3803091ef49c48d 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -31,7 +31,7 @@ bool qIsInsertSql(const char* pStr, size_t length) { } while (1); } -int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen) { +int32_t qParseQuerySql(const char* pStr, size_t length, SParseBasicCtx* pParseCtx, int32_t *type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen) { SSqlInfo info = doGenerateAST(pStr); if (!info.valid) { strncpy(msg, info.msg, msgLen); @@ -39,8 +39,8 @@ int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *typ return terrno; } - if (isDclSqlStatement(&info)) { - int32_t code = qParserValidateDclSqlNode(&info, id, pOutput, outputLen, type, msg, msgLen); + if (!isDqlSqlStatement(&info)) { + int32_t code = qParserValidateDclSqlNode(&info, pParseCtx, pOutput, outputLen, type, msg, msgLen); if (code == TSDB_CODE_SUCCESS) { // do nothing } @@ -53,7 +53,7 @@ int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *typ struct SCatalog* pCatalog = NULL; int32_t code = catalogGetHandle(NULL, &pCatalog); - code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, id, msg, msgLen); + code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, pParseCtx->requestId, msg, msgLen); if (code == TSDB_CODE_SUCCESS) { *pOutput = pQueryInfo; } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index b72bc06324d3a33e960b713c421e0af77a3dd86d..307062ef155d952e7c95dcdcfe6b1b480d26fdff 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1167,7 +1167,7 @@ void cleanupTagCond(STagCond* pTagCond) { * @param tableIndex denote the table index for join query, where more than one table exists * @return */ -STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex) { +STableMetaInfo* getMetaInfo(const SQueryStmtInfo* pQueryInfo, int32_t tableIndex) { assert(pQueryInfo != NULL); if (pQueryInfo->pTableMetaInfo == NULL) { assert(pQueryInfo->numOfTables == 0); @@ -1613,7 +1613,18 @@ uint32_t convertRelationalOperator(SToken *pToken) { } bool isDclSqlStatement(SSqlInfo* pSqlInfo) { - return (pSqlInfo->type != TSDB_SQL_SELECT); + int32_t type = pSqlInfo->type; + return (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_CREATE_ACCT || type == TSDB_SQL_DROP_USER || + type == TSDB_SQL_DROP_ACCT || type == TSDB_SQL_SHOW); +} + +bool isDdlSqlStatement(SSqlInfo* pSqlInfo) { + int32_t type = pSqlInfo->type; + return (type == TSDB_SQL_CREATE_TABLE || type == TSDB_SQL_CREATE_DB || type == TSDB_SQL_DROP_DB); +} + +bool isDqlSqlStatement(SSqlInfo* pSqlInfo) { + return pSqlInfo->type == TSDB_SQL_SELECT; } #if 0 diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index c548f1556a64daeedb9c99033245404f594297ce..1ae0d9211a2925b27b415becfab1fc385d242965 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -1,10 +1,7 @@ +#include "os.h" #include "queryInfoUtil.h" -#include -#include "astGenerator.h" #include "function.h" -#include "os.h" #include "parser.h" -#include "parserInt.h" #include "parserUtil.h" static struct SSchema _s = { @@ -357,7 +354,7 @@ bool tscHasColumnFilter(SQueryStmtInfo* pQueryInfo) { return false; } -int32_t getExprFunctionLevel(SQueryStmtInfo* pQueryInfo) { +int32_t getExprFunctionLevel(const SQueryStmtInfo* pQueryInfo) { int32_t n = 10; int32_t level = 0; diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index d674462fc0f277fa5be0a7a5988f28eed15c1499..9922ff1be0e591916e03419131f273701be6f597 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -31,6 +31,7 @@ #include #include #include "astGenerator.h" +#include "parserInt.h" #include "tmsgtype.h" #include "ttoken.h" #include "ttokendef.h" diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index feae00827310a93e9117361cc5d7c682e3582a76..e722b873b6882f625fbc81e9347dcd90437a3337 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -17,5 +17,3 @@ TARGET_LINK_LIBRARIES( parserTest PUBLIC os util common parser catalog transport gtest function planner qcom ) - -TARGET_LINK_OPTIONS(parserTest PRIVATE -Wl,-wrap,malloc) diff --git a/source/libs/parser/test/insertParserTest.cpp b/source/libs/parser/test/insertParserTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9e681c7ccb8b8d5412892c3b41413c0f455cf69b --- /dev/null +++ b/source/libs/parser/test/insertParserTest.cpp @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include "insertParser.h" +// #include "mockCatalog.h" + +using namespace std; +using namespace testing; + +namespace { + string toString(int32_t code) { + return tstrerror(code); + } +} + +// syntax: +// INSERT INTO +// tb_name +// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] +// [(field1_name, ...)] +// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path +// [...]; +class InsertTest : public Test { +protected: + void setDatabase(const string& acctId, const string& db) { + acctId_ = acctId; + db_ = db; + } + + void bind(const char* sql) { + reset(); + cxt_.ctx.acctId = atoi(acctId_.c_str()); + cxt_.ctx.db = (char*) db_.c_str(); + strcpy(sqlBuf_, sql); + cxt_.sqlLen = strlen(sql); + sqlBuf_[cxt_.sqlLen] = '\0'; + cxt_.pSql = sqlBuf_; + + } + + int32_t run() { + code_ = parseInsertSql(&cxt_, &res_); + if (code_ != TSDB_CODE_SUCCESS) { + cout << "code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; + } + return code_; + } + + SInsertStmtInfo* reslut() { + return res_; + } + + void dumpReslut() { + size_t num = taosArrayGetSize(res_->pDataBlocks); + cout << "schemaAttache:" << (int32_t)res_->schemaAttache << ", payloadType:" << (int32_t)res_->payloadType << ", insertType:" << res_->insertType << ", numOfVgs:" << num << endl; + for (size_t i = 0; i < num; ++i) { + SVgDataBlocks* vg = (SVgDataBlocks*)taosArrayGetP(res_->pDataBlocks, i); + cout << "vgId:" << vg->vg.vgId << ", numOfTables:" << vg->numOfTables << ", dataSize:" << vg->size << endl; + SMsgDesc* desc = (SMsgDesc*)(vg->pData); + cout << "numOfVnodes:" << ntohl(desc->numOfVnodes) << endl; + SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); + cout << "length:" << ntohl(submit->length) << ", numOfBlocks:" << ntohl(submit->numOfBlocks) << endl; + int32_t numOfBlocks = ntohl(submit->numOfBlocks); + SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); + for (int32_t i = 0; i < numOfBlocks; ++i) { + cout << "Block:" << i << endl; + cout << "\tuid:" << be64toh(blk->uid) << ", tid:" << ntohl(blk->tid) << ", padding:" << ntohl(blk->padding) << ", sversion:" << ntohl(blk->sversion) + << ", dataLen:" << ntohl(blk->dataLen) << ", schemaLen:" << ntohl(blk->schemaLen) << ", numOfRows:" << ntohs(blk->numOfRows) << endl; + blk = (SSubmitBlk*)(blk->data + ntohl(blk->dataLen)); + } + } + } + + void checkReslut(int32_t numOfTables, int16_t numOfRows1, int16_t numOfRows2 = -1) { + ASSERT_EQ(res_->schemaAttache, 0); + ASSERT_EQ(res_->payloadType, PAYLOAD_TYPE_KV); + ASSERT_EQ(res_->insertType, TSDB_QUERY_TYPE_INSERT); + size_t num = taosArrayGetSize(res_->pDataBlocks); + ASSERT_GE(num, 0); + for (size_t i = 0; i < num; ++i) { + SVgDataBlocks* vg = (SVgDataBlocks*)taosArrayGetP(res_->pDataBlocks, i); + ASSERT_EQ(vg->numOfTables, numOfTables); + ASSERT_GE(vg->size, 0); + SMsgDesc* desc = (SMsgDesc*)(vg->pData); + ASSERT_EQ(ntohl(desc->numOfVnodes), 1); + SSubmitMsg* submit = (SSubmitMsg*)(desc + 1); + ASSERT_GE(ntohl(submit->length), 0); + ASSERT_GE(ntohl(submit->numOfBlocks), 0); + int32_t numOfBlocks = ntohl(submit->numOfBlocks); + SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); + for (int32_t i = 0; i < numOfBlocks; ++i) { + ASSERT_EQ(ntohs(blk->numOfRows), (0 == i ? numOfRows1 : (numOfRows2 > 0 ? numOfRows2 : numOfRows1))); + blk = (SSubmitBlk*)(blk->data + ntohl(blk->dataLen)); + } + } + } + +private: + static const int max_err_len = 1024; + static const int max_sql_len = 1024 * 1024; + + void reset() { + memset(&cxt_, 0, sizeof(cxt_)); + memset(errMagBuf_, 0, max_err_len); + cxt_.pMsg = errMagBuf_; + cxt_.msgLen = max_err_len; + code_ = TSDB_CODE_SUCCESS; + res_ = nullptr; + } + + string acctId_; + string db_; + char errMagBuf_[max_err_len]; + char sqlBuf_[max_sql_len]; + SParseContext cxt_; + int32_t code_; + SInsertStmtInfo* res_; +}; + +// INSERT INTO tb_name VALUES (field1_value, ...) +TEST_F(InsertTest, singleTableSingleRowTest) { + setDatabase("root", "test"); + + bind("insert into t1 values (now, 1, \"beijing\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(1, 1); +} + +// INSERT INTO tb_name VALUES (field1_value, ...)(field1_value, ...) +TEST_F(InsertTest, singleTableMultiRowTest) { + setDatabase("root", "test"); + + bind("insert into t1 values (now, 1, \"beijing\")(now+1s, 2, \"shanghai\")(now+2s, 3, \"guangzhou\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(1, 3); +} + +// INSERT INTO tb1_name VALUES (field1_value, ...) tb2_name VALUES (field1_value, ...) +TEST_F(InsertTest, multiTableSingleRowTest) { + setDatabase("root", "test"); + + bind("insert into st1s1 values (now, 1, \"beijing\") st1s2 values (now, 10, \"131028\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(2, 1); +} + +// INSERT INTO tb1_name VALUES (field1_value, ...) tb2_name VALUES (field1_value, ...) +TEST_F(InsertTest, multiTableMultiRowTest) { + setDatabase("root", "test"); + + bind("insert into st1s1 values (now, 1, \"beijing\")(now+1s, 2, \"shanghai\")(now+2s, 3, \"guangzhou\")" + " st1s2 values (now, 10, \"131028\")(now+1s, 20, \"132028\")"); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + dumpReslut(); + checkReslut(2, 3, 2); +} + +TEST_F(InsertTest, toleranceTest) { + setDatabase("root", "test"); + + bind("insert into"); + ASSERT_NE(run(), TSDB_CODE_SUCCESS); + bind("insert into t"); + ASSERT_NE(run(), TSDB_CODE_SUCCESS); +} diff --git a/source/libs/parser/test/insertTest.cpp b/source/libs/parser/test/insertTest.cpp deleted file mode 100644 index 85db46d7bf1cfa29576926a0dafc7ef52310a2a8..0000000000000000000000000000000000000000 --- a/source/libs/parser/test/insertTest.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include - -#include "insertParser.h" -#include "mockCatalog.h" - -using namespace std; -using namespace testing; - -namespace { - string toString(int32_t code) { - return tstrerror(code); - } -} - -extern "C" { - -#include - -void *__real_malloc(size_t); - -void *__wrap_malloc(size_t c) { - // printf("My MALLOC called: %d\n", c); - // void *array[32]; - // int size = backtrace(array, 32); - // char **symbols = backtrace_symbols(array, size); - // for (int i = 0; i < size; ++i) { - // cout << symbols[i] << endl; - // } - // free(symbols); - - return __real_malloc(c); -} - -} - -// syntax: -// INSERT INTO -// tb_name -// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] -// [(field1_name, ...)] -// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path -// [...]; -class InsertTest : public Test { -protected: - void setDatabase(const string& acctId, const string& db) { - acctId_ = acctId; - db_ = db; - } - - void bind(const char* sql) { - reset(); - cxt_.pAcctId = acctId_.c_str(); - cxt_.pDbname = db_.c_str(); - strcpy(sqlBuf_, sql); - cxt_.sqlLen = strlen(sql); - sqlBuf_[cxt_.sqlLen] = '\0'; - cxt_.pSql = sqlBuf_; - - } - - int32_t run() { - code_ = parseInsertSql(&cxt_, &res_); - if (code_ != TSDB_CODE_SUCCESS) { - cout << "code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; - } - return code_; - } - - SInsertStmtInfo* reslut() { - return res_; - } - -private: - static const int max_err_len = 1024; - static const int max_sql_len = 1024 * 1024; - - void reset() { - memset(&cxt_, 0, sizeof(cxt_)); - memset(errMagBuf_, 0, max_err_len); - cxt_.pMsg = errMagBuf_; - cxt_.msgLen = max_err_len; - code_ = TSDB_CODE_SUCCESS; - res_ = nullptr; - } - - string acctId_; - string db_; - char errMagBuf_[max_err_len]; - char sqlBuf_[max_sql_len]; - SParseContext cxt_; - int32_t code_; - SInsertStmtInfo* res_; -}; - -// INSERT INTO tb_name VALUES (field1_value, ...) -TEST_F(InsertTest, simpleTest) { - setDatabase("root", "test"); - - bind("insert into t1 values (now, 1, \"wxy\")"); - ASSERT_EQ(run(), TSDB_CODE_SUCCESS); - SInsertStmtInfo* res = reslut(); - // todo check - ASSERT_EQ(res->insertType, TSDB_QUERY_TYPE_INSERT); - // ASSERT_EQ(taosArrayGetSize(res->pDataBlocks), 1); -} - -TEST_F(InsertTest, toleranceTest) { - setDatabase("root", "test"); - - bind("insert into"); - ASSERT_NE(run(), TSDB_CODE_SUCCESS); - bind("insert into t"); - ASSERT_NE(run(), TSDB_CODE_SUCCESS); -} diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 4e6b94081b4e993a17d8aeb567fb228b55c8d55b..6d8973366803dc30f5cd29f5baed40f0563facf3 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -17,20 +17,23 @@ #include +#include "stub.h" +#include "addr_any.h" + namespace { void generateTestT1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("root.test", "t1", TSDB_NORMAL_TABLE, 3) .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) - .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20); builder.done(); } void generateTestST1(MockCatalogService* mcs) { ITableBuilder& builder = mcs->createTableBuilder("root.test", "st1", TSDB_SUPER_TABLE, 3, 2) .setPrecision(TSDB_TIME_PRECISION_MILLI).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) - .addTag("tag1", TSDB_DATA_TYPE_INT).addTag("tag2", TSDB_DATA_TYPE_BINARY, 10) - .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); + .addTag("tag1", TSDB_DATA_TYPE_INT).addTag("tag2", TSDB_DATA_TYPE_BINARY, 20) + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20); builder.done(); mcs->createSubTable("root.test", "st1", "st1s1", 1); mcs->createSubTable("root.test", "st1", "st1s2", 2); @@ -38,16 +41,49 @@ void generateTestST1(MockCatalogService* mcs) { } -int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { - return mockCatalogService->catalogGetHandle(clusterId, catalogHandle); +int32_t __catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { + return 0; +} + +int32_t __catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { + return mockCatalogService->catalogGetTableMeta(pDBName, pTableName, pTableMeta); } -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { - return mockCatalogService->catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, pTableMeta); +int32_t __catalogGetTableHashVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) { + return mockCatalogService->catalogGetTableHashVgroup(pDBName, pTableName, vgInfo); } void initMetaDataEnv() { mockCatalogService.reset(new MockCatalogService()); + + static Stub stub; + stub.set(catalogGetHandle, __catalogGetHandle); + stub.set(catalogGetTableMeta, __catalogGetTableMeta); + stub.set(catalogGetTableHashVgroup, __catalogGetTableHashVgroup); + { + AddrAny any("libcatalog.so"); + std::map result; + any.get_global_func_addr_dynsym("^catalogGetHandle$", result); + for (const auto& f : result) { + stub.set(f.second, __catalogGetHandle); + } + } + { + AddrAny any("libcatalog.so"); + std::map result; + any.get_global_func_addr_dynsym("^catalogGetTableMeta$", result); + for (const auto& f : result) { + stub.set(f.second, __catalogGetTableMeta); + } + } + { + AddrAny any("libcatalog.so"); + std::map result; + any.get_global_func_addr_dynsym("^catalogGetTableHashVgroup$", result); + for (const auto& f : result) { + stub.set(f.second, __catalogGetTableHashVgroup); + } + } } void generateMetaData() { diff --git a/source/libs/parser/test/mockCatalog.h b/source/libs/parser/test/mockCatalog.h index c9812d5b946fc4ce99d9be4bd7100ec03b131e68..f8e80a48aa5dacc7dc6d90d1a9db4623bfc73278 100644 --- a/source/libs/parser/test/mockCatalog.h +++ b/source/libs/parser/test/mockCatalog.h @@ -22,8 +22,4 @@ void initMetaDataEnv(); void generateMetaData(); void destroyMetaDataEnv(); -// mock -int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); -int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); - #endif // MOCK_CATALOG_H diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 9bc029cbd8d23906872f954aad998caff6262c68..e234f82da93aad8612e57673fd89f64025d52685 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -33,12 +33,13 @@ public: col->colId = colId_++; col->bytes = bytes; strcpy(col->name, name.c_str()); + rowsize_ += bytes; return *this; } virtual TableBuilder& setVgid(int16_t vgid) { schema()->vgId = vgid; - meta_->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + meta_->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); return *this; } @@ -89,11 +90,11 @@ public: MockCatalogServiceImpl() : id_(1) { } - int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const { + int32_t catalogGetHandle() const { return 0; } - int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { + int32_t catalogGetTableMeta(const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { std::unique_ptr table; int32_t code = copyTableSchemaMeta(pDBName, pTableName, &table); if (TSDB_CODE_SUCCESS != code) { @@ -103,6 +104,11 @@ public: return TSDB_CODE_SUCCESS; } + int32_t catalogGetTableHashVgroup(const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) const { + // todo + return 0; + } + TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags); meta_[db][tbname] = builder_->table(); @@ -118,9 +124,9 @@ public: meta_[db][tbname].reset(new MockTableMeta()); meta_[db][tbname]->schema.reset(table.release()); meta_[db][tbname]->schema->uid = id_++; - meta_[db][tbname]->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + meta_[db][tbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); // super table - meta_[db][stbname]->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + meta_[db][stbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); } void showTables() const { @@ -147,11 +153,11 @@ public: for (const auto& db : meta_) { std::cout << "Databse:" << db.first << std::endl; - std::cout << SH("Table") << SH("Type") << SH("Precision") << IH("Vgid") << std::endl; + std::cout << SH("Table") << SH("Type") << SH("Precision") << IH("Vgid") << IH("RowSize") << std::endl; std::cout << SL(3, 1) << std::endl; for (const auto& table : db.second) { const auto& schema = table.second->schema; - std::cout << SF(table.first) << SF(ttToString(schema->tableType)) << SF(pToString(schema->tableInfo.precision)) << IF(schema->vgId) << std::endl; + std::cout << SF(table.first) << SF(ttToString(schema->tableType)) << SF(pToString(schema->tableInfo.precision)) << IF(schema->vgId) << IF(schema->tableInfo.rowSize) << std::endl; } std::cout << std::endl; } @@ -269,10 +275,10 @@ std::shared_ptr MockCatalogService::getTableMeta(const std::strin return impl_->getTableMeta(db, tbname); } -int32_t MockCatalogService::catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const { - return impl_->catalogGetHandle(clusterId, catalogHandle); +int32_t MockCatalogService::catalogGetTableMeta(const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { + return impl_->catalogGetTableMeta(pDBName, pTableName, pTableMeta); } -int32_t MockCatalogService::catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const { - return impl_->catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, pTableMeta); -} +int32_t MockCatalogService::catalogGetTableHashVgroup(const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) const { + return impl_->catalogGetTableHashVgroup(pDBName, pTableName, vgInfo); +} \ No newline at end of file diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index cd01db09cce7ff4c5949671ff44af041a47d80cb..889a45439715792d09a33d4c9073b4175a275dca 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -57,9 +57,8 @@ public: void showTables() const; std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const; - // mock interface - int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) const; - int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const; + int32_t catalogGetTableMeta(const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const; + int32_t catalogGetTableHashVgroup(const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) const; private: std::unique_ptr impl_; diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 396e8a12fe584cdd2a4af47279c1ac01b4ba5f98..be68149e09a7d57f1315a3d7f6a56dcfdbcdeb18 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -717,7 +717,9 @@ TEST(testCase, show_user_Test) { void* output = NULL; int32_t type = 0; int32_t len = 0; - int32_t code = qParserValidateDclSqlNode(&info1, 1, &output, &len, &type, msg, buf.len); + + SParseBasicCtx ct= {.db= "abc", .acctId = 1, .requestId = 1}; + int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, &len, &type, msg, buf.len); ASSERT_EQ(code, 0); // convert the show command to be the select query @@ -734,13 +736,14 @@ TEST(testCase, create_user_Test) { SSqlInfo info1 = doGenerateAST(sql); ASSERT_EQ(info1.valid, true); - ASSERT_EQ(isDclSqlStatement(&info1), true); void* output = NULL; int32_t type = 0; int32_t len = 0; - int32_t code = qParserValidateDclSqlNode(&info1, 1, &output, &len, &type, msg, buf.len); + + SParseBasicCtx ct= {.db= "abc", .acctId = 1, .requestId = 1}; + int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, &len, &type, msg, buf.len); ASSERT_EQ(code, 0); destroySqlInfo(&info1); diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index a2078defdaf1acbe5b9e0824f08a934371345d6c..f9b172210627cede4727f8e1a3eeaa2eb9ee3ae1 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -93,7 +93,7 @@ void generateLogicplan(const char* sql) { ASSERT_EQ(ret, 0); struct SQueryPlanNode* n = nullptr; - code = createQueryPlan(pQueryInfo, &n); + code = createQueryPlan((const SQueryNode*)pQueryInfo, &n); char* str = NULL; queryPlanToString(n, &str); @@ -156,7 +156,7 @@ TEST(testCase, planner_test) { ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2); struct SQueryPlanNode* n = nullptr; - code = createQueryPlan(pQueryInfo, &n); + code = createQueryPlan((const SQueryNode*)pQueryInfo, &n); char* str = NULL; queryPlanToString(n, &str); diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index b1c4643f154f56be96a096d09161e1f0aa74540a..1bee95b8e56965f424488e38c3fe2da6a66de677 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -40,6 +40,7 @@ extern "C" { #define QNODE_SESSIONWINDOW 12 #define QNODE_STATEWINDOW 13 #define QNODE_FILL 14 +#define QNODE_INSERT 15 typedef struct SQueryDistPlanNodeInfo { bool stableQuery; // super table query or not @@ -82,7 +83,7 @@ int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode); * @param pQueryNode * @return */ -int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode); +int32_t createQueryPlan(const SQueryNode* pNode, struct SQueryPlanNode** pQueryPlan); /** * Convert the query plan to string, in order to display it in the shell. diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index ce923314bd9530b6425fff926cb3b9662f57b813..136073aa6030945962b6740bfd3f74d55b30fc78 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -29,7 +29,7 @@ typedef struct SJoinCond { SColumn *colCond[2]; } SJoinCond; -static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo); +static SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo); static void doDestroyQueryNode(SQueryPlanNode* pQueryNode); int32_t printExprInfo(char* buf, const SQueryPlanNode* pQueryNode, int32_t len); @@ -37,13 +37,37 @@ int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { return 0; } -int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) { - SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo); +int32_t createInsertPlan(const SInsertStmtInfo* pInsert, SQueryPlanNode** pQueryPlan) { + *pQueryPlan = calloc(1, sizeof(SQueryPlanNode)); + SArray* blocks = taosArrayInit(taosArrayGetSize(pInsert->pDataBlocks), POINTER_BYTES); + if (NULL == *pQueryPlan || NULL == blocks) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + (*pQueryPlan)->info.type = QNODE_INSERT; + taosArrayAddAll(blocks, pInsert->pDataBlocks); + (*pQueryPlan)->pExtInfo = blocks; + return TSDB_CODE_SUCCESS; +} + +int32_t createSelectPlan(const SQueryStmtInfo* pSelect, SQueryPlanNode** pQueryPlan) { + SArray* upstream = createQueryPlanImpl(pSelect); assert(taosArrayGetSize(upstream) == 1); + *pQueryPlan = taosArrayGetP(upstream, 0); + taosArrayDestroy(upstream); + return TSDB_CODE_SUCCESS; +} - *pQueryNode = taosArrayGetP(upstream, 0); +int32_t createQueryPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) { + switch (nodeType(pNode)) { + case TSDB_SQL_SELECT: { + return createSelectPlan((const SQueryStmtInfo*)pNode, pQueryPlan); + } + case TSDB_SQL_INSERT: + return createInsertPlan((const SInsertStmtInfo*)pNode, pQueryPlan); + default: + return TSDB_CODE_FAILED; + } - taosArrayDestroy(upstream); return TSDB_CODE_SUCCESS; } @@ -62,7 +86,7 @@ void destroyQueryPlan(SQueryPlanNode* pQueryNode) { //====================================================================================================================== static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev, - SExprInfo** pExpr, int32_t numOfOutput, void* pExtInfo) { + SExprInfo** pExpr, int32_t numOfOutput, const void* pExtInfo) { SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode)); pNode->info.type = type; @@ -123,7 +147,7 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla } case QNODE_FILL: { // todo !! - pNode->pExtInfo = pExtInfo; + pNode->pExtInfo = (void*)pExtInfo; break; } @@ -145,7 +169,7 @@ static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPla return pNode; } -static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, +static SQueryPlanNode* doAddTableColumnNode(const SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, SArray* pExprs, SArray* tableCols) { if (pQueryInfo->info.onlyTagQuery) { int32_t num = (int32_t) taosArrayGetSize(pExprs); @@ -186,7 +210,7 @@ static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMe return pNode; } -static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) { +static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(const SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) { // group by column not by tag size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); @@ -239,7 +263,7 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQuer memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr); SArray* p = pQueryInfo->exprList[0]; // top expression in select clause - pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p, taosArrayGetSize(p), pInfo); + pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p->pData, taosArrayGetSize(p), pInfo); } if (pQueryInfo->order != NULL) { @@ -254,7 +278,7 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQuer return pNode; } -static SQueryPlanNode* doCreateQueryPlanForSingleTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, +static SQueryPlanNode* doCreateQueryPlanForSingleTable(const SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, SArray* tableCols) { char name[TSDB_TABLE_FNAME_LEN] = {0}; tstrncpy(name, pTableMetaInfo->name.tname, TSDB_TABLE_FNAME_LEN); @@ -286,7 +310,7 @@ static bool isAllAggExpr(SArray* pList) { return true; } -SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { +SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) { SArray* upstream = NULL; if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 1d29e48e308e1b60ee60e35ea6e6c3dcf489e643..9db9e059b33df0bfe8f6e70492b8b6151bb0a395 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -14,6 +14,7 @@ */ #include "plannerInt.h" +#include "exception.h" #include "parser.h" #define STORE_CURRENT_SUBPLAN(cxt) SSubplan* _ = cxt->pCurrentSubplan @@ -33,6 +34,13 @@ static const char* gOpName[] = { #undef INCLUDE_AS_NAME }; +static void* vailidPointer(void* p) { + if (NULL == p) { + THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); + } + return p; +} + const char* opTypeToOpName(int32_t type) { return gOpName[type]; } @@ -46,6 +54,25 @@ int32_t opNameToOpType(const char* name) { return OP_Unknown; } +static SDataSink* initDataSink(int32_t type, int32_t size) { + SDataSink* sink = (SDataSink*)vailidPointer(calloc(1, size)); + sink->info.type = type; + return sink; +} + +static SDataSink* createDataDispatcher(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + SDataDispatcher* dispatcher = (SDataDispatcher*)initDataSink(DSINK_Dispatch, sizeof(SDataDispatcher)); + return (SDataSink*)dispatcher; +} + +static SDataSink* createDataInserter(SPlanContext* pCxt, SVgDataBlocks* pBlocks) { + SDataInserter* inserter = (SDataInserter*)initDataSink(DSINK_Insert, sizeof(SDataInserter)); + inserter->numOfTables = pBlocks->numOfTables; + inserter->size = pBlocks->size; + SWAP(inserter->pData, pBlocks->pData, char*); + return (SDataSink*)inserter; +} + static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { dataBlockSchema->numOfCols = pPlanNode->numOfCols; dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfCols); @@ -72,10 +99,7 @@ static bool cloneExprArray(SArray** dst, SArray* src) { } static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) { - SPhyNode* node = (SPhyNode*)calloc(1, size); - if (NULL == node) { - return NULL; - } + SPhyNode* node = (SPhyNode*)vailidPointer(calloc(1, size)); node->info.type = type; node->info.name = opTypeToOpName(type); if (!cloneExprArray(&node->pTargets, pPlanNode->pExpr) || !toDataBlockSchema(pPlanNode, &(node->targetSchema))) { @@ -138,7 +162,7 @@ static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTable } static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { - SSubplan* subplan = calloc(1, sizeof(SSubplan)); + SSubplan* subplan = vailidPointer(calloc(1, sizeof(SSubplan))); subplan->id = pCxt->nextId; ++(pCxt->nextId.subplanId); subplan->type = type; @@ -146,15 +170,15 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { if (NULL != pCxt->pCurrentSubplan) { subplan->level = pCxt->pCurrentSubplan->level + 1; if (NULL == pCxt->pCurrentSubplan->pChildern) { - pCxt->pCurrentSubplan->pChildern = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + pCxt->pCurrentSubplan->pChildern = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); } taosArrayPush(pCxt->pCurrentSubplan->pChildern, &subplan); - subplan->pParents = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + subplan->pParents = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); taosArrayPush(subplan->pParents, &pCxt->pCurrentSubplan); } SArray* currentLevel; if (subplan->level >= taosArrayGetSize(pCxt->pDag->pSubplans)) { - currentLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + currentLevel = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); taosArrayPush(pCxt->pDag->pSubplans, ¤tLevel); } else { currentLevel = taosArrayGetP(pCxt->pDag->pSubplans, subplan->level); @@ -164,7 +188,17 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { return subplan; } -static void vgroupToEpSet(const SVgroupMsg* vg, SEpSet* epSet) { +static void vgroupInfoToEpSet(const SVgroupInfo* vg, SEpSet* epSet) { + epSet->inUse = 0; // todo + epSet->numOfEps = vg->numOfEps; + for (int8_t i = 0; i < vg->numOfEps; ++i) { + epSet->port[i] = vg->epAddr[i].port; + strcpy(epSet->fqdn[i], vg->epAddr[i].fqdn); + } + return; +} + +static void vgroupMsgToEpSet(const SVgroupMsg* vg, SEpSet* epSet) { epSet->inUse = 0; // todo epSet->numOfEps = vg->numOfEps; for (int8_t i = 0; i < vg->numOfEps; ++i) { @@ -179,8 +213,9 @@ static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNod for (int32_t i = 0; i < pTable->pMeta->vgroupList->numOfVgroups; ++i) { STORE_CURRENT_SUBPLAN(pCxt); SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN); - vgroupToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execEpSet); + vgroupMsgToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execEpSet); subplan->pNode = createMultiTableScanNode(pPlanNode, pTable); + subplan->pDataSink = createDataDispatcher(pCxt, pPlanNode); RECOVERY_CURRENT_SUBPLAN(pCxt); } return pCxt->nextId.templateId++; @@ -214,6 +249,9 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { case QNODE_TABLESCAN: node = createTableScanNode(pCxt, pPlanNode); break; + case QNODE_INSERT: + // Insert is not an operator in a physical plan. + break; default: assert(false); } @@ -221,7 +259,7 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { node->pChildren = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); size_t size = taosArrayGetSize(pPlanNode->pChildren); for(int32_t i = 0; i < size; ++i) { - SPhyNode* child = createPhyNode(pCxt, taosArrayGet(pPlanNode->pChildren, i)); + SPhyNode* child = createPhyNode(pCxt, taosArrayGetP(pPlanNode->pChildren, i)); child->pParent = node; taosArrayPush(node->pChildren, &child); } @@ -229,26 +267,46 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { return node; } +static void splitInsertSubplan(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + SArray* vgs = (SArray*)pPlanNode->pExtInfo; + size_t numOfVg = taosArrayGetSize(vgs); + for (int32_t i = 0; i < numOfVg; ++i) { + SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MODIFY); + SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(vgs, i); + vgroupInfoToEpSet(&blocks->vg, &subplan->execEpSet); + subplan->pNode = NULL; + subplan->pDataSink = createDataInserter(pCxt, blocks); + } +} + static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { - SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE); - ++(pCxt->nextId.templateId); - subplan->pNode = createPhyNode(pCxt, pRoot); + if (QNODE_INSERT == pRoot->info.type) { + splitInsertSubplan(pCxt, pRoot); + } else { + SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE); + ++(pCxt->nextId.templateId); + subplan->pNode = createPhyNode(pCxt, pRoot); + subplan->pDataSink = createDataDispatcher(pCxt, pRoot); + } // todo deal subquery } int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag) { - SPlanContext context = { - .pCatalog = pCatalog, - .pDag = calloc(1, sizeof(SQueryDag)), - .pCurrentSubplan = NULL, - .nextId = {0} // todo queryid - }; - if (NULL == context.pDag) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - context.pDag->pSubplans = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); - createSubplanByLevel(&context, pQueryNode); - *pDag = context.pDag; + TRY(TSDB_MAX_TAG_CONDITIONS) { + SPlanContext context = { + .pCatalog = pCatalog, + .pDag = vailidPointer(calloc(1, sizeof(SQueryDag))), + .pCurrentSubplan = NULL, + .nextId = {0} // todo queryid + }; + *pDag = context.pDag; + context.pDag->pSubplans = vailidPointer(taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES)); + createSubplanByLevel(&context, pQueryNode); + } CATCH(code) { + CLEANUP_EXECUTE(); + terrno = code; + return TSDB_CODE_FAILED; + } END_TRY return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 2510797158978e3ba8fc4f0a6732a2f0a7c40ac6..15c0e632a7d7ccf1e2fa32f89e66f85f86e0259f 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -651,7 +651,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { static const char* jkPnodeName = "Name"; static const char* jkPnodeTargets = "Targets"; static const char* jkPnodeConditions = "Conditions"; -static const char* jkPnodeSchema = "Schema"; +static const char* jkPnodeSchema = "InputSchema"; static const char* jkPnodeChildren = "Children"; // The 'pParent' field do not need to be serialized. static bool phyNodeToJson(const void* obj, cJSON* jNode) { diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index e9a4591d4afe8b4b0efed0f322c9be6ba5c4ee40..7722a7d3639ffcfef62ef9610de5d1ff90daa4c4 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -24,9 +24,9 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { // todo } -int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag) { +int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SEpSet* pEpSet, struct SQueryDag** pDag) { SQueryPlanNode* logicPlan; - int32_t code = createQueryPlan(pQueryInfo, &logicPlan); + int32_t code = createQueryPlan(pNode, &logicPlan); if (TSDB_CODE_SUCCESS != code) { destroyQueryPlan(logicPlan); return code; diff --git a/source/libs/planner/test/plannerTests.cpp b/source/libs/planner/test/plannerTests.cpp index 9379b06ac8594bc961c30a57fb18cece1859a407..11a31d15eb330981549992c1dc40272b76c28b32 100644 --- a/source/libs/planner/test/plannerTests.cpp +++ b/source/libs/planner/test/plannerTests.cpp @@ -28,7 +28,7 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -class ParserEnv : public testing::Environment { +class PlannerEnv : public testing::Environment { public: virtual void SetUp() { initMetaDataEnv(); @@ -39,12 +39,12 @@ public: destroyMetaDataEnv(); } - ParserEnv() {} - virtual ~ParserEnv() {} + PlannerEnv() {} + virtual ~PlannerEnv() {} }; int main(int argc, char* argv[]) { - testing::AddGlobalTestEnvironment(new ParserEnv()); + testing::AddGlobalTestEnvironment(new PlannerEnv()); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/source/libs/qcom/CMakeLists.txt b/source/libs/qcom/CMakeLists.txt index 41cf1826bcd5e99a20093bf33dde102811f31039..8e09f3d97a7637a16d1617d5b675a6470dd2622d 100644 --- a/source/libs/qcom/CMakeLists.txt +++ b/source/libs/qcom/CMakeLists.txt @@ -10,3 +10,5 @@ target_link_libraries( qcom PRIVATE os util transport ) + +ADD_SUBDIRECTORY(test) diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 2a13b708ec537a0eb3d7ff8611c681b8b4e285f9..829f426c9ddf1e2be1d3cab0220188eb1ad0d0c8 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -1,5 +1,8 @@ #include "os.h" #include "taosmsg.h" +#include "query.h" +#include "tglobal.h" +#include "tsched.h" #define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) #define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) @@ -75,4 +78,46 @@ bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTag } return true; -} \ No newline at end of file +} + +static void* pTaskQueue = NULL; + +int32_t initTaskQueue() { + double factor = 4.0; + int32_t numOfThreads = MAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + + int32_t queueSize = tsMaxConnections * 2; + pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); + if (NULL == pTaskQueue) { + qError("failed to init task queue"); + return -1; + } + + qDebug("task queue is initialized, numOfThreads: %d", numOfThreads); +} + +int32_t cleanupTaskQueue() { + taosCleanUpScheduler(pTaskQueue); +} + +static void execHelper(struct SSchedMsg* pSchedMsg) { + assert(pSchedMsg != NULL && pSchedMsg->ahandle != NULL); + + __async_exec_fn_t* execFn = (__async_exec_fn_t*) pSchedMsg->ahandle; + int32_t code = execFn(pSchedMsg->thandle); + if (code != 0 && pSchedMsg->msg != NULL) { + *(int32_t*) pSchedMsg->msg = code; + } +} + +int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) { + assert(execFn != NULL); + + SSchedMsg schedMsg = {0}; + schedMsg.fp = execHelper; + schedMsg.ahandle = execFn; + schedMsg.thandle = execParam; + schedMsg.msg = code; + + taosScheduleTask(pTaskQueue, &schedMsg); +} diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 9ca8b6789b328dc00f8d23e1c35264b70ece8413..728c47af1e71e4c8b15d1f2a921c34149f3c098f 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -294,7 +294,7 @@ void msgInit() { tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; - tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAG_VAL] = tscBuildUpdateTagMsg; tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; diff --git a/source/libs/qcom/test/CMakeLists.txt b/source/libs/qcom/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..7adec3752aeadefb791b203b66d5c40bb3fcae5b --- /dev/null +++ b/source/libs/qcom/test/CMakeLists.txt @@ -0,0 +1,19 @@ + +MESSAGE(STATUS "build qcom unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(queryUtilTest ${SOURCE_LIST}) + +TARGET_INCLUDE_DIRECTORIES( + queryUtilTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qcom/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/qcom/inc" +) + +TARGET_LINK_LIBRARIES( + queryUtilTest + PUBLIC os util gtest qcom common +) diff --git a/source/libs/qcom/test/queryTest.cpp b/source/libs/qcom/test/queryTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d2aedff8fdee781d1531e3aeea04e523f1a90beb --- /dev/null +++ b/source/libs/qcom/test/queryTest.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "taosmsg.h" +#include "query.h" +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +namespace { +typedef struct SParam { + int32_t v; +} SParam; +int32_t testPrint(void* p) { + SParam* param = (SParam*)p; + printf("hello world, %d\n", param->v); + tfree(p); + return 0; +} + +int32_t testPrintError(void* p) { + SParam* param = (SParam*) p; + tfree(p); + + return -1; +} +} // namespace + +class QueryTestEnv : public testing::Environment { + public: + virtual void SetUp() { initTaskQueue(); } + + virtual void TearDown() { cleanupTaskQueue(); } + + QueryTestEnv() {} + virtual ~QueryTestEnv() {} +}; + +int main(int argc, char** argv) { + testing::AddGlobalTestEnvironment(new QueryTestEnv()); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, async_task_test) { + SParam* p = (SParam*)calloc(1, sizeof(SParam)); + taosAsyncExec(testPrint, p, NULL); + usleep(5000); +} + +TEST(testCase, many_async_task_test) { + for(int32_t i = 0; i < 50; ++i) { + SParam* p = (SParam*) calloc(1, sizeof(SParam)); + p->v = i; + taosAsyncExec(testPrint, p, NULL); + } + + usleep(10000); +} + +TEST(testCase, error_in_async_test) { + int32_t code = 0; + SParam* p = (SParam*) calloc(1, sizeof(SParam)); + taosAsyncExec(testPrintError, p, &code); + usleep(1000); + printf("Error code:%d after asynchronously exec function\n", code); +} \ No newline at end of file diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..4eafa50bdc200615391cfc2f364fc9c9c20430bd --- /dev/null +++ b/source/libs/qworker/CMakeLists.txt @@ -0,0 +1,12 @@ +aux_source_directory(src QWORKER_SRC) +add_library(qworker ${QWORKER_SRC}) +target_include_directories( + qworker + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + qworker + PRIVATE os util transport planner qcom +) diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h new file mode 100644 index 0000000000000000000000000000000000000000..1adc09def41234a26d3557916eb40917e6e1b57e --- /dev/null +++ b/source/libs/qworker/inc/qworkerInt.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_QWORKER_INT_H_ +#define _TD_QWORKER_INT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define QWORKER_DEFAULT_SCHEDULER_NUMBER 10000 +#define QWORKER_DEFAULT_RES_CACHE_NUMBER 10000 +#define QWORKER_DEFAULT_SCH_TASK_NUMBER 10000 + +enum { + QW_READY_NOT_RECEIVED = 0, + QW_READY_RECEIVED, + QW_READY_RESPONSED, +}; + +typedef struct SQWorkerTaskStatus { + int8_t status; + int8_t ready; +} SQWorkerTaskStatus; + +typedef struct SQWorkerResCache { + void *data; +} SQWorkerResCache; + +typedef struct SQWorkerSchTaskStatus { + int32_t lastAccessTs; // timestamp in second + SHashObj *taskStatus; // key:queryId+taskId, value: SQWorkerTaskStatus +} SQWorkerSchTaskStatus; + +// Qnode/Vnode level task management +typedef struct SQWorkerMgmt { + SQWorkerCfg cfg; + SHashObj *scheduleHash; //key: schedulerId, value: SQWorkerSchTaskStatus + SHashObj *resHash; //key: queryId+taskId, value: SQWorkerResCache +} SQWorkerMgmt; + +#define QW_TASK_DONE(status) (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == status == JOB_TASK_STATUS_CANCELLED) +#define QW_SET_QTID(id, qid, tid) do { *(uint64_t *)(id) = (qid); *(uint64_t *)((char *)(id) + sizeof(qid)) = (tid); } while (0) +#define QW_GET_QTID(id, qid, tid) do { (qid) = *(uint64_t *)(id); (tid) = *(uint64_t *)((char *)(id) + sizeof(qid)); } while (0) + +#define QW_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define QW_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define QW_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); terrno = _code; return _code; } } while (0) +#define QW_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_QWORKER_INT_H_*/ diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c new file mode 100644 index 0000000000000000000000000000000000000000..82bfd75b6a1d2ffb22a4c11b6f286b185d6a643a --- /dev/null +++ b/source/libs/qworker/src/qworker.c @@ -0,0 +1,298 @@ +#include "taosmsg.h" +#include "query.h" +#include "qworker.h" +#include "qworkerInt.h" +#include "planner.h" + +int32_t qwAddTaskStatus(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId, int8_t taskStatus) { + SQWorkerTaskStatus tStatus = {0}; + tStatus.status = taskStatus; + + char id[sizeof(queryId) + sizeof(taskId)] = {0}; + QW_SET_QTID(id, queryId, taskId); + + SQWorkerSchTaskStatus *schStatus = taosHashGet(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId)); + if (NULL == schStatus) { + SQWorkerSchTaskStatus newSchStatus = {0}; + newSchStatus.taskStatus = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == newSchStatus.taskStatus) { + qError("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + if (0 != taosHashPut(newSchStatus.taskStatus, id, sizeof(id), &tStatus, sizeof(tStatus))) { + qError("taosHashPut schedulerId[%"PRIx64"]queryId[%"PRIx64"] taskId[%"PRIx64"] to scheduleHash failed", schedulerId, queryId, taskId); + taosHashCleanup(newSchStatus.taskStatus); + return TSDB_CODE_QRY_APP_ERROR; + } + + newSchStatus.lastAccessTs = taosGetTimestampSec(); + + if (0 != taosHashPut(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId), &newSchStatus, sizeof(newSchStatus))) { + qError("taosHashPut schedulerId[%"PRIx64"] to scheduleHash failed", schedulerId); + taosHashCleanup(newSchStatus.taskStatus); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; + } + + schStatus->lastAccessTs = taosGetTimestampSec(); + + if (0 != taosHashPut(schStatus->taskStatus, id, sizeof(id), &tStatus, sizeof(tStatus))) { + qError("taosHashPut schedulerId[%"PRIx64"]queryId[%"PRIx64"] taskId[%"PRIx64"] to scheduleHash failed", schedulerId, queryId, taskId); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t schedulerId, uint64_t queryId, uint64_t taskId, SQWorkerTaskStatus **taskStatus) { + SQWorkerSchTaskStatus *schStatus = taosHashGet(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId)); + if (NULL == schStatus) { + qError("no scheduler for schedulerId[%"PRIx64"]", schedulerId); + return TSDB_CODE_QRY_APP_ERROR; + } + + schStatus->lastAccessTs = taosGetTimestampSec(); + + char id[sizeof(queryId) + sizeof(taskId)] = {0}; + QW_SET_QTID(id, queryId, taskId); + + SQWorkerTaskStatus *tStatus = taosHashGet(schStatus->taskStatus, id, sizeof(id)); + if (NULL == tStatus) { + qError("no task status for schedulerId[%"PRIx64"] queryId[%"PRIx64"] taskId[%"PRIx64"]", schedulerId, queryId, taskId); + return TSDB_CODE_QRY_APP_ERROR; + } + + *taskStatus = tStatus; + + return TSDB_CODE_SUCCESS; +} + +int32_t qwAddTaskResult(SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, void *data) { + char id[sizeof(queryId) + sizeof(taskId)] = {0}; + QW_SET_QTID(id, queryId, taskId); + + SQWorkerResCache resCache = {0}; + resCache.data = data; + + if (0 != taosHashPut(mgmt->resHash, id, sizeof(id), &resCache, sizeof(SQWorkerResCache))) { + qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to resHash failed", queryId, taskId); + return TSDB_CODE_QRY_APP_ERROR; + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t qwGetTaskResult(SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, void **data) { + char id[sizeof(queryId) + sizeof(taskId)] = {0}; + QW_SET_QTID(id, queryId, taskId); + + SQWorkerResCache *resCache = taosHashGet(mgmt->resHash, id, sizeof(id)); + if (NULL == resCache) { + qError("no task res for queryId[%"PRIx64"] taskId[%"PRIx64"]", queryId, taskId); + return TSDB_CODE_QRY_APP_ERROR; + } + + *data = resCache->data; + + return TSDB_CODE_SUCCESS; +} + +int32_t qwUpdateSchLastAccess(SQWorkerMgmt *mgmt, uint64_t schedulerId) { + SQWorkerSchTaskStatus *schStatus = taosHashGet(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId)); + if (NULL == schStatus) { + qError("no scheduler for schedulerId[%"PRIx64"]", schedulerId); + return TSDB_CODE_QRY_APP_ERROR; + } + + schStatus->lastAccessTs = taosGetTimestampSec(); + + return TSDB_CODE_SUCCESS; +} + +int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t schedulerId, SSchedulerStatusRsp **rsp) { + SQWorkerSchTaskStatus *schStatus = taosHashGet(mgmt->scheduleHash, &schedulerId, sizeof(schedulerId)); + if (NULL == schStatus) { + qError("no scheduler for schedulerId[%"PRIx64"]", schedulerId); + return TSDB_CODE_QRY_APP_ERROR; + } + + schStatus->lastAccessTs = taosGetTimestampSec(); + + int32_t i = 0; + int32_t taskNum = taosHashGetSize(schStatus->taskStatus); + int32_t size = sizeof(SSchedulerStatusRsp) + sizeof((*rsp)->status[0]) * taskNum; + *rsp = calloc(1, size); + if (NULL == *rsp) { + qError("calloc %d failed", size); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + void *key = NULL; + size_t keyLen = 0; + void *pIter = taosHashIterate(schStatus->taskStatus, NULL); + while (pIter) { + SQWorkerTaskStatus *taskStatus = (SQWorkerTaskStatus *)pIter; + taosHashGetKey(pIter, &key, &keyLen); + + QW_GET_QTID(key, (*rsp)->status[i].queryId, (*rsp)->status[i].taskId); + (*rsp)->status[i].status = taskStatus->status; + + pIter = taosHashIterate(schStatus->taskStatus, pIter); + } + + (*rsp)->num = taskNum; + + return TSDB_CODE_SUCCESS; +} + +int32_t qwBuildRspMsg(void *data, int32_t msgType); + + +int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) { + SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt)); + if (NULL == mgmt) { + qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt)); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + if (cfg) { + mgmt->cfg = *cfg; + } else { + mgmt->cfg.maxSchedulerNum = QWORKER_DEFAULT_SCHEDULER_NUMBER; + mgmt->cfg.maxResCacheNum = QWORKER_DEFAULT_RES_CACHE_NUMBER; + mgmt->cfg.maxSchTaskNum = QWORKER_DEFAULT_SCH_TASK_NUMBER; + } + + mgmt->scheduleHash = taosHashInit(mgmt->cfg.maxSchedulerNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == mgmt->scheduleHash) { + tfree(mgmt); + QW_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler hash failed", mgmt->cfg.maxSchedulerNum); + } + + mgmt->resHash = taosHashInit(mgmt->cfg.maxResCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == mgmt->resHash) { + taosHashCleanup(mgmt->scheduleHash); + mgmt->scheduleHash = NULL; + tfree(mgmt); + + QW_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d res cache hash failed", mgmt->cfg.maxResCacheNum); + } + + *qWorkerMgmt = mgmt; + + return TSDB_CODE_SUCCESS; +} + +int32_t qWorkerProcessQueryMsg(void *qWorkerMgmt, SSchedulerQueryMsg *msg, SRpcMsg *rsp) { + if (NULL == qWorkerMgmt || NULL == msg || NULL == rsp) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SSubplan *plan = NULL; + SQWorkerTaskStatus *tStatus = NULL; + + int32_t code = qStringToSubplan(msg->msg, &plan); + if (TSDB_CODE_SUCCESS != code) { + qError("schId:%"PRIx64",qId:%"PRIx64",taskId:%"PRIx64" string to subplan failed, code:%d", msg->schedulerId, msg->queryId, msg->taskId, code); + return code; + } + + //TODO call executer to init subquery + + QW_ERR_JRET(qwAddTaskStatus(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, JOB_TASK_STATUS_EXECUTING)); + + QW_ERR_JRET(qwBuildRspMsg(NULL, TSDB_MSG_TYPE_QUERY_RSP)); + + //TODO call executer to execute subquery + code = 0; + void *data = NULL; + //TODO call executer to execute subquery + + QW_ERR_JRET(qwGetTaskStatus(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, &tStatus)); + + tStatus->status = (code) ? JOB_TASK_STATUS_FAILED : JOB_TASK_STATUS_SUCCEED; + + QW_ERR_JRET(qwAddTaskResult(qWorkerMgmt, msg->queryId, msg->taskId, data)); + +_return: + + if (tStatus && QW_TASK_DONE(tStatus->status) && QW_READY_RECEIVED == tStatus->ready) { + QW_ERR_RET(qwBuildRspMsg(NULL, TSDB_MSG_TYPE_RES_READY_RSP)); + } + + qDestroySubplan(plan); + + return code; +} + +int32_t qWorkerProcessReadyMsg(void *qWorkerMgmt, SSchedulerReadyMsg *msg, SRpcMsg *rsp){ + if (NULL == qWorkerMgmt || NULL == msg || NULL == rsp) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SQWorkerTaskStatus *tStatus = NULL; + + QW_ERR_RET(qwGetTaskStatus(qWorkerMgmt, msg->schedulerId, msg->queryId, msg->taskId, &tStatus)); + + if (QW_TASK_DONE(tStatus->status)) { + QW_ERR_RET(qwBuildRspMsg(tStatus, TSDB_MSG_TYPE_RES_READY_RSP)); + } else { + tStatus->ready = QW_READY_RECEIVED; + + return TSDB_CODE_SUCCESS; + } + + tStatus->ready = QW_READY_RESPONSED; + + return TSDB_CODE_SUCCESS; +} + +int32_t qWorkerProcessStatusMsg(void *qWorkerMgmt, SSchedulerStatusMsg *msg, SRpcMsg *rsp) { + if (NULL == qWorkerMgmt || NULL == msg || NULL == rsp) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SSchedulerStatusRsp *sStatus = NULL; + + QW_ERR_RET(qwGetSchTasksStatus(qWorkerMgmt, msg->schedulerId, &sStatus)); + + return TSDB_CODE_SUCCESS; +} + +int32_t qWorkerProcessFetchMsg(void *qWorkerMgmt, SSchedulerFetchMsg *msg, SRpcMsg *rsp) { + if (NULL == qWorkerMgmt || NULL == msg || NULL == rsp) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + QW_ERR_RET(qwUpdateSchLastAccess(qWorkerMgmt, msg->schedulerId)); + + void *data = NULL; + + QW_ERR_RET(qwGetTaskResult(qWorkerMgmt, msg->queryId, msg->taskId, &data)); + + QW_ERR_RET(qwBuildRspMsg(data, TSDB_MSG_TYPE_FETCH_RSP)); + + return TSDB_CODE_SUCCESS; +} + +int32_t qWorkerProcessCancelMsg(void *qWorkerMgmt, SSchedulerCancelMsg *msg, SRpcMsg *rsp); + +void qWorkerDestroy(void **qWorkerMgmt) { + if (NULL == qWorkerMgmt || NULL == *qWorkerMgmt) { + return; + } + + SQWorkerMgmt *mgmt = *qWorkerMgmt; + + //TODO STOP ALL QUERY + + //TODO FREE ALL + + tfree(*qWorkerMgmt); +} + + diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 73e7c4d24e1c2693d174dcb7acad4112939778c3..3fab91edacd77096fbacceee5df3f762452ad16d 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -31,17 +31,9 @@ extern "C" { #define SCH_MAX_CONDIDATE_EP_NUM TSDB_MAX_REPLICA -enum { - SCH_STATUS_NOT_START = 1, - SCH_STATUS_EXECUTING, - SCH_STATUS_SUCCEED, - SCH_STATUS_FAILED, - SCH_STATUS_CANCELLING, - SCH_STATUS_CANCELLED -}; - typedef struct SSchedulerMgmt { - uint64_t taskId; + uint64_t taskId; + uint64_t schedulerId; SSchedulerCfg cfg; SHashObj *Jobs; // key: queryId, value: SQueryJob* } SSchedulerMgmt; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 6014ff9ab6282248f52bfe1aba0535e5b4d4cb71..99a9b06fe441832d42fbc5bc86ad24bcc3ebc263 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -161,7 +161,7 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { SArray *levelPlans = NULL; int32_t levelPlanNum = 0; - level.status = SCH_STATUS_NOT_START; + level.status = JOB_TASK_STATUS_NOT_START; for (int32_t i = 0; i < levelNum; ++i) { level.level = i; @@ -191,7 +191,7 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { task.taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); task.plan = plan; - task.status = SCH_STATUS_NOT_START; + task.status = JOB_TASK_STATUS_NOT_START; void *p = taosArrayPush(level.subTasks, &task); if (NULL == p) { @@ -304,13 +304,15 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { } SSchedulerQueryMsg *pMsg = msg; - pMsg->queryId = job->queryId; - pMsg->taskId = task->taskId; - pMsg->contentLen = len; + + pMsg->schedulerId = htobe64(schMgmt.schedulerId); + pMsg->queryId = htobe64(job->queryId); + pMsg->taskId = htobe64(task->taskId); + pMsg->contentLen = htonl(len); memcpy(pMsg->msg, task->msg, len); break; } - case TSDB_MSG_TYPE_RSP_READY: { + case TSDB_MSG_TYPE_RES_READY: { msgSize = sizeof(SSchedulerReadyMsg); msg = calloc(1, msgSize); if (NULL == msg) { @@ -319,8 +321,8 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { } SSchedulerReadyMsg *pMsg = msg; - pMsg->queryId = job->queryId; - pMsg->taskId = task->taskId; + pMsg->queryId = htobe64(job->queryId); + pMsg->taskId = htobe64(task->taskId); break; } case TSDB_MSG_TYPE_FETCH: { @@ -332,8 +334,8 @@ int32_t schAsyncSendMsg(SQueryJob *job, SQueryTask *task, int32_t msgType) { } SSchedulerFetchMsg *pMsg = msg; - pMsg->queryId = job->queryId; - pMsg->taskId = task->taskId; + pMsg->queryId = htobe64(job->queryId); + pMsg->taskId = htobe64(task->taskId); break; } default: @@ -376,7 +378,7 @@ _return: int32_t schProcessOnJobSuccess(SQueryJob *job) { - job->status = SCH_STATUS_SUCCEED; + job->status = JOB_TASK_STATUS_SUCCEED; if (job->userFetch) { SCH_ERR_RET(schFetchFromRemote(job)); @@ -386,7 +388,7 @@ int32_t schProcessOnJobSuccess(SQueryJob *job) { } int32_t schProcessOnJobFailure(SQueryJob *job) { - job->status = SCH_STATUS_FAILED; + job->status = JOB_TASK_STATUS_FAILED; atomic_val_compare_exchange_32(&job->remoteFetch, 1, 0); @@ -413,7 +415,7 @@ int32_t schProcessOnTaskSuccess(SQueryJob *job, SQueryTask *task) { return TSDB_CODE_SUCCESS; } - task->status = SCH_STATUS_SUCCEED; + task->status = JOB_TASK_STATUS_SUCCEED; int32_t parentNum = (int32_t)taosArrayGetSize(task->parents); if (parentNum == 0) { @@ -459,7 +461,7 @@ int32_t schProcessOnTaskFailure(SQueryJob *job, SQueryTask *task, int32_t errCod if (!needRetry) { SCH_TASK_ERR_LOG("task failed[%x], no more retry", errCode); - job->status = SCH_STATUS_FAILED; + job->status = JOB_TASK_STATUS_FAILED; SCH_ERR_RET(schProcessOnJobFailure(job)); return TSDB_CODE_SUCCESS; @@ -478,13 +480,13 @@ int32_t schHandleRspMsg(SQueryJob *job, SQueryTask *task, int32_t msgType, int32 if (rspCode != TSDB_CODE_SUCCESS) { SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); } else { - code = schAsyncSendMsg(job, task, TSDB_MSG_TYPE_RSP_READY); + code = schAsyncSendMsg(job, task, TSDB_MSG_TYPE_RES_READY); if (code) { goto _task_error; } } break; - case TSDB_MSG_TYPE_RSP_READY: + case TSDB_MSG_TYPE_RES_READY: if (rspCode != TSDB_CODE_SUCCESS) { SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); } else { @@ -534,7 +536,7 @@ int32_t schLaunchTask(SQueryJob *job, SQueryTask *task) { SCH_ERR_RET(schPushTaskToExecList(job, task)); - task->status = SCH_STATUS_EXECUTING; + task->status = JOB_TASK_STATUS_EXECUTING; return TSDB_CODE_SUCCESS; } @@ -546,22 +548,26 @@ int32_t schLaunchJob(SQueryJob *job) { SCH_ERR_RET(schLaunchTask(job, task)); } - job->status = SCH_STATUS_EXECUTING; + job->status = JOB_TASK_STATUS_EXECUTING; return TSDB_CODE_SUCCESS; } int32_t schedulerInit(SSchedulerCfg *cfg) { - schMgmt.Jobs = taosHashInit(SCHEDULE_DEFAULT_JOB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); - if (NULL == schMgmt.Jobs) { - SCH_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler jobs failed", SCHEDULE_DEFAULT_JOB_NUMBER); - } - if (cfg) { schMgmt.cfg = *cfg; + } else { + schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_JOB_NUMBER; } + schMgmt.Jobs = taosHashInit(schMgmt.cfg.maxJobNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == schMgmt.Jobs) { + SCH_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler jobs failed", schMgmt.cfg.maxJobNum); + } + + schMgmt.schedulerId = 1; //TODO GENERATE A UUID + return TSDB_CODE_SUCCESS; } @@ -605,7 +611,7 @@ int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, voi SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } - job->status = SCH_STATUS_NOT_START; + job->status = JOB_TASK_STATUS_NOT_START; SCH_ERR_JRET(schLaunchJob(job)); @@ -634,7 +640,7 @@ int32_t scheduleFetchRows(void *pJob, void **data) { return TSDB_CODE_QRY_APP_ERROR; } - if (job->status == SCH_STATUS_SUCCEED) { + if (job->status == JOB_TASK_STATUS_SUCCEED) { SCH_ERR_JRET(schFetchFromRemote(job)); } @@ -668,7 +674,7 @@ void scheduleFreeJob(void *pJob) { return; } - if (job->status == SCH_STATUS_EXECUTING) { + if (job->status == JOB_TASK_STATUS_EXECUTING) { scheduleCancelJob(pJob); } } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ffc59c5d6975833c25d5d53f1b6a391be0a07090..43ac7606432b6380fbac9b683ef8737e1903c8f3 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -240,6 +240,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment" TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") +// mnode-trans +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") + // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_DND_EXITING, "Dnode is exiting") diff --git a/source/util/src/thash.c b/source/util/src/thash.c index bfc6b47fa6abd80245b021954ee908156e9eebeb..c4f6f78106e1ee6703e0429e705f62ab83db1881 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -776,9 +776,16 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) { return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); } -FORCE_INLINE void *taosHashGetDataKey(SHashObj *pHashObj, void *data) { +FORCE_INLINE int32_t taosHashGetKey(void *data, void** key, size_t* keyLen) { + if (NULL == data || NULL == key) { + return -1; + } + SHashNode * node = GET_HASH_PNODE(data); - return GET_HASH_NODE_KEY(node); + *key = GET_HASH_NODE_KEY(node); + *keyLen = node->keyLen; + + return 0; } FORCE_INLINE uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data) { diff --git a/source/util/src/tthread.c b/source/util/src/tthread.c index 5ed7fb5aa00d1a5c5c3fffae2de46dbea57e011f..44fce1c882c4f0d7d66f36d378db1594e59f19ea 100644 --- a/source/util/src/tthread.c +++ b/source/util/src/tthread.c @@ -13,16 +13,16 @@ * along with this program. If not, see . */ -#include "os.h" #include "tthread.h" +#include "os.h" +#include "taoserror.h" #include "tdef.h" #include "tutil.h" #include "ulog.h" -#include "taoserror.h" // create new thread -pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param) { - pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t)); +pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param) { + pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t)); pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -36,26 +36,24 @@ pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param) { return pthread; } -// destory thread +// destory thread bool taosDestoryThread(pthread_t* pthread) { - if(pthread == NULL) return false; - if(taosThreadRunning(pthread)) { + if (pthread == NULL) return false; + if (taosThreadRunning(pthread)) { pthread_cancel(*pthread); pthread_join(*pthread, NULL); } - + free(pthread); return true; } // thread running return true bool taosThreadRunning(pthread_t* pthread) { - if(pthread == NULL) return false; + if (pthread == NULL) return false; int ret = pthread_kill(*pthread, 0); - if(ret == ESRCH) - return false; - if(ret == EINVAL) - return false; + if (ret == ESRCH) return false; + if (ret == EINVAL) return false; // alive return true; } diff --git a/src/tsdb/CMakeLists.txt b/src/tsdb/CMakeLists.txt deleted file mode 100644 index c5b77df5a25f9f0b1e9294228520f171b9befddd..0000000000000000000000000000000000000000 --- a/src/tsdb/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(inc) -AUX_SOURCE_DIRECTORY(src SRC) -ADD_LIBRARY(tsdb ${SRC}) -TARGET_LINK_LIBRARIES(tsdb tfs common tutil) - -IF (TD_TSDB_PLUGINS) - TARGET_LINK_LIBRARIES(tsdb tsdbPlugins) -ENDIF () - -IF (TD_LINUX) - # Someone has no gtest directory, so comment it - # ADD_SUBDIRECTORY(tests) -ENDIF () diff --git a/src/tsdb/inc/tsdbBuffer.h b/src/tsdb/inc/tsdbBuffer.h deleted file mode 100644 index 4b650d3993a54f6a98caf00a3605feb37e972ebd..0000000000000000000000000000000000000000 --- a/src/tsdb/inc/tsdbBuffer.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_BUFFER_H_ -#define _TD_TSDB_BUFFER_H_ - -typedef struct { - int64_t blockId; - int offset; - int remain; - char data[]; -} STsdbBufBlock; - -typedef struct { - pthread_cond_t poolNotEmpty; - int bufBlockSize; - int tBufBlocks; - int nBufBlocks; - int nRecycleBlocks; - int nElasticBlocks; - int64_t index; - SList* bufBlockList; -} STsdbBufPool; - -#define TSDB_BUFFER_RESERVE 1024 // Reseve 1K as commit threshold - -STsdbBufPool* tsdbNewBufPool(); -void tsdbFreeBufPool(STsdbBufPool* pBufPool); -int tsdbOpenBufPool(STsdbRepo* pRepo); -void tsdbCloseBufPool(STsdbRepo* pRepo); -SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); -int tsdbExpandPool(STsdbRepo* pRepo, int32_t oldTotalBlocks); -void tsdbRecycleBufferBlock(STsdbBufPool* pPool, SListNode *pNode, bool bELastic); - -// health cite -STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize); -void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock); - -#endif /* _TD_TSDB_BUFFER_H_ */ diff --git a/src/tsdb/inc/tsdbLog.h b/src/tsdb/inc/tsdbLog.h deleted file mode 100644 index fdd04e968a2ce5345030aa4cef2bb87becd8f3f9..0000000000000000000000000000000000000000 --- a/src/tsdb/inc/tsdbLog.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_LOG_H_ -#define _TD_TSDB_LOG_H_ - -extern int32_t tsdbDebugFlag; - -#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) -#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) -#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) -#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) -#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) -#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) - -#endif /* _TD_TSDB_LOG_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbMeta.h b/src/tsdb/inc/tsdbMeta.h deleted file mode 100644 index 8ce5e7ade80b2006ac8c39fec178994073c5a26d..0000000000000000000000000000000000000000 --- a/src/tsdb/inc/tsdbMeta.h +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_META_H_ -#define _TD_TSDB_META_H_ - -#define TSDB_MAX_TABLE_SCHEMAS 16 - -typedef struct STable { - STableId tableId; - ETableType type; - tstr* name; // NOTE: there a flexible string here - uint64_t suid; - struct STable* pSuper; // super table pointer - SArray* schema; - STSchema* tagSchema; - SKVRow tagVal; - SSkipList* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index - void* eventHandler; // TODO - void* streamHandler; // TODO - TSKEY lastKey; - SMemRow lastRow; - char* sql; - void* cqhandle; - SRWLatch latch; // TODO: implementa latch functions - - SDataCol *lastCols; - int16_t maxColNum; - int16_t restoreColumnNum; - bool hasRestoreLastColumn; - int lastColSVersion; - T_REF_DECLARE() -} STable; - -typedef struct { - pthread_rwlock_t rwLock; - - int32_t nTables; - int32_t maxTables; - STable** tables; - SList* superList; - SHashObj* uidMap; - int maxRowBytes; - int maxCols; -} STsdbMeta; - -#define TSDB_INIT_NTABLES 1024 -#define TABLE_TYPE(t) (t)->type -#define TABLE_NAME(t) (t)->name -#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data -#define TABLE_UID(t) (t)->tableId.uid -#define TABLE_TID(t) (t)->tableId.tid -#define TABLE_SUID(t) (t)->suid -// #define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore) -#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch)) -#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch)) -#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch)) -#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch)) - -STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg); -void tsdbFreeMeta(STsdbMeta* pMeta); -int tsdbOpenMeta(STsdbRepo* pRepo); -int tsdbCloseMeta(STsdbRepo* pRepo); -STable* tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid); -STSchema* tsdbGetTableSchemaByVersion(STable* pTable, int16_t _version); -int tsdbWLockRepoMeta(STsdbRepo* pRepo); -int tsdbRLockRepoMeta(STsdbRepo* pRepo); -int tsdbUnlockRepoMeta(STsdbRepo* pRepo); -void tsdbRefTable(STable* pTable); -void tsdbUnRefTable(STable* pTable); -void tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct); -int tsdbRestoreTable(STsdbRepo* pRepo, void* cont, int contLen); -void tsdbOrgMeta(STsdbRepo* pRepo); -int tsdbInitColIdCacheWithSchema(STable* pTable, STSchema* pSchema); -int16_t tsdbGetLastColumnsIndexByColId(STable* pTable, int16_t colId); -int tsdbUpdateLastColSchema(STable *pTable, STSchema *pNewSchema); -STSchema* tsdbGetTableLatestSchema(STable *pTable); -void tsdbFreeLastColumns(STable* pTable); - -static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) { - if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) { - return -1; - } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) { - return 1; - } else { - return 0; - } -} - -static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t _version) { - STable* pDTable = (pTable->pSuper != NULL) ? pTable->pSuper : pTable; // for performance purpose - STSchema* pSchema = NULL; - STSchema* pTSchema = NULL; - - if (lock) TSDB_RLOCK_TABLE(pDTable); - if (_version < 0) { // get the latest version of schema - pTSchema = *(STSchema **)taosArrayGetLast(pDTable->schema); - } else { // get the schema with version - void* ptr = taosArraySearch(pDTable->schema, &_version, tsdbCompareSchemaVersion, TD_EQ); - if (ptr == NULL) { - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - goto _exit; - } - pTSchema = *(STSchema**)ptr; - } - - ASSERT(pTSchema != NULL); - - if (copy) { - if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - } else { - pSchema = pTSchema; - } - -_exit: - if (lock) TSDB_RUNLOCK_TABLE(pDTable); - return pSchema; -} - -static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) { - return tsdbGetTableSchemaImpl(pTable, false, false, -1); -} - -static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) { - if (pTable->type == TSDB_CHILD_TABLE) { // check child table first - STable *pSuper = pTable->pSuper; - if (pSuper == NULL) return NULL; - return pSuper->tagSchema; - } else if (pTable->type == TSDB_SUPER_TABLE) { - return pTable->tagSchema; - } else { - return NULL; - } -} - -static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) { - ASSERT((pTable->lastRow == NULL) || (pTable->lastKey == memRowKey(pTable->lastRow))); - return pTable->lastKey; -} - -#endif /* _TD_TSDB_META_H_ */ \ No newline at end of file diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c deleted file mode 100644 index 03110487807076bf8ac2ac7026ffdb828ea4c7c6..0000000000000000000000000000000000000000 --- a/src/tsdb/src/tsdbCommit.c +++ /dev/null @@ -1,1666 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "tsdbint.h" - -extern int32_t tsTsdbMetaCompactRatio; - -#define TSDB_MAX_SUBBLOCKS 8 -static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { - if (key < 0) { - return (int)((key + 1) / tsTickPerDay[precision] / days - 1); - } else { - return (int)((key / tsTickPerDay[precision] / days)); - } -} - -typedef struct { - SRtn rtn; // retention snapshot - SFSIter fsIter; // tsdb file iterator - int niters; // memory iterators - SCommitIter *iters; - bool isRFileSet; // read and commit FSET - SReadH readh; - SDFileSet wSet; - bool isDFileSame; - bool isLFileSame; - TSKEY minKey; - TSKEY maxKey; - SArray * aBlkIdx; // SBlockIdx array - STable * pTable; - SArray * aSupBlk; // Table super-block array - SArray * aSubBlk; // table sub-block array - SDataCols * pDataCols; -} SCommitH; - -#define TSDB_COMMIT_REPO(ch) TSDB_READ_REPO(&(ch->readh)) -#define TSDB_COMMIT_REPO_ID(ch) REPO_ID(TSDB_READ_REPO(&(ch->readh))) -#define TSDB_COMMIT_WRITE_FSET(ch) (&((ch)->wSet)) -#define TSDB_COMMIT_TABLE(ch) ((ch)->pTable) -#define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD) -#define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA) -#define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST) -#define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh)) -#define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh)) -#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock) -#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) - -static int tsdbCommitMeta(STsdbRepo *pRepo); -static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact); -static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); -static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile); -static int tsdbCommitTSData(STsdbRepo *pRepo); -static void tsdbStartCommit(STsdbRepo *pRepo); -static void tsdbEndCommit(STsdbRepo *pRepo, int eno); -static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); -static int tsdbCreateCommitIters(SCommitH *pCommith); -static void tsdbDestroyCommitIters(SCommitH *pCommith); -static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key); -static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo); -static void tsdbDestroyCommitH(SCommitH *pCommith); -static int tsdbGetFidLevel(int fid, SRtn *pRtn); -static int tsdbNextCommitFid(SCommitH *pCommith); -static int tsdbCommitToTable(SCommitH *pCommith, int tid); -static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable); -static int tsdbComparKeyBlock(const void *arg1, const void *arg2); -static int tsdbWriteBlockInfo(SCommitH *pCommih); -static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData); -static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx); -static int tsdbMoveBlock(SCommitH *pCommith, int bidx); -static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks); -static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, - bool isLastOneBlock); -static void tsdbResetCommitFile(SCommitH *pCommith); -static void tsdbResetCommitTable(SCommitH *pCommith); -static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); -static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError); -static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo); -static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows, int8_t update); - -void *tsdbCommitData(STsdbRepo *pRepo) { - if (pRepo->imem == NULL) { - return NULL; - } - tsdbStartCommit(pRepo); - - // Commit to update meta file - if (tsdbCommitMeta(pRepo) < 0) { - tsdbError("vgId:%d error occurs while committing META data since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - // Create the iterator to read from cache - if (tsdbCommitTSData(pRepo) < 0) { - tsdbError("vgId:%d error occurs while committing TS data since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS); - return NULL; - -_err: - ASSERT(terrno != TSDB_CODE_SUCCESS); - pRepo->code = terrno; - - tsdbEndCommit(pRepo, terrno); - return NULL; -} - -int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) { - SDiskID did; - SDFileSet nSet; - STsdbFS * pfs = REPO_FS(pRepo); - int level; - - ASSERT(pSet->fid >= pRtn->minFid); - - level = tsdbGetFidLevel(pSet->fid, pRtn); - - tfsAllocDisk(level, &(did.level), &(did.id)); - if (did.level == TFS_UNDECIDED_LEVEL) { - terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; - return -1; - } - - if (did.level > TSDB_FSET_LEVEL(pSet)) { - // Need to move the FSET to higher level - tsdbInitDFileSet(&nSet, did, REPO_ID(pRepo), pSet->fid, FS_TXN_VERSION(pfs)); - - if (tsdbCopyDFileSet(pSet, &nSet) < 0) { - tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno)); - return -1; - } - - if (tsdbUpdateDFileSet(pfs, &nSet) < 0) { - return -1; - } - - tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id); - } else { - // On a correct level - if (tsdbUpdateDFileSet(pfs, pSet) < 0) { - return -1; - } - } - - return 0; -} - -int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray *pSubA, void **ppBuf, - SBlockIdx *pIdx) { - size_t nSupBlocks; - size_t nSubBlocks; - uint32_t tlen; - SBlockInfo *pBlkInfo; - int64_t offset; - SBlock * pBlock; - - memset(pIdx, 0, sizeof(*pIdx)); - - nSupBlocks = taosArrayGetSize(pSupA); - nSubBlocks = (pSubA == NULL) ? 0 : taosArrayGetSize(pSubA); - - if (nSupBlocks <= 0) { - // No data (data all deleted) - return 0; - } - - tlen = (uint32_t)(sizeof(SBlockInfo) + sizeof(SBlock) * (nSupBlocks + nSubBlocks) + sizeof(TSCKSUM)); - if (tsdbMakeRoom(ppBuf, tlen) < 0) return -1; - pBlkInfo = *ppBuf; - - pBlkInfo->delimiter = TSDB_FILE_DELIMITER; - pBlkInfo->tid = TABLE_TID(pTable); - pBlkInfo->uid = TABLE_UID(pTable); - - memcpy((void *)(pBlkInfo->blocks), taosArrayGet(pSupA, 0), nSupBlocks * sizeof(SBlock)); - if (nSubBlocks > 0) { - memcpy((void *)(pBlkInfo->blocks + nSupBlocks), taosArrayGet(pSubA, 0), nSubBlocks * sizeof(SBlock)); - - for (int i = 0; i < nSupBlocks; i++) { - pBlock = pBlkInfo->blocks + i; - - if (pBlock->numOfSubBlocks > 1) { - pBlock->offset += (sizeof(SBlockInfo) + sizeof(SBlock) * nSupBlocks); - } - } - } - - taosCalcChecksumAppend(0, (uint8_t *)pBlkInfo, tlen); - - if (tsdbAppendDFile(pHeadf, (void *)pBlkInfo, tlen, &offset) < 0) { - return -1; - } - - tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(pBlkInfo, tlen - sizeof(TSCKSUM))); - - // Set pIdx - pBlock = taosArrayGetLast(pSupA); - - pIdx->tid = TABLE_TID(pTable); - pIdx->uid = TABLE_UID(pTable); - pIdx->hasLast = pBlock->last ? 1 : 0; - pIdx->maxKey = pBlock->keyLast; - pIdx->numOfBlocks = (uint32_t)nSupBlocks; - pIdx->len = tlen; - pIdx->offset = (uint32_t)offset; - - return 0; -} - -int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { - SBlockIdx *pBlkIdx; - size_t nidx = taosArrayGetSize(pIdxA); - int tlen = 0, size; - int64_t offset; - - if (nidx <= 0) { - // All data are deleted - pHeadf->info.offset = 0; - pHeadf->info.len = 0; - return 0; - } - - for (size_t i = 0; i < nidx; i++) { - pBlkIdx = (SBlockIdx *)taosArrayGet(pIdxA, i); - - size = tsdbEncodeSBlockIdx(NULL, pBlkIdx); - if (tsdbMakeRoom(ppBuf, tlen + size) < 0) return -1; - - void *ptr = POINTER_SHIFT(*ppBuf, tlen); - tsdbEncodeSBlockIdx(&ptr, pBlkIdx); - - tlen += size; - } - - tlen += sizeof(TSCKSUM); - if (tsdbMakeRoom(ppBuf, tlen) < 0) return -1; - taosCalcChecksumAppend(0, (uint8_t *)(*ppBuf), tlen); - - if (tsdbAppendDFile(pHeadf, *ppBuf, tlen, &offset) < tlen) { - return -1; - } - - tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(*ppBuf, tlen - sizeof(TSCKSUM))); - pHeadf->info.offset = (uint32_t)offset; - pHeadf->info.len = tlen; - - return 0; -} - - -// =================== Commit Meta Data -static int tsdbInitCommitMetaFile(STsdbRepo *pRepo, SMFile* pMf, bool open) { - STsdbFS * pfs = REPO_FS(pRepo); - SMFile * pOMFile = pfs->cstatus->pmf; - SDiskID did; - - // Create/Open a meta file or open the existing file - if (pOMFile == NULL) { - // Create a new meta file - did.level = TFS_PRIMARY_LEVEL; - did.id = TFS_PRIMARY_ID; - tsdbInitMFile(pMf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); - - if (open && tsdbCreateMFile(pMf, true) < 0) { - tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMf)); - } else { - tsdbInitMFileEx(pMf, pOMFile); - if (open && tsdbOpenMFile(pMf, O_WRONLY) < 0) { - tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - } - - return 0; -} - -static int tsdbCommitMeta(STsdbRepo *pRepo) { - STsdbFS * pfs = REPO_FS(pRepo); - SMemTable *pMem = pRepo->imem; - SMFile * pOMFile = pfs->cstatus->pmf; - SMFile mf; - SActObj * pAct = NULL; - SActCont * pCont = NULL; - SListNode *pNode = NULL; - - ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0); - - if (listNEles(pMem->actList) <= 0) { - // no meta data to commit, just keep the old meta file - tsdbUpdateMFile(pfs, pOMFile); - if (tsTsdbMetaCompactRatio > 0) { - if (tsdbInitCommitMetaFile(pRepo, &mf, false) < 0) { - return -1; - } - int ret = tsdbCompactMetaFile(pRepo, pfs, &mf); - if (ret < 0) tsdbError("compact meta file error"); - - return ret; - } - return 0; - } else { - if (tsdbInitCommitMetaFile(pRepo, &mf, true) < 0) { - return -1; - } - } - - // Loop to write - while ((pNode = tdListPopHead(pMem->actList)) != NULL) { - pAct = (SActObj *)pNode->data; - if (pAct->act == TSDB_UPDATE_META) { - pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); - if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len, false) < 0) { - tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, - tstrerror(terrno)); - tsdbCloseMFile(&mf); - (void)tsdbApplyMFileChange(&mf, pOMFile); - // TODO: need to reload metaCache - return -1; - } - } else if (pAct->act == TSDB_DROP_META) { - if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) { - tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, - tstrerror(terrno)); - tsdbCloseMFile(&mf); - tsdbApplyMFileChange(&mf, pOMFile); - // TODO: need to reload metaCache - return -1; - } - } else { - ASSERT(false); - } - } - - if (tsdbUpdateMFileHeader(&mf) < 0) { - tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno)); - tsdbApplyMFileChange(&mf, pOMFile); - // TODO: need to reload metaCache - return -1; - } - - TSDB_FILE_FSYNC(&mf); - tsdbCloseMFile(&mf); - tsdbUpdateMFile(pfs, &mf); - - if (tsTsdbMetaCompactRatio > 0 && tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) { - tsdbError("compact meta file error"); - } - - return 0; -} - -int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord) { - int tlen = 0; - tlen += taosEncodeFixedU64(buf, pRecord->uid); - tlen += taosEncodeFixedI64(buf, pRecord->offset); - tlen += taosEncodeFixedI64(buf, pRecord->size); - - return tlen; -} - -void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord) { - buf = taosDecodeFixedU64(buf, &(pRecord->uid)); - buf = taosDecodeFixedI64(buf, &(pRecord->offset)); - buf = taosDecodeFixedI64(buf, &(pRecord->size)); - - return buf; -} - -void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) { - STsdbCfg *pCfg = REPO_CFG(pRepo); - TSKEY minKey, midKey, maxKey, now; - - now = taosGetTimestamp(pCfg->precision); - minKey = now - pCfg->keep * tsTickPerDay[pCfg->precision]; - midKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision]; - maxKey = now - pCfg->keep1 * tsTickPerDay[pCfg->precision]; - - pRtn->minKey = minKey; - pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision)); - pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision)); - pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision)); - tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey, - pRtn->minFid, pRtn->midFid, pRtn->maxFid); -} - -static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact) { - char buf[64] = "\0"; - void * pBuf = buf; - SKVRecord rInfo; - int64_t offset; - - // Seek to end of meta file - offset = tsdbSeekMFile(pMFile, 0, SEEK_END); - if (offset < 0) { - return -1; - } - - rInfo.offset = offset; - rInfo.uid = uid; - rInfo.size = contLen; - - int tlen = tsdbEncodeKVRecord((void **)(&pBuf), &rInfo); - if (tsdbAppendMFile(pMFile, buf, tlen, NULL) < tlen) { - return -1; - } - - if (tsdbAppendMFile(pMFile, cont, contLen, NULL) < contLen) { - return -1; - } - - tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM))); - - SHashObj* cache = compact ? pfs->metaCacheComp : pfs->metaCache; - - pMFile->info.nRecords++; - - SKVRecord *pRecord = taosHashGet(cache, (void *)&uid, sizeof(uid)); - if (pRecord != NULL) { - pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord)); - } else { - pMFile->info.nRecords++; - } - taosHashPut(cache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); - - return 0; -} - -static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { - SKVRecord rInfo = {0}; - char buf[128] = "\0"; - - SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid)); - if (pRecord == NULL) { - tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid); - return -1; - } - - rInfo.offset = -pRecord->offset; - rInfo.uid = pRecord->uid; - rInfo.size = pRecord->size; - - void *pBuf = buf; - tsdbEncodeKVRecord(&pBuf, &rInfo); - - if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) { - return -1; - } - - pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord)); - pMFile->info.nDels++; - pMFile->info.nRecords--; - pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); - - taosHashRemove(pfs->metaCache, (void *)(&uid), sizeof(uid)); - return 0; -} - -static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { - float delPercent = (float)(pMFile->info.nDels) / (float)(pMFile->info.nRecords); - float tombPercent = (float)(pMFile->info.tombSize) / (float)(pMFile->info.size); - float compactRatio = (float)(tsTsdbMetaCompactRatio)/100; - - if (delPercent < compactRatio && tombPercent < compactRatio) { - return 0; - } - - if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) { - tsdbError("open meta file %s compact fail", pMFile->f.rname); - return -1; - } - - tsdbInfo("begin compact tsdb meta file, ratio:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 ",size:%" PRId64, - tsTsdbMetaCompactRatio, pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size); - - SMFile mf; - SDiskID did; - - // first create tmp meta file - did.level = TFS_PRIMARY_LEVEL; - did.id = TFS_PRIMARY_ID; - tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)) + 1); - - if (tsdbCreateMFile(&mf, true) < 0) { - tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - tsdbInfo("vgId:%d meta file %s is created to compact meta data", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); - - // second iterator metaCache - int code = -1; - int64_t maxBufSize = 1024; - SKVRecord *pRecord; - void *pBuf = NULL; - - pBuf = malloc((size_t)maxBufSize); - if (pBuf == NULL) { - goto _err; - } - - // init Comp - assert(pfs->metaCacheComp == NULL); - pfs->metaCacheComp = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - if (pfs->metaCacheComp == NULL) { - goto _err; - } - - pRecord = taosHashIterate(pfs->metaCache, NULL); - while (pRecord) { - if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { - tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), - tstrerror(terrno)); - goto _err; - } - if (pRecord->size > maxBufSize) { - maxBufSize = pRecord->size; - void* tmp = realloc(pBuf, (size_t)maxBufSize); - if (tmp == NULL) { - goto _err; - } - pBuf = tmp; - } - int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size); - if (nread < 0) { - tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), - tstrerror(terrno)); - goto _err; - } - - if (nread < pRecord->size) { - tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", - REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); - goto _err; - } - - if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, true) < 0) { - tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid, - tstrerror(terrno)); - goto _err; - } - - pRecord = taosHashIterate(pfs->metaCache, pRecord); - } - code = 0; - -_err: - if (code == 0) TSDB_FILE_FSYNC(&mf); - tsdbCloseMFile(&mf); - tsdbCloseMFile(pMFile); - - if (code == 0) { - // rename meta.tmp -> meta - tsdbInfo("vgId:%d meta file rename %s -> %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf), TSDB_FILE_FULL_NAME(pMFile)); - taosRename(mf.f.aname,pMFile->f.aname); - tstrncpy(mf.f.aname, pMFile->f.aname, TSDB_FILENAME_LEN); - tstrncpy(mf.f.rname, pMFile->f.rname, TSDB_FILENAME_LEN); - // update current meta file info - pfs->nstatus->pmf = NULL; - tsdbUpdateMFile(pfs, &mf); - - taosHashCleanup(pfs->metaCache); - pfs->metaCache = pfs->metaCacheComp; - pfs->metaCacheComp = NULL; - } else { - // remove meta.tmp file - remove(mf.f.aname); - taosHashCleanup(pfs->metaCacheComp); - pfs->metaCacheComp = NULL; - } - - tfree(pBuf); - - ASSERT(mf.info.nDels == 0); - ASSERT(mf.info.tombSize == 0); - - tsdbInfo("end compact tsdb meta file,code:%d,nRecords:%" PRId64 ",size:%" PRId64, - code,mf.info.nRecords,mf.info.size); - return code; -} - -// =================== Commit Time-Series Data -static int tsdbCommitTSData(STsdbRepo *pRepo) { - SMemTable *pMem = pRepo->imem; - SCommitH commith; - SDFileSet *pSet = NULL; - int fid; - - memset(&commith, 0, sizeof(commith)); - - if (pMem->numOfRows <= 0) { - // No memory data, just apply retention on each file on disk - if (tsdbApplyRtn(pRepo) < 0) { - return -1; - } - return 0; - } - - // Resource initialization - if (tsdbInitCommitH(&commith, pRepo) < 0) { - return -1; - } - - // Skip expired memory data and expired FSET - tsdbSeekCommitIter(&commith, commith.rtn.minKey); - while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { - if (pSet->fid < commith.rtn.minFid) { - tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); - } else { - break; - } - } - - // Loop to commit to each file - fid = tsdbNextCommitFid(&(commith)); - while (true) { - // Loop over both on disk and memory - if (pSet == NULL && fid == TSDB_IVLD_FID) break; - - if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) { - // Only has existing FSET but no memory data to commit in this - // existing FSET, only check if file in correct retention - if (tsdbApplyRtnOnFSet(pRepo, pSet, &(commith.rtn)) < 0) { - tsdbDestroyCommitH(&commith); - return -1; - } - - pSet = tsdbFSIterNext(&(commith.fsIter)); - } else { - // Has memory data to commit - SDFileSet *pCSet; - int cfid; - - if (pSet == NULL || pSet->fid > fid) { - // Commit to a new FSET with fid: fid - pCSet = NULL; - cfid = fid; - } else { - // Commit to an existing FSET - pCSet = pSet; - cfid = pSet->fid; - pSet = tsdbFSIterNext(&(commith.fsIter)); - } - - if (tsdbCommitToFile(&commith, pCSet, cfid) < 0) { - tsdbDestroyCommitH(&commith); - return -1; - } - - fid = tsdbNextCommitFid(&commith); - } - } - - tsdbDestroyCommitH(&commith); - return 0; -} - -static void tsdbStartCommit(STsdbRepo *pRepo) { - SMemTable *pMem = pRepo->imem; - - ASSERT(pMem->numOfRows > 0 || listNEles(pMem->actList) > 0); - - tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64 " meta rows: %d", - REPO_ID(pRepo), pMem->keyFirst, pMem->keyLast, pMem->numOfRows, listNEles(pMem->actList)); - - tsdbStartFSTxn(pRepo, pMem->pointsAdd, pMem->storageAdd); - - pRepo->code = TSDB_CODE_SUCCESS; -} - -static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { - if (eno != TSDB_CODE_SUCCESS) { - tsdbEndFSTxnWithError(REPO_FS(pRepo)); - } else { - tsdbEndFSTxn(pRepo); - } - - tsdbInfo("vgId:%d commit over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); - - if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno); - - SMemTable *pIMem = pRepo->imem; - (void)tsdbLockRepo(pRepo); - pRepo->imem = NULL; - (void)tsdbUnlockRepo(pRepo); - tsdbUnRefMemTable(pRepo, pIMem); - tsem_post(&(pRepo->readyToCommit)); -} - -#if 0 -static bool tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { - for (int i = 0; i < nIters; i++) { - TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter); - if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return true; - } - return false; -} -#endif - -static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); - - ASSERT(pSet == NULL || pSet->fid == fid); - - tsdbResetCommitFile(pCommith); - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &(pCommith->minKey), &(pCommith->maxKey)); - - // Set and open files - if (tsdbSetAndOpenCommitFile(pCommith, pSet, fid) < 0) { - return -1; - } - - // Loop to commit each table data - for (int tid = 1; tid < pCommith->niters; tid++) { - SCommitIter *pIter = pCommith->iters + tid; - - if (pIter->pTable == NULL) continue; - - if (tsdbCommitToTable(pCommith, tid) < 0) { - tsdbCloseCommitFile(pCommith, true); - // revert the file change - tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); - return -1; - } - } - - if (tsdbWriteBlockIdx(TSDB_COMMIT_HEAD_FILE(pCommith), pCommith->aBlkIdx, (void **)(&(TSDB_COMMIT_BUF(pCommith)))) < - 0) { - tsdbError("vgId:%d failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); - tsdbCloseCommitFile(pCommith, true); - // revert the file change - tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); - return -1; - } - - if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) { - tsdbError("vgId:%d failed to update FSET %d header since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); - tsdbCloseCommitFile(pCommith, true); - // revert the file change - tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); - return -1; - } - - // Close commit file - tsdbCloseCommitFile(pCommith, false); - - if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) { - return -1; - } - - return 0; -} - -static int tsdbCreateCommitIters(SCommitH *pCommith) { - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - SMemTable *pMem = pRepo->imem; - STsdbMeta *pMeta = pRepo->tsdbMeta; - - pCommith->niters = pMem->maxTables; - pCommith->iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter)); - if (pCommith->iters == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (tsdbRLockRepoMeta(pRepo) < 0) return -1; - - // reference all tables - for (int i = 0; i < pMem->maxTables; i++) { - if (pMeta->tables[i] != NULL) { - tsdbRefTable(pMeta->tables[i]); - pCommith->iters[i].pTable = pMeta->tables[i]; - } - } - - if (tsdbUnlockRepoMeta(pRepo) < 0) return -1; - - for (int i = 0; i < pMem->maxTables; i++) { - if ((pCommith->iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && - (TABLE_UID(pCommith->iters[i].pTable) == pMem->tData[i]->uid)) { - if ((pCommith->iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - tSkipListIterNext(pCommith->iters[i].pIter); - } - } - - return 0; -} - -static void tsdbDestroyCommitIters(SCommitH *pCommith) { - if (pCommith->iters == NULL) return; - - for (int i = 1; i < pCommith->niters; i++) { - if (pCommith->iters[i].pTable != NULL) { - tsdbUnRefTable(pCommith->iters[i].pTable); - tSkipListDestroyIter(pCommith->iters[i].pIter); - } - } - - free(pCommith->iters); - pCommith->iters = NULL; - pCommith->niters = 0; -} - -// Skip all keys until key (not included) -static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key) { - for (int i = 0; i < pCommith->niters; i++) { - SCommitIter *pIter = pCommith->iters + i; - if (pIter->pTable == NULL || pIter->pIter == NULL) continue; - - tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, key - 1, INT32_MAX, NULL, NULL, 0, true, NULL); - } -} - -static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo) { - STsdbCfg *pCfg = REPO_CFG(pRepo); - - memset(pCommith, 0, sizeof(*pCommith)); - tsdbGetRtnSnap(pRepo, &(pCommith->rtn)); - - TSDB_FSET_SET_CLOSED(TSDB_COMMIT_WRITE_FSET(pCommith)); - - // Init read handle - if (tsdbInitReadH(&(pCommith->readh), pRepo) < 0) { - return -1; - } - - // Init file iterator - tsdbFSIterInit(&(pCommith->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD); - - if (tsdbCreateCommitIters(pCommith) < 0) { - tsdbDestroyCommitH(pCommith); - return -1; - } - - pCommith->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx)); - if (pCommith->aBlkIdx == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCommitH(pCommith); - return -1; - } - - pCommith->aSupBlk = taosArrayInit(1024, sizeof(SBlock)); - if (pCommith->aSupBlk == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCommitH(pCommith); - return -1; - } - - pCommith->aSubBlk = taosArrayInit(1024, sizeof(SBlock)); - if (pCommith->aSubBlk == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCommitH(pCommith); - return -1; - } - - pCommith->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); - if (pCommith->pDataCols == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbDestroyCommitH(pCommith); - return -1; - } - - return 0; -} - -static void tsdbDestroyCommitH(SCommitH *pCommith) { - pCommith->pDataCols = tdFreeDataCols(pCommith->pDataCols); - pCommith->aSubBlk = taosArrayDestroy(pCommith->aSubBlk); - pCommith->aSupBlk = taosArrayDestroy(pCommith->aSupBlk); - pCommith->aBlkIdx = taosArrayDestroy(pCommith->aBlkIdx); - tsdbDestroyCommitIters(pCommith); - tsdbDestroyReadH(&(pCommith->readh)); - tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); -} - -static int tsdbNextCommitFid(SCommitH *pCommith) { - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); - int fid = TSDB_IVLD_FID; - - for (int i = 0; i < pCommith->niters; i++) { - SCommitIter *pIter = pCommith->iters + i; - if (pIter->pTable == NULL || pIter->pIter == NULL) continue; - - TSKEY nextKey = tsdbNextIterKey(pIter->pIter); - if (nextKey == TSDB_DATA_TIMESTAMP_NULL) { - continue; - } else { - int tfid = (int)(TSDB_KEY_FID(nextKey, pCfg->daysPerFile, pCfg->precision)); - if (fid == TSDB_IVLD_FID || fid > tfid) { - fid = tfid; - } - } - } - - return fid; -} - -static int tsdbCommitToTable(SCommitH *pCommith, int tid) { - SCommitIter *pIter = pCommith->iters + tid; - TSKEY nextKey = tsdbNextIterKey(pIter->pIter); - - tsdbResetCommitTable(pCommith); - - TSDB_RLOCK_TABLE(pIter->pTable); - - // Set commit table - if (tsdbSetCommitTable(pCommith, pIter->pTable) < 0) { - TSDB_RUNLOCK_TABLE(pIter->pTable); - return -1; - } - - // No disk data and no memory data, just return - if (pCommith->readh.pBlkIdx == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) { - TSDB_RUNLOCK_TABLE(pIter->pTable); - return 0; - } - - // Must has disk data or has memory data - int nBlocks; - int bidx = 0; - SBlock *pBlock; - - if (pCommith->readh.pBlkIdx) { - if (tsdbLoadBlockInfo(&(pCommith->readh), NULL) < 0) { - TSDB_RUNLOCK_TABLE(pIter->pTable); - return -1; - } - - nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; - } else { - nBlocks = 0; - } - - if (bidx < nBlocks) { - pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - } else { - pBlock = NULL; - } - - while (true) { - if (pBlock == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) break; - - if ((nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey) || - (pBlock && (!pBlock->last) && tsdbComparKeyBlock((void *)(&nextKey), pBlock) > 0)) { - if (tsdbMoveBlock(pCommith, bidx) < 0) { - TSDB_RUNLOCK_TABLE(pIter->pTable); - return -1; - } - - bidx++; - if (bidx < nBlocks) { - pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - } else { - pBlock = NULL; - } - } else if (pBlock && (pBlock->last || tsdbComparKeyBlock((void *)(&nextKey), pBlock) == 0)) { - // merge pBlock data and memory data - if (tsdbMergeMemData(pCommith, pIter, bidx) < 0) { - TSDB_RUNLOCK_TABLE(pIter->pTable); - return -1; - } - - bidx++; - if (bidx < nBlocks) { - pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - } else { - pBlock = NULL; - } - nextKey = tsdbNextIterKey(pIter->pIter); - } else { - // Only commit memory data - if (pBlock == NULL) { - if (tsdbCommitMemData(pCommith, pIter, pCommith->maxKey, false) < 0) { - TSDB_RUNLOCK_TABLE(pIter->pTable); - return -1; - } - } else { - if (tsdbCommitMemData(pCommith, pIter, pBlock->keyFirst - 1, true) < 0) { - TSDB_RUNLOCK_TABLE(pIter->pTable); - return -1; - } - } - nextKey = tsdbNextIterKey(pIter->pIter); - } - } - - TSDB_RUNLOCK_TABLE(pIter->pTable); - - if (tsdbWriteBlockInfo(pCommith) < 0) { - tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), - TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); - return -1; - } - - return 0; -} - -static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) { - STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - - pCommith->pTable = pTable; - - if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (pCommith->isRFileSet) { - if (tsdbSetReadTable(&(pCommith->readh), pTable) < 0) { - return -1; - } - } else { - pCommith->readh.pBlkIdx = NULL; - } - return 0; -} - -static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { - TSKEY key = *(TSKEY *)arg1; - SBlock *pBlock = (SBlock *)arg2; - - if (key < pBlock->keyFirst) { - return -1; - } else if (key > pBlock->keyLast) { - return 1; - } else { - return 0; - } -} - -int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, - bool isLast, bool isSuper, void **ppBuf, void **ppCBuf) { - STsdbCfg * pCfg = REPO_CFG(pRepo); - SBlockData *pBlockData; - int64_t offset = 0; - int rowsToWrite = pDataCols->numOfRows; - - ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock); - ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock); - - // Make buffer space - if (tsdbMakeRoom(ppBuf, TSDB_BLOCK_STATIS_SIZE(pDataCols->numOfCols)) < 0) { - return -1; - } - pBlockData = (SBlockData *)(*ppBuf); - - // Get # of cols not all NULL(not including key column) - int nColsNotAllNull = 0; - for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column - SDataCol * pDataCol = pDataCols->cols + ncol; - SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; - - if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it - continue; - } - - memset(pBlockCol, 0, sizeof(*pBlockCol)); - - pBlockCol->colId = pDataCol->colId; - pBlockCol->type = pDataCol->type; - if (tDataTypes[pDataCol->type].statisFunc) { - (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), - &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), - &(pBlockCol->numOfNull)); - } - nColsNotAllNull++; - } - - ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols); - - // Compress the data if neccessary - int tcol = 0; // counter of not all NULL and written columns - uint32_t toffset = 0; - int32_t tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull); - int32_t lsize = tsize; - int32_t keyLen = 0; - for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { - // All not NULL columns finish - if (ncol != 0 && tcol >= nColsNotAllNull) break; - - SDataCol * pDataCol = pDataCols->cols + ncol; - SBlockCol *pBlockCol = pBlockData->cols + tcol; - - if (ncol != 0 && (pDataCol->colId != pBlockCol->colId)) continue; - - int32_t flen; // final length - int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite); - void * tptr; - - // Make room - if (tsdbMakeRoom(ppBuf, lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { - return -1; - } - pBlockData = (SBlockData *)(*ppBuf); - pBlockCol = pBlockData->cols + tcol; - tptr = POINTER_SHIFT(pBlockData, lsize); - - if (pCfg->compression == TWO_STAGE_COMP && - tsdbMakeRoom(ppCBuf, tlen + COMP_OVERFLOW_BYTES) < 0) { - return -1; - } - - // Compress or just copy - if (pCfg->compression) { - flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, - tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, - tlen + COMP_OVERFLOW_BYTES); - } else { - flen = tlen; - memcpy(tptr, pDataCol->pData, flen); - } - - // Add checksum - ASSERT(flen > 0); - flen += sizeof(TSCKSUM); - taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); - tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM))); - - if (ncol != 0) { - tsdbSetBlockColOffset(pBlockCol, toffset); - pBlockCol->len = flen; - tcol++; - } else { - keyLen = flen; - } - - toffset += flen; - lsize += flen; - } - - pBlockData->delimiter = TSDB_FILE_DELIMITER; - pBlockData->uid = TABLE_UID(pTable); - pBlockData->numOfCols = nColsNotAllNull; - - taosCalcChecksumAppend(0, (uint8_t *)pBlockData, tsize); - tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(pBlockData, tsize - sizeof(TSCKSUM))); - - // Write the whole block to file - if (tsdbAppendDFile(pDFile, (void *)pBlockData, lsize, &offset) < lsize) { - return -1; - } - - // Update pBlock membership vairables - pBlock->last = isLast; - pBlock->offset = offset; - pBlock->algorithm = pCfg->compression; - pBlock->numOfRows = rowsToWrite; - pBlock->len = lsize; - pBlock->keyLen = keyLen; - pBlock->numOfSubBlocks = isSuper ? 1 : 0; - pBlock->numOfCols = nColsNotAllNull; - pBlock->keyFirst = dataColsKeyFirst(pDataCols); - pBlock->keyLast = dataColsKeyLast(pDataCols); - - tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 - " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, - REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, - pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); - - return 0; -} - -static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, - bool isSuper) { - return tsdbWriteBlockImpl(TSDB_COMMIT_REPO(pCommith), TSDB_COMMIT_TABLE(pCommith), pDFile, pDataCols, pBlock, isLast, - isSuper, (void **)(&(TSDB_COMMIT_BUF(pCommith))), - (void **)(&(TSDB_COMMIT_COMP_BUF(pCommith)))); -} - - -static int tsdbWriteBlockInfo(SCommitH *pCommih) { - SDFile * pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); - SBlockIdx blkIdx; - STable * pTable = TSDB_COMMIT_TABLE(pCommih); - - if (tsdbWriteBlockInfoImpl(pHeadf, pTable, pCommih->aSupBlk, pCommih->aSubBlk, (void **)(&(TSDB_COMMIT_BUF(pCommih))), - &blkIdx) < 0) { - return -1; - } - - if (blkIdx.numOfBlocks == 0) { - return 0; - } - - if (taosArrayPush(pCommih->aBlkIdx, (void *)(&blkIdx)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData) { - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); - SMergeInfo mInfo; - int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); - SDFile * pDFile; - bool isLast; - SBlock block; - - while (true) { - tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, defaultRows, pCommith->pDataCols, NULL, 0, - pCfg->update, &mInfo); - - if (pCommith->pDataCols->numOfRows <= 0) break; - - if (toData || pCommith->pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - isLast = false; - } else { - pDFile = TSDB_COMMIT_LAST_FILE(pCommith); - isLast = true; - } - - if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; - - if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) { - return -1; - } - } - - return 0; -} - -static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); - int nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; - SBlock * pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - TSKEY keyLimit; - int16_t colId = 0; - SMergeInfo mInfo; - SBlock subBlocks[TSDB_MAX_SUBBLOCKS]; - SBlock block, supBlock; - SDFile * pDFile; - - if (bidx == nBlocks - 1) { - keyLimit = pCommith->maxKey; - } else { - keyLimit = pBlock[1].keyFirst - 1; - } - - SSkipListIterator titer = *(pIter->pIter); - if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1) < 0) return -1; - - tsdbLoadDataFromCache(pIter->pTable, &titer, keyLimit, INT32_MAX, NULL, pCommith->readh.pDCols[0]->cols[0].pData, - pCommith->readh.pDCols[0]->numOfRows, pCfg->update, &mInfo); - - if (mInfo.nOperations == 0) { - // no new data to insert (all updates denied) - if (tsdbMoveBlock(pCommith, bidx) < 0) { - return -1; - } - *(pIter->pIter) = titer; - } else if (pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed == 0) { - // Ignore the block - ASSERT(0); - *(pIter->pIter) = titer; - } else if (tsdbCanAddSubBlock(pCommith, pBlock, &mInfo)) { - // Add a sub-block - tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, INT32_MAX, pCommith->pDataCols, - pCommith->readh.pDCols[0]->cols[0].pData, pCommith->readh.pDCols[0]->numOfRows, pCfg->update, - &mInfo); - if (pBlock->last) { - pDFile = TSDB_COMMIT_LAST_FILE(pCommith); - } else { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - } - - if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, pBlock->last, false) < 0) return -1; - - if (pBlock->numOfSubBlocks == 1) { - subBlocks[0] = *pBlock; - subBlocks[0].numOfSubBlocks = 0; - } else { - memcpy(subBlocks, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), - sizeof(SBlock) * pBlock->numOfSubBlocks); - } - subBlocks[pBlock->numOfSubBlocks] = block; - supBlock = *pBlock; - supBlock.keyFirst = mInfo.keyFirst; - supBlock.keyLast = mInfo.keyLast; - supBlock.numOfSubBlocks++; - supBlock.numOfRows = pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed; - supBlock.offset = taosArrayGetSize(pCommith->aSubBlk) * sizeof(SBlock); - - if (tsdbCommitAddBlock(pCommith, &supBlock, subBlocks, supBlock.numOfSubBlocks) < 0) return -1; - } else { - if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; - if (tsdbMergeBlockData(pCommith, pIter, pCommith->readh.pDCols[0], keyLimit, bidx == (nBlocks - 1)) < 0) return -1; - } - - return 0; -} - -static int tsdbMoveBlock(SCommitH *pCommith, int bidx) { - SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; - SDFile *pDFile; - SBlock block; - bool isSameFile; - - ASSERT(pBlock->numOfSubBlocks > 0); - - if (pBlock->last) { - pDFile = TSDB_COMMIT_LAST_FILE(pCommith); - isSameFile = pCommith->isLFileSame; - } else { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - isSameFile = pCommith->isDFileSame; - } - - if (isSameFile) { - if (pBlock->numOfSubBlocks == 1) { - if (tsdbCommitAddBlock(pCommith, pBlock, NULL, 0) < 0) { - return -1; - } - } else { - block = *pBlock; - block.offset = sizeof(SBlock) * taosArrayGetSize(pCommith->aSubBlk); - - if (tsdbCommitAddBlock(pCommith, &block, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), - pBlock->numOfSubBlocks) < 0) { - return -1; - } - } - } else { - if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; - if (tsdbWriteBlock(pCommith, pDFile, pCommith->readh.pDCols[0], &block, pBlock->last, true) < 0) return -1; - if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; - } - - return 0; -} - -static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks) { - if (taosArrayPush(pCommith->aSupBlk, pSupBlock) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (pSubBlocks && taosArrayAddBatch(pCommith->aSubBlk, pSubBlocks, nSubBlocks) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - -static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, bool isLastOneBlock) { - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); - SBlock block; - SDFile * pDFile; - bool isLast; - int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); - - int biter = 0; - while (true) { - tsdbLoadAndMergeFromCache(pCommith->readh.pDCols[0], &biter, pIter, pCommith->pDataCols, keyLimit, defaultRows, - pCfg->update); - - if (pCommith->pDataCols->numOfRows == 0) break; - - if (isLastOneBlock) { - if (pCommith->pDataCols->numOfRows < pCfg->minRowsPerFileBlock) { - pDFile = TSDB_COMMIT_LAST_FILE(pCommith); - isLast = true; - } else { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - isLast = false; - } - } else { - pDFile = TSDB_COMMIT_DATA_FILE(pCommith); - isLast = false; - } - - if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; - if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; - } - - return 0; -} - -static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows, int8_t update) { - TSKEY key1 = INT64_MAX; - TSKEY key2 = INT64_MAX; - STSchema *pSchema = NULL; - - ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); - tdResetDataCols(pTarget); - - while (true) { - key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); - SMemRow row = tsdbNextIterRow(pCommitIter->pIter); - if (row == NULL || memRowKey(row) > maxKey) { - key2 = INT64_MAX; - } else { - key2 = memRowKey(row); - } - - if (key1 == INT64_MAX && key2 == INT64_MAX) break; - - if (key1 < key2) { - for (int i = 0; i < pDataCols->numOfCols; i++) { - //TODO: dataColAppendVal may fail - dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, - pTarget->maxPoints); - } - - pTarget->numOfRows++; - (*iter)++; - } else if (key1 > key2) { - if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row)); - ASSERT(pSchema != NULL); - } - - tdAppendMemRowToDataCol(row, pSchema, pTarget, true); - - tSkipListIterNext(pCommitIter->pIter); - } else { - if (update != TD_ROW_OVERWRITE_UPDATE) { - //copy disk data - for (int i = 0; i < pDataCols->numOfCols; i++) { - //TODO: dataColAppendVal may fail - dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, - pTarget->maxPoints); - } - - if(update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++; - } - if (update != TD_ROW_DISCARD_UPDATE) { - //copy mem data - if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row)); - ASSERT(pSchema != NULL); - } - - tdAppendMemRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE); - } - (*iter)++; - tSkipListIterNext(pCommitIter->pIter); - } - - if (pTarget->numOfRows >= maxRows) break; - } -} - -static void tsdbResetCommitFile(SCommitH *pCommith) { - pCommith->isRFileSet = false; - pCommith->isDFileSame = false; - pCommith->isLFileSame = false; - taosArrayClear(pCommith->aBlkIdx); -} - -static void tsdbResetCommitTable(SCommitH *pCommith) { - taosArrayClear(pCommith->aSubBlk); - taosArrayClear(pCommith->aSupBlk); - pCommith->pTable = NULL; -} - -static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { - SDiskID did; - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith); - - tfsAllocDisk(tsdbGetFidLevel(fid, &(pCommith->rtn)), &(did.level), &(did.id)); - if (did.level == TFS_UNDECIDED_LEVEL) { - terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; - return -1; - } - - // Open read FSET - if (pSet) { - if (tsdbSetAndOpenReadFSet(&(pCommith->readh), pSet) < 0) { - return -1; - } - - pCommith->isRFileSet = true; - - if (tsdbLoadBlockIdx(&(pCommith->readh)) < 0) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - - tsdbDebug("vgId:%d FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet), - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); - } else { - pCommith->isRFileSet = false; - } - - // Set and open commit FSET - if (pSet == NULL || did.level > TSDB_FSET_LEVEL(pSet)) { - // Create a new FSET to write data - tsdbInitDFileSet(pWSet, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo))); - - if (tsdbCreateDFileSet(pWSet, true) < 0) { - tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), - TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno)); - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - } - return -1; - } - - pCommith->isDFileSame = false; - pCommith->isLFileSame = false; - - tsdbDebug("vgId:%d FSET %d at level %d disk id %d is created to commit", REPO_ID(pRepo), TSDB_FSET_FID(pWSet), - TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet)); - } else { - did.level = TSDB_FSET_LEVEL(pSet); - did.id = TSDB_FSET_ID(pSet); - - pCommith->wSet.fid = fid; - pCommith->wSet.state = 0; - - // TSDB_FILE_HEAD - SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); - tsdbInitDFile(pWHeadf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); - if (tsdbCreateDFile(pWHeadf, true) < 0) { - tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), - tstrerror(terrno)); - - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - } - - // TSDB_FILE_DATA - SDFile *pRDataf = TSDB_READ_DATA_FILE(&(pCommith->readh)); - SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith); - tsdbInitDFileEx(pWDataf, pRDataf); - if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { - tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), - tstrerror(terrno)); - - tsdbCloseDFileSet(pWSet); - tsdbRemoveDFile(pWHeadf); - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - } - pCommith->isDFileSame = true; - - // TSDB_FILE_LAST - SDFile *pRLastf = TSDB_READ_LAST_FILE(&(pCommith->readh)); - SDFile *pWLastf = TSDB_COMMIT_LAST_FILE(pCommith); - if (pRLastf->info.size < 32 * 1024) { - tsdbInitDFileEx(pWLastf, pRLastf); - pCommith->isLFileSame = true; - - if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { - tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), - tstrerror(terrno)); - - tsdbCloseDFileSet(pWSet); - tsdbRemoveDFile(pWHeadf); - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - } - } else { - tsdbInitDFile(pWLastf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST); - pCommith->isLFileSame = false; - - if (tsdbCreateDFile(pWLastf, true) < 0) { - tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), - tstrerror(terrno)); - - tsdbCloseDFileSet(pWSet); - (void)tsdbRemoveDFile(pWHeadf); - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - return -1; - } - } - } - } - - return 0; -} - -static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError) { - if (pCommith->isRFileSet) { - tsdbCloseAndUnsetFSet(&(pCommith->readh)); - } - - if (!hasError) { - TSDB_FSET_FSYNC(TSDB_COMMIT_WRITE_FSET(pCommith)); - } - tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); -} - -static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo) { - STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); - int mergeRows = pBlock->numOfRows + pInfo->rowsInserted - pInfo->rowsDeleteSucceed; - - ASSERT(mergeRows > 0); - - if (pBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pInfo->nOperations <= pCfg->maxRowsPerFileBlock) { - if (pBlock->last) { - if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true; - } else { - if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true; - } - } - - return false; -} - -int tsdbApplyRtn(STsdbRepo *pRepo) { - SRtn rtn; - SFSIter fsiter; - STsdbFS * pfs = REPO_FS(pRepo); - SDFileSet *pSet; - - // Get retention snapshot - tsdbGetRtnSnap(pRepo, &rtn); - - tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); - while ((pSet = tsdbFSIterNext(&fsiter))) { - if (pSet->fid < rtn.minFid) { - tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, - TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); - continue; - } - - if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) { - return -1; - } - } - - return 0; -} diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c deleted file mode 100644 index c2021963e0d0c8be4ed42588549153dcd20be63c..0000000000000000000000000000000000000000 --- a/src/tsdb/src/tsdbMain.c +++ /dev/null @@ -1,1008 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// no test file errors here -#include "taosdef.h" -#include "tsdbint.h" -#include "ttimer.h" -#include "tthread.h" - -#define IS_VALID_PRECISION(precision) \ - (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO)) -#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP -#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) - -static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); -static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH); -static void tsdbFreeRepo(STsdbRepo *pRepo); -static void tsdbStartStream(STsdbRepo *pRepo); -static void tsdbStopStream(STsdbRepo *pRepo); -static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh); -static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, SBlockIdx *pIdx); - -// Function declaration -int32_t tsdbCreateRepo(int repoid) { - char tsdbDir[TSDB_FILENAME_LEN] = "\0"; - char dataDir[TSDB_FILENAME_LEN] = "\0"; - - tsdbGetRootDir(repoid, tsdbDir); - if (tfsMkdir(tsdbDir) < 0) { - goto _err; - } - - tsdbGetDataDir(repoid, dataDir); - if (tfsMkdir(dataDir) < 0) { - goto _err; - } - - // TODO: need to create current file with nothing in - - return 0; - -_err: - tsdbError("vgId:%d failed to create TSDB repository since %s", repoid, tstrerror(terrno)); - return -1; -} - -int32_t tsdbDropRepo(int repoid) { - char tsdbDir[TSDB_FILENAME_LEN] = "\0"; - - tsdbGetRootDir(repoid, tsdbDir); - return tfsRmdir(tsdbDir); -} - -STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { - STsdbRepo *pRepo; - STsdbCfg config = *pCfg; - - terrno = TSDB_CODE_SUCCESS; - - // Check and set default configurations - if (tsdbCheckAndSetDefaultCfg(&config) < 0) { - tsdbError("vgId:%d failed to open TSDB repository since %s", config.tsdbId, tstrerror(terrno)); - return NULL; - } - - // Create new TSDB object - if ((pRepo = tsdbNewRepo(&config, pAppH)) == NULL) { - tsdbError("vgId:%d failed to open TSDB repository while creating TSDB object since %s", config.tsdbId, - tstrerror(terrno)); - return NULL; - } - - // Open meta - if (tsdbOpenMeta(pRepo) < 0) { - tsdbError("vgId:%d failed to open TSDB repository while opening Meta since %s", config.tsdbId, tstrerror(terrno)); - tsdbCloseRepo(pRepo, false); - return NULL; - } - - if (tsdbOpenBufPool(pRepo) < 0) { - tsdbError("vgId:%d failed to open TSDB repository while opening buffer pool since %s", config.tsdbId, - tstrerror(terrno)); - tsdbCloseRepo(pRepo, false); - return NULL; - } - - if (tsdbOpenFS(pRepo) < 0) { - tsdbError("vgId:%d failed to open TSDB repository while opening FS since %s", config.tsdbId, tstrerror(terrno)); - tsdbCloseRepo(pRepo, false); - return NULL; - } - - // TODO: Restore information from data - if ((!(pRepo->state & TSDB_STATE_BAD_DATA)) && tsdbRestoreInfo(pRepo) < 0) { - tsdbError("vgId:%d failed to open TSDB repository while restore info since %s", config.tsdbId, tstrerror(terrno)); - tsdbCloseRepo(pRepo, false); - return NULL; - } - - pRepo->mergeBuf = NULL; - - tsdbStartStream(pRepo); - - tsdbDebug("vgId:%d, TSDB repository opened", REPO_ID(pRepo)); - - return pRepo; -} - -// Note: all working thread and query thread must stopped when calling this function -int tsdbCloseRepo(STsdbRepo *repo, int toCommit) { - if (repo == NULL) return 0; - - STsdbRepo *pRepo = repo; - int vgId = REPO_ID(pRepo); - - terrno = TSDB_CODE_SUCCESS; - - tsdbStopStream(pRepo); - if(pRepo->pthread){ - taosDestoryThread(pRepo->pthread); - pRepo->pthread = NULL; - } - - if (toCommit) { - tsdbSyncCommit(repo); - } - - tsem_wait(&(pRepo->readyToCommit)); - - tsdbUnRefMemTable(pRepo, pRepo->mem); - tsdbUnRefMemTable(pRepo, pRepo->imem); - pRepo->mem = NULL; - pRepo->imem = NULL; - - tsdbCloseFS(pRepo); - tsdbCloseBufPool(pRepo); - tsdbCloseMeta(pRepo); - tsdbFreeRepo(pRepo); - tsdbDebug("vgId:%d repository is closed", vgId); - - if (terrno != TSDB_CODE_SUCCESS) { - return -1; - } else { - return 0; - } -} - -STsdbCfg *tsdbGetCfg(const STsdbRepo *repo) { - ASSERT(repo != NULL); - return &((STsdbRepo *)repo)->config; -} - -int tsdbLockRepo(STsdbRepo *pRepo) { - int code = pthread_mutex_lock(&pRepo->mutex); - if (code != 0) { - tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - pRepo->repoLocked = true; - return 0; -} - -int tsdbUnlockRepo(STsdbRepo *pRepo) { - ASSERT(IS_REPO_LOCKED(pRepo)); - pRepo->repoLocked = false; - int code = pthread_mutex_unlock(&pRepo->mutex); - if (code != 0) { - tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -int tsdbCheckCommit(STsdbRepo *pRepo) { - ASSERT(pRepo->mem != NULL); - STsdbCfg *pCfg = &(pRepo->config); - - STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); - ASSERT(pBufBlock != NULL); - if ((pRepo->mem->extraBuffList != NULL) || - ((listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 3) && (pBufBlock->remain < TSDB_BUFFER_RESERVE))) { - // trigger commit - if (tsdbAsyncCommit(pRepo) < 0) return -1; - } - - return 0; -} - -STsdbMeta *tsdbGetMeta(STsdbRepo *pRepo) { return pRepo->tsdbMeta; } - -STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo) { return NULL; } - -int tsdbGetState(STsdbRepo *repo) { return repo->state; } - -int8_t tsdbGetCompactState(STsdbRepo *repo) { return (int8_t)(repo->compactState); } - -void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) { - ASSERT(repo != NULL); - STsdbRepo *pRepo = repo; - *totalPoints = pRepo->stat.pointsWritten; - *totalStorage = pRepo->stat.totalStorage; - *compStorage = pRepo->stat.compStorage; -} - -int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { - // TODO: think about multithread cases - if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; - - STsdbCfg * pRCfg = &repo->config; - - ASSERT(pRCfg->tsdbId == pCfg->tsdbId); - ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); - ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); - ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); - ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); - ASSERT(pRCfg->precision == pCfg->precision); - - bool configChanged = false; - if (pRCfg->compression != pCfg->compression) { - configChanged = true; - } - if (pRCfg->keep != pCfg->keep) { - configChanged = true; - } - if (pRCfg->keep1 != pCfg->keep1) { - configChanged = true; - } - if (pRCfg->keep2 != pCfg->keep2) { - configChanged = true; - } - if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { - configChanged = true; - } - if (pRCfg->totalBlocks != pCfg->totalBlocks) { - configChanged = true; - } - - if (!configChanged) { - tsdbError("vgId:%d no config changed", REPO_ID(repo)); - } - - int code = pthread_mutex_lock(&repo->save_mutex); - if (code != 0) { - tsdbError("vgId:%d failed to lock tsdb save config mutex since %s", REPO_ID(repo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - - STsdbCfg * pSaveCfg = &repo->save_config; - *pSaveCfg = repo->config; - - pSaveCfg->compression = pCfg->compression; - pSaveCfg->keep = pCfg->keep; - pSaveCfg->keep1 = pCfg->keep1; - pSaveCfg->keep2 = pCfg->keep2; - pSaveCfg->cacheLastRow = pCfg->cacheLastRow; - pSaveCfg->totalBlocks = pCfg->totalBlocks; - - tsdbInfo("vgId:%d old config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d),totalBlocks(%d)", - REPO_ID(repo), - pRCfg->compression, pRCfg->keep, pRCfg->keep1,pRCfg->keep2, - pRCfg->cacheLastRow, pRCfg->totalBlocks); - tsdbInfo("vgId:%d new config: compression(%d), keep(%d,%d,%d), cacheLastRow(%d),totalBlocks(%d)", - REPO_ID(repo), - pSaveCfg->compression, pSaveCfg->keep,pSaveCfg->keep1, pSaveCfg->keep2, - pSaveCfg->cacheLastRow,pSaveCfg->totalBlocks); - - repo->config_changed = true; - - pthread_mutex_unlock(&repo->save_mutex); - - // schedule a commit msg and wait for the new config applied - tsdbSyncCommitConfig(repo); - - return 0; -#if 0 - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbCfg config = pRepo->config; - STsdbCfg * pRCfg = &pRepo->config; - - if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; - - ASSERT(pRCfg->tsdbId == pCfg->tsdbId); - ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); - ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); - ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); - ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); - ASSERT(pRCfg->precision == pCfg->precision); - - bool configChanged = false; - if (pRCfg->compression != pCfg->compression) { - tsdbAlterCompression(pRepo, pCfg->compression); - config.compression = pCfg->compression; - configChanged = true; - } - if (pRCfg->keep != pCfg->keep) { - if (tsdbAlterKeep(pRepo, pCfg->keep) < 0) { - tsdbError("vgId:%d failed to configure repo when alter keep since %s", REPO_ID(pRepo), tstrerror(terrno)); - config.keep = pCfg->keep; - return -1; - } - configChanged = true; - } - if (pRCfg->totalBlocks != pCfg->totalBlocks) { - tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks); - config.totalBlocks = pCfg->totalBlocks; - configChanged = true; - } - if (pRCfg->cacheLastRow != pCfg->cacheLastRow) { - config.cacheLastRow = pCfg->cacheLastRow; - configChanged = true; - } - - if (configChanged) { - if (tsdbSaveConfig(pRepo->rootDir, &config) < 0) { - tsdbError("vgId:%d failed to configure repository while save config since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - } - - return 0; -#endif -} - -uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) { - // TODO - return 0; -#if 0 - STsdbRepo *pRepo = (STsdbRepo *)repo; - // STsdbMeta *pMeta = pRepo->tsdbMeta; - STsdbFileH *pFileH = pRepo->tsdbFileH; - uint32_t magic = 0; - char * fname = NULL; - - struct stat fState; - - tsdbDebug("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex); - ASSERT(*index <= eindex); - - if (name[0] == 0) { // get the file from index or after, but not larger than eindex - int fid = (*index) / TSDB_FILE_TYPE_MAX; - - if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { - if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { - fname = tsdbGetMetaFileName(pRepo->rootDir); - *index = TSDB_META_FILE_INDEX; - magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); - sprintf(name, "tsdb/%s", TSDB_META_FILE_NAME); - } else { - return 0; - } - } else { - SFileGroup *pFGroup = - taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); - if (pFGroup->fileId == fid) { - SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX]; - fname = strdup(TSDB_FILE_NAME(pFile)); - magic = pFile->info.magic; - char *tfname = strdup(fname); - sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); - tfree(tfname); - } else { - if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) { - SFile *pFile = &pFGroup->files[0]; - fname = strdup(TSDB_FILE_NAME(pFile)); - *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; - magic = pFile->info.magic; - char *tfname = strdup(fname); - sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); - tfree(tfname); - } else { - return 0; - } - } - } - } else { // get the named file at the specified index. If not there, return 0 - fname = malloc(256); - sprintf(fname, "%s/vnode/vnode%d/%s", TFS_PRIMARY_PATH(), REPO_ID(pRepo), name); - if (access(fname, F_OK) != 0) { - tfree(fname); - return 0; - } - if (*index == TSDB_META_FILE_INDEX) { // get meta file - tsdbGetStoreInfo(fname, &magic, size); - } else { - char tfname[TSDB_FILENAME_LEN] = "\0"; - sprintf(tfname, "vnode/vnode%d/tsdb/%s/%s", REPO_ID(pRepo), TSDB_DATA_DIR_NAME, basename(name)); - tsdbGetFileInfoImpl(tfname, &magic, size); - } - tfree(fname); - return magic; - } - - if (stat(fname, &fState) < 0) { - tfree(fname); - return 0; - } - - *size = fState.st_size; - // magic = *size; - - tfree(fname); - return magic; -#endif -} - -void tsdbGetRootDir(int repoid, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb", repoid); -} - -void tsdbGetDataDir(int repoid, char dirName[]) { - snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data", repoid); -} - -static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { - // Check tsdbId - if (pCfg->tsdbId < 0) { - tsdbError("vgId:%d invalid vgroup ID", pCfg->tsdbId); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - - // Check precision - if (pCfg->precision == -1) { - pCfg->precision = TSDB_DEFAULT_PRECISION; - } else { - if (!IS_VALID_PRECISION(pCfg->precision)) { - tsdbError("vgId:%d invalid precision configuration %d", pCfg->tsdbId, pCfg->precision); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - // Check compression - if (pCfg->compression == -1) { - pCfg->compression = TSDB_DEFAULT_COMPRESSION; - } else { - if (!IS_VALID_COMPRESSION(pCfg->compression)) { - tsdbError("vgId:%d invalid compression configuration %d", pCfg->tsdbId, pCfg->precision); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - // Check daysPerFile - if (pCfg->daysPerFile == -1) { - pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; - } else { - if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { - tsdbError( - "vgId:%d invalid daysPerFile configuration! daysPerFile %d TSDB_MIN_DAYS_PER_FILE %d TSDB_MAX_DAYS_PER_FILE " - "%d", - pCfg->tsdbId, pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - // Check minRowsPerFileBlock and maxRowsPerFileBlock - if (pCfg->minRowsPerFileBlock == -1) { - pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; - } else { - if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { - tsdbError( - "vgId:%d invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MIN_ROW_FBLOCK %d " - "TSDB_MAX_MIN_ROW_FBLOCK %d", - pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - if (pCfg->maxRowsPerFileBlock == -1) { - pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; - } else { - if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { - tsdbError( - "vgId:%d invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAX_ROW_FBLOCK %d " - "TSDB_MAX_MAX_ROW_FBLOCK %d", - pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { - tsdbError("vgId:%d invalid configuration! minRowsPerFileBlock %d maxRowsPerFileBlock %d", pCfg->tsdbId, - pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - - // Check keep - if (pCfg->keep == -1) { - pCfg->keep = TSDB_DEFAULT_KEEP; - } else { - if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) { - tsdbError( - "vgId:%d invalid keep configuration! keep %d TSDB_MIN_KEEP %d " - "TSDB_MAX_KEEP %d", - pCfg->tsdbId, pCfg->keep, TSDB_MIN_KEEP, TSDB_MAX_KEEP); - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; - } - } - - if (pCfg->keep1 == 0) { - pCfg->keep1 = pCfg->keep; - } - - if (pCfg->keep2 == 0) { - pCfg->keep2 = pCfg->keep; - } - - // update check - if (pCfg->update < TD_ROW_DISCARD_UPDATE || pCfg->update > TD_ROW_PARTIAL_UPDATE) - pCfg->update = TD_ROW_DISCARD_UPDATE; - - // update cacheLastRow - if (pCfg->cacheLastRow != 0) { - if (pCfg->cacheLastRow > 3) - pCfg->cacheLastRow = 1; - } - return 0; -} - -static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { - STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(*pRepo)); - if (pRepo == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - pRepo->state = TSDB_STATE_OK; - pRepo->code = TSDB_CODE_SUCCESS; - pRepo->compactState = 0; - pRepo->config = *pCfg; - if (pAppH) { - pRepo->appH = *pAppH; - } - pRepo->repoLocked = false; - pRepo->pthread = NULL; - - int code = pthread_mutex_init(&(pRepo->mutex), NULL); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - tsdbFreeRepo(pRepo); - return NULL; - } - - code = pthread_mutex_init(&(pRepo->save_mutex), NULL); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - tsdbFreeRepo(pRepo); - return NULL; - } - pRepo->config_changed = false; - atomic_store_8(&pRepo->hasCachedLastColumn, 0); - - code = tsem_init(&(pRepo->readyToCommit), 0, 1); - if (code != 0) { - code = errno; - terrno = TAOS_SYSTEM_ERROR(code); - tsdbFreeRepo(pRepo); - return NULL; - } - - pRepo->tsdbMeta = tsdbNewMeta(pCfg); - if (pRepo->tsdbMeta == NULL) { - tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno)); - tsdbFreeRepo(pRepo); - return NULL; - } - - pRepo->pPool = tsdbNewBufPool(pCfg); - if (pRepo->pPool == NULL) { - tsdbError("vgId:%d failed to create buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno)); - tsdbFreeRepo(pRepo); - return NULL; - } - - pRepo->fs = tsdbNewFS(pCfg); - if (pRepo->fs == NULL) { - tsdbError("vgId:%d failed to TSDB file system since %s", REPO_ID(pRepo), tstrerror(terrno)); - tsdbFreeRepo(pRepo); - return NULL; - } - - return pRepo; -} - -static void tsdbFreeRepo(STsdbRepo *pRepo) { - if (pRepo) { - tsdbFreeFS(pRepo->fs); - tsdbFreeBufPool(pRepo->pPool); - tsdbFreeMeta(pRepo->tsdbMeta); - tsdbFreeMergeBuf(pRepo->mergeBuf); - // tsdbFreeMemTable(pRepo->mem); - // tsdbFreeMemTable(pRepo->imem); - tsem_destroy(&(pRepo->readyToCommit)); - pthread_mutex_destroy(&pRepo->mutex); - free(pRepo); - } -} - -static void tsdbStartStream(STsdbRepo *pRepo) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - - for (int i = 0; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable && pTable->type == TSDB_STREAM_TABLE) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, - tsdbGetTableSchemaImpl(pTable, false, false, -1), 0); - } - } -} - -static void tsdbStopStream(STsdbRepo *pRepo) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - - for (int i = 0; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable && pTable->type == TSDB_STREAM_TABLE) { - (*pRepo->appH.cqDropFunc)(pTable->cqhandle); - } - } -} - -static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh) { - //tsdbInfo("tsdbRestoreLastColumns of table %s", pTable->name->data); - - STSchema *pSchema = tsdbGetTableLatestSchema(pTable); - if (pSchema == NULL) { - tsdbError("tsdbGetTableLatestSchema of table %s fail", pTable->name->data); - return 0; - } - - SBlock* pBlock; - int numColumns; - int32_t blockIdx; - SDataStatis* pBlockStatis = NULL; - SMemRow row = NULL; - // restore last column data with last schema - - int err = 0; - - numColumns = schemaNCols(pSchema); - if (numColumns <= pTable->restoreColumnNum) { - pTable->hasRestoreLastColumn = true; - return 0; - } - if (pTable->lastColSVersion != schemaVersion(pSchema)) { - if (tsdbInitColIdCacheWithSchema(pTable, pSchema) < 0) { - return -1; - } - } - - row = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); - if (row == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - err = -1; - goto out; - } - - memRowSetType(row, SMEM_ROW_DATA); - tdInitDataRow(memRowDataBody(row), pSchema); - - // first load block index info - if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { - err = -1; - goto out; - } - - pBlockStatis = calloc(numColumns, sizeof(SDataStatis)); - if (pBlockStatis == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - err = -1; - goto out; - } - memset(pBlockStatis, 0, numColumns * sizeof(SDataStatis)); - for(int32_t i = 0; i < numColumns; ++i) { - STColumn *pCol = schemaColAt(pSchema, i); - pBlockStatis[i].colId = pCol->colId; - } - - // load block from backward - SBlockIdx *pIdx = pReadh->pBlkIdx; - blockIdx = (int32_t)(pIdx->numOfBlocks - 1); - - while (numColumns > pTable->restoreColumnNum && blockIdx >= 0) { - bool loadStatisData = false; - pBlock = pReadh->pBlkInfo->blocks + blockIdx; - blockIdx -= 1; - - // load block data - if (tsdbLoadBlockData(pReadh, pBlock, NULL) < 0) { - err = -1; - goto out; - } - - // file block with sub-blocks has no statistics data - if (pBlock->numOfSubBlocks <= 1) { - tsdbLoadBlockStatis(pReadh, pBlock); - tsdbGetBlockStatis(pReadh, pBlockStatis, (int)numColumns); - loadStatisData = true; - } - - for (int16_t i = 0; i < numColumns && numColumns > pTable->restoreColumnNum; ++i) { - STColumn *pCol = schemaColAt(pSchema, i); - // ignore loaded columns - if (pTable->lastCols[i].bytes != 0) { - continue; - } - - // ignore block which has no not-null colId column - if (loadStatisData && pBlockStatis[i].numOfNull == pBlock->numOfRows) { - continue; - } - - // OK,let's load row from backward to get not-null column - for (int32_t rowId = pBlock->numOfRows - 1; rowId >= 0; rowId--) { - SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; - const void* pColData = tdGetColDataOfRow(pDataCol, rowId); - tdAppendColVal(memRowDataBody(row), pColData, pCol->type, pCol->offset); - //SDataCol *pDataCol = readh.pDCols[0]->cols + j; - void *value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset); - if (isNull(value, pCol->type)) { - continue; - } - - int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pCol->colId); - if (idx == -1) { - tsdbError("tsdbRestoreLastColumns restore vgId:%d,table:%s cache column %d fail", REPO_ID(pRepo), pTable->name->data, pCol->colId); - continue; - } - // save not-null column - uint16_t bytes = IS_VAR_DATA_TYPE(pCol->type) ? varDataTLen(pColData) : pCol->bytes; - SDataCol *pLastCol = &(pTable->lastCols[idx]); - pLastCol->pData = malloc(bytes); - pLastCol->bytes = bytes; - pLastCol->colId = pCol->colId; - memcpy(pLastCol->pData, value, bytes); - - // save row ts(in column 0) - pDataCol = pReadh->pDCols[0]->cols + 0; - pCol = schemaColAt(pSchema, 0); - tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->offset); - pLastCol->ts = memRowKey(row); - - pTable->restoreColumnNum += 1; - - tsdbDebug("tsdbRestoreLastColumns restore vgId:%d,table:%s cache column %d, %" PRId64, REPO_ID(pRepo), pTable->name->data, pLastCol->colId, pLastCol->ts); - break; - } - } - } - -out: - taosTZfree(row); - tfree(pBlockStatis); - - if (err == 0 && numColumns <= pTable->restoreColumnNum) { - pTable->hasRestoreLastColumn = true; - } - - return err; -} - -static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh, SBlockIdx *pIdx) { - ASSERT(pTable->lastRow == NULL); - if (tsdbLoadBlockInfo(pReadh, NULL) < 0) { - return -1; - } - - SBlock* pBlock = pReadh->pBlkInfo->blocks + pIdx->numOfBlocks - 1; - - if (tsdbLoadBlockData(pReadh, pBlock, NULL) < 0) { - return -1; - } - - // Get the data in row - - STSchema *pSchema = tsdbGetTableSchema(pTable); - pTable->lastRow = taosTMalloc(memRowMaxBytesFromSchema(pSchema)); - if (pTable->lastRow == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - memRowSetType(pTable->lastRow, SMEM_ROW_DATA); - tdInitDataRow(memRowDataBody(pTable->lastRow), pSchema); - for (int icol = 0; icol < schemaNCols(pSchema); icol++) { - STColumn *pCol = schemaColAt(pSchema, icol); - SDataCol *pDataCol = pReadh->pDCols[0]->cols + icol; - tdAppendColVal(memRowDataBody(pTable->lastRow), tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, - pCol->offset); - } - - return 0; -} - -int tsdbRestoreInfo(STsdbRepo *pRepo) { - SFSIter fsiter; - SReadH readh; - SDFileSet *pSet; - STsdbMeta *pMeta = pRepo->tsdbMeta; - STsdbCfg * pCfg = REPO_CFG(pRepo); - - if (tsdbInitReadH(&readh, pRepo) < 0) { - return -1; - } - - tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); - - if (CACHE_LAST_NULL_COLUMN(pCfg)) { - for (int i = 1; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - pTable->restoreColumnNum = 0; - pTable->hasRestoreLastColumn = false; - } - } - - while ((pSet = tsdbFSIterNext(&fsiter)) != NULL) { - if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - if (tsdbLoadBlockIdx(&readh) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - for (int i = 1; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - - //tsdbInfo("tsdbRestoreInfo restore vgId:%d,table:%s", REPO_ID(pRepo), pTable->name->data); - - if (tsdbSetReadTable(&readh, pTable) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); - SBlockIdx *pIdx = readh.pBlkIdx; - if (pIdx && lastKey < pIdx->maxKey) { - pTable->lastKey = pIdx->maxKey; - - if (CACHE_LAST_ROW(pCfg) && tsdbRestoreLastRow(pRepo, pTable, &readh, pIdx) != 0) { - tsdbDestroyReadH(&readh); - return -1; - } - } - - // restore NULL columns - if (pIdx && CACHE_LAST_NULL_COLUMN(pCfg) && !pTable->hasRestoreLastColumn) { - if (tsdbRestoreLastColumns(pRepo, pTable, &readh) != 0) { - tsdbDestroyReadH(&readh); - return -1; - } - } - } - } - - tsdbDestroyReadH(&readh); - - if (CACHE_LAST_NULL_COLUMN(pCfg)) { - atomic_store_8(&pRepo->hasCachedLastColumn, 1); - } - - return 0; -} - -int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) { - bool cacheLastRow = false, cacheLastCol = false; - SFSIter fsiter; - SReadH readh; - SDFileSet *pSet; - STsdbMeta *pMeta = pRepo->tsdbMeta; - int tableNum = 0; - int maxTableIdx = 0; - int cacheLastRowTableNum = 0; - int cacheLastColTableNum = 0; - - bool need_free_last_row = CACHE_LAST_ROW(oldCfg) && !CACHE_LAST_ROW(&(pRepo->config)); - bool need_free_last_col = CACHE_LAST_NULL_COLUMN(oldCfg) && !CACHE_LAST_NULL_COLUMN(&(pRepo->config)); - - if (CACHE_LAST_ROW(&(pRepo->config)) || CACHE_LAST_NULL_COLUMN(&(pRepo->config))) { - tsdbInfo("tsdbCacheLastData cache last data since cacheLast option changed"); - cacheLastRow = !CACHE_LAST_ROW(oldCfg) && CACHE_LAST_ROW(&(pRepo->config)); - cacheLastCol = !CACHE_LAST_NULL_COLUMN(oldCfg) && CACHE_LAST_NULL_COLUMN(&(pRepo->config)); - } - - // calc max table idx and table num - for (int i = 1; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - tableNum += 1; - maxTableIdx = i; - if (cacheLastCol) { - pTable->restoreColumnNum = 0; - pTable->hasRestoreLastColumn = false; - } - } - - // if close last option,need to free data - if (need_free_last_row || need_free_last_col) { - if (need_free_last_col) { - atomic_store_8(&pRepo->hasCachedLastColumn, 0); - } - tsdbInfo("free cache last data since cacheLast option changed"); - for (int i = 1; i <= maxTableIdx; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - if (need_free_last_row) { - taosTZfree(pTable->lastRow); - pTable->lastRow = NULL; - } - if (need_free_last_col) { - tsdbFreeLastColumns(pTable); - pTable->hasRestoreLastColumn = false; - } - } - } - - if (!cacheLastRow && !cacheLastCol) { - return 0; - } - - cacheLastRowTableNum = cacheLastRow ? tableNum : 0; - cacheLastColTableNum = cacheLastCol ? tableNum : 0; - - if (tsdbInitReadH(&readh, pRepo) < 0) { - return -1; - } - - tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); - - while ((pSet = tsdbFSIterNext(&fsiter)) != NULL && (cacheLastRowTableNum > 0 || cacheLastColTableNum > 0)) { - if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - if (tsdbLoadBlockIdx(&readh) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - for (int i = 1; i <= maxTableIdx; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - - //tsdbInfo("tsdbRestoreInfo restore vgId:%d,table:%s", REPO_ID(pRepo), pTable->name->data); - - if (tsdbSetReadTable(&readh, pTable) < 0) { - tsdbDestroyReadH(&readh); - return -1; - } - - SBlockIdx *pIdx = readh.pBlkIdx; - - if (pIdx && cacheLastRowTableNum > 0 && pTable->lastRow == NULL) { - pTable->lastKey = pIdx->maxKey; - - if (tsdbRestoreLastRow(pRepo, pTable, &readh, pIdx) != 0) { - tsdbDestroyReadH(&readh); - return -1; - } - cacheLastRowTableNum -= 1; - } - - // restore NULL columns - if (pIdx && cacheLastColTableNum > 0 && !pTable->hasRestoreLastColumn) { - if (tsdbRestoreLastColumns(pRepo, pTable, &readh) != 0) { - tsdbDestroyReadH(&readh); - return -1; - } - if (pTable->hasRestoreLastColumn) { - cacheLastColTableNum -= 1; - } - } - } - } - - tsdbDestroyReadH(&readh); - - if (cacheLastCol) { - atomic_store_8(&pRepo->hasCachedLastColumn, 1); - } - - return 0; -} diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c deleted file mode 100644 index a311868de6f7254d776f08a4f4a247293609aef5..0000000000000000000000000000000000000000 --- a/src/tsdb/src/tsdbMeta.c +++ /dev/null @@ -1,1509 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "tsdbint.h" - -#define TSDB_SUPER_TABLE_SL_LEVEL 5 -#define DEFAULT_TAG_INDEX_COLUMN 0 - -static char * getTagIndexKey(const void *pData); -static STable *tsdbNewTable(); -static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable); -static void tsdbFreeTable(STable *pTable); -static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, bool lock); -static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx, bool lock); -static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper); -static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable); -static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid); -static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup); -static int tsdbTableSetName(STableCfg *config, char *name, bool dup); -static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup); -static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); -static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid); -static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup); -static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup); -static int tsdbEncodeTableName(void **buf, tstr *name); -static void * tsdbDecodeTableName(void *buf, tstr **name); -static int tsdbEncodeTable(void **buf, STable *pTable); -static void * tsdbDecodeTable(void *buf, STable **pRTable); -static int tsdbGetTableEncodeSize(int8_t act, STable *pTable); -static void * tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STable *pTable); -static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable); -static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable); -static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid); -static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema); -static int tsdbInsertNewTableAction(STsdbRepo *pRepo, STable* pTable); -static int tsdbAddSchema(STable *pTable, STSchema *pSchema); -static void tsdbFreeTableSchema(STable *pTable); - -// ------------------ OUTER FUNCTIONS ------------------ -int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbMeta *pMeta = pRepo->tsdbMeta; - STable * super = NULL; - STable * table = NULL; - bool newSuper = false; - bool superChanged = false; - int tid = pCfg->tableId.tid; - STable * pTable = NULL; - - if (tid < 1 || tid > TSDB_MAX_TABLES) { - tsdbError("vgId:%d failed to create table since invalid tid %d", REPO_ID(pRepo), tid); - terrno = TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO; - goto _err; - } - - if (tid < pMeta->maxTables && pMeta->tables[tid] != NULL) { - if (TABLE_UID(pMeta->tables[tid]) == pCfg->tableId.uid) { - tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), - TABLE_CHAR_NAME(pMeta->tables[tid]), TABLE_TID(pMeta->tables[tid]), TABLE_UID(pMeta->tables[tid])); - return 0; - } else { - tsdbInfo("vgId:%d table %s at tid %d uid %" PRIu64 - " exists, replace it with new table, this can be not reasonable", - REPO_ID(pRepo), TABLE_CHAR_NAME(pMeta->tables[tid]), TABLE_TID(pMeta->tables[tid]), - TABLE_UID(pMeta->tables[tid])); - tsdbDropTable(pRepo, pMeta->tables[tid]->tableId); - } - } - - pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid); - if (pTable != NULL) { - tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - TABLE_TID(pTable), TABLE_UID(pTable)); - terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; - goto _err; - } - - if (pCfg->type == TSDB_CHILD_TABLE) { - super = tsdbGetTableByUid(pMeta, pCfg->superUid); - if (super == NULL) { // super table not exists, try to create it - newSuper = true; - super = tsdbCreateTableFromCfg(pCfg, true, NULL); - if (super == NULL) goto _err; - } else { - if (TABLE_TYPE(super) != TSDB_SUPER_TABLE || TABLE_UID(super) != pCfg->superUid) { - terrno = TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO; - goto _err; - } - - if (schemaVersion(pCfg->tagSchema) > schemaVersion(super->tagSchema)) { - // tag schema out of date, need to update super table tag version - STSchema *pOldSchema = super->tagSchema; - TSDB_WLOCK_TABLE(super); - super->tagSchema = tdDupSchema(pCfg->tagSchema); - TSDB_WUNLOCK_TABLE(super); - tdFreeSchema(pOldSchema); - - superChanged = true; - } - } - } - - table = tsdbCreateTableFromCfg(pCfg, false, super); - if (table == NULL) goto _err; - - // Register to meta - tsdbWLockRepoMeta(pRepo); - if (newSuper) { - if (tsdbAddTableToMeta(pRepo, super, true, false) < 0) { - tsdbUnlockRepoMeta(pRepo); - goto _err; - } - } - if (tsdbAddTableToMeta(pRepo, table, true, false) < 0) { - tsdbUnlockRepoMeta(pRepo); - goto _err; - } - tsdbUnlockRepoMeta(pRepo); - - // Write to memtable action - if (newSuper || superChanged) { - // add insert new super table action - if (tsdbInsertNewTableAction(pRepo, super) != 0) { - goto _err; - } - } - // add insert new table action - if (tsdbInsertNewTableAction(pRepo, table) != 0) { - goto _err; - } - - if (tsdbCheckCommit(pRepo) < 0) return -1; - - return 0; - -_err: - if (newSuper) { - tsdbFreeTable(super); - } - tsdbFreeTable(table); - return -1; -} - -int tsdbDropTable(STsdbRepo *repo, STableId tableId) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbMeta *pMeta = pRepo->tsdbMeta; - uint64_t uid = tableId.uid; - int tid = 0; - char * tbname = NULL; - - STable *pTable = tsdbGetTableByUid(pMeta, uid); - if (pTable == NULL) { - tsdbError("vgId:%d failed to drop table since table not exists! tid:%d uid %" PRIu64, REPO_ID(pRepo), tableId.tid, - uid); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - return -1; - } - - tsdbDebug("vgId:%d try to drop table %s type %d", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TYPE(pTable)); - - tid = TABLE_TID(pTable); - tbname = strdup(TABLE_CHAR_NAME(pTable)); - if (tbname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - // Write to KV store first - if (tsdbRemoveTableFromStore(pRepo, pTable) < 0) { - tsdbError("vgId:%d failed to drop table %s since %s", REPO_ID(pRepo), tbname, tstrerror(terrno)); - goto _err; - } - - // Remove table from Meta - if (tsdbRmTableFromMeta(pRepo, pTable) < 0) { - tsdbError("vgId:%d failed to drop table %s since %s", REPO_ID(pRepo), tbname, tstrerror(terrno)); - goto _err; - } - - tsdbDebug("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, tbname, tid, uid); - free(tbname); - - if (tsdbCheckCommit(pRepo) < 0) goto _err; - - return 0; - -_err: - tfree(tbname); - return -1; -} - -void *tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_t bytes) { - // TODO: this function should be changed also - - STSchema *pSchema = tsdbGetTableTagSchema((STable*) pTable); - STColumn *pCol = tdGetColOfID(pSchema, colId); - if (pCol == NULL) { - return NULL; // No matched tag volumn - } - - char *val = tdGetKVRowValOfCol(((STable*)pTable)->tagVal, colId); - assert(type == pCol->type && bytes >= pCol->bytes); - - // if (val != NULL && IS_VAR_DATA_TYPE(type)) { - // assert(varDataLen(val) < pCol->bytes); - // } - - return val; -} - -char *tsdbGetTableName(void* pTable) { - // TODO: need to change as thread-safe - - if (pTable == NULL) { - return NULL; - } else { - return (char*) (((STable *)pTable)->name); - } -} - -STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { - if (pMsg == NULL) return NULL; - - SSchema *pSchema = (SSchema *)pMsg->data; - int16_t numOfCols = htons(pMsg->numOfColumns); - int16_t numOfTags = htons(pMsg->numOfTags); - - STSchemaBuilder schemaBuilder = {0}; - - STableCfg *pCfg = (STableCfg *)calloc(1, sizeof(STableCfg)); - if (pCfg == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - if (tsdbInitTableCfg(pCfg, pMsg->tableType, htobe64(pMsg->uid), htonl(pMsg->tid)) < 0) goto _err; - if (tdInitTSchemaBuilder(&schemaBuilder, htonl(pMsg->sversion)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - for (int i = 0; i < numOfCols; i++) { - if (tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - } - if (tsdbTableSetSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err; - if (tsdbTableSetName(pCfg, pMsg->tableFname, true) < 0) goto _err; - - if (numOfTags > 0) { - // Decode tag schema - tdResetTSchemaBuilder(&schemaBuilder, htonl(pMsg->tversion)); - for (int i = numOfCols; i < numOfCols + numOfTags; i++) { - if (tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - } - if (tsdbTableSetTagSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err; - if (tsdbTableSetSName(pCfg, pMsg->stableFname, true) < 0) goto _err; - if (tsdbTableSetSuperUid(pCfg, htobe64(pMsg->superTableUid)) < 0) goto _err; - - int32_t tagDataLen = htonl(pMsg->tagDataLen); - if (tagDataLen) { - char *pTagData = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema); - tsdbTableSetTagValue(pCfg, pTagData, true); - } - } - - if (pMsg->tableType == TSDB_STREAM_TABLE) { - char *sql = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema); - tsdbTableSetStreamSql(pCfg, sql, true); - } - - tdDestroyTSchemaBuilder(&schemaBuilder); - - return pCfg; - -_err: - tdDestroyTSchemaBuilder(&schemaBuilder); - tsdbClearTableCfg(pCfg); - return NULL; -} - -static UNUSED_FUNC int32_t colIdCompar(const void* left, const void* right) { - int16_t colId = *(int16_t*) left; - STColumn* p2 = (STColumn*) right; - - if (colId == p2->colId) { - return 0; - } - - return (colId < p2->colId)? -1:1; -} - -int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbMeta *pMeta = pRepo->tsdbMeta; - STSchema * pNewSchema = NULL; - - pMsg->uid = htobe64(pMsg->uid); - pMsg->tid = htonl(pMsg->tid); - pMsg->tversion = htons(pMsg->tversion); - pMsg->colId = htons(pMsg->colId); - pMsg->bytes = htons(pMsg->bytes); - pMsg->tagValLen = htonl(pMsg->tagValLen); - pMsg->numOfTags = htons(pMsg->numOfTags); - pMsg->schemaLen = htonl(pMsg->schemaLen); - for (int i = 0; i < pMsg->numOfTags; i++) { - STColumn *pTCol = (STColumn *)pMsg->data + i; - pTCol->bytes = htons(pTCol->bytes); - pTCol->colId = htons(pTCol->colId); - } - - STable *pTable = tsdbGetTableByUid(pMeta, pMsg->uid); - if (pTable == NULL || TABLE_TID(pTable) != pMsg->tid) { - tsdbError("vgId:%d failed to update table tag value since invalid table id %d uid %" PRIu64, REPO_ID(pRepo), - pMsg->tid, pMsg->uid); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - return -1; - } - - if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { - tsdbError("vgId:%d try to update tag value of a non-child table, invalid action", REPO_ID(pRepo)); - terrno = TSDB_CODE_TDB_INVALID_ACTION; - return -1; - } - - if (schemaVersion(pTable->pSuper->tagSchema) > pMsg->tversion) { - tsdbError( - "vgId:%d failed to update tag value of table %s since version out of date, client tag version %d server tag " - "version %d", - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), pMsg->tversion, schemaVersion(pTable->tagSchema)); - terrno = TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE; - return -1; - } - - if (schemaVersion(pTable->pSuper->tagSchema) < pMsg->tversion) { // tag schema out of data, - tsdbDebug("vgId:%d need to update tag schema of table %s tid %d uid %" PRIu64 - " since out of date, current version %d new version %d", - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), - schemaVersion(pTable->pSuper->tagSchema), pMsg->tversion); - - STSchemaBuilder schemaBuilder = {0}; - - STColumn *pTCol = (STColumn *)pMsg->data; - ASSERT(pMsg->schemaLen % sizeof(STColumn) == 0 && pTCol[0].colId == colColId(schemaColAt(pTable->pSuper->tagSchema, 0))); - if (tdInitTSchemaBuilder(&schemaBuilder, pMsg->tversion) < 0) { - tsdbDebug("vgId:%d failed to update tag schema of table %s tid %d uid %" PRIu64 " since out of memory", - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - for (int i = 0; i < (pMsg->schemaLen / sizeof(STColumn)); i++) { - if (tdAddColToSchema(&schemaBuilder, pTCol[i].type, pTCol[i].colId, pTCol[i].bytes) < 0) { - tdDestroyTSchemaBuilder(&schemaBuilder); - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } - pNewSchema = tdGetSchemaFromBuilder(&schemaBuilder); - if (pNewSchema == NULL) { - tdDestroyTSchemaBuilder(&schemaBuilder); - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - tdDestroyTSchemaBuilder(&schemaBuilder); - } - - // Change in memory - if (pNewSchema != NULL) { // change super table tag schema - TSDB_WLOCK_TABLE(pTable->pSuper); - STSchema *pOldSchema = pTable->pSuper->tagSchema; - pTable->pSuper->tagSchema = pNewSchema; - tdFreeSchema(pOldSchema); - TSDB_WUNLOCK_TABLE(pTable->pSuper); - } - - bool isChangeIndexCol = (pMsg->colId == colColId(schemaColAt(pTable->pSuper->tagSchema, 0))); - // STColumn *pCol = bsearch(&(pMsg->colId), pMsg->data, pMsg->numOfTags, sizeof(STColumn), colIdCompar); - // ASSERT(pCol != NULL); - - if (isChangeIndexCol) { - tsdbWLockRepoMeta(pRepo); - tsdbRemoveTableFromIndex(pMeta, pTable); - } - TSDB_WLOCK_TABLE(pTable); - tdSetKVRowDataOfCol(&(pTable->tagVal), pMsg->colId, pMsg->type, POINTER_SHIFT(pMsg->data, pMsg->schemaLen)); - TSDB_WUNLOCK_TABLE(pTable); - if (isChangeIndexCol) { - tsdbAddTableIntoIndex(pMeta, pTable, false); - tsdbUnlockRepoMeta(pRepo); - } - - // Update on file - int tlen1 = (pNewSchema) ? tsdbGetTableEncodeSize(TSDB_UPDATE_META, pTable->pSuper) : 0; - int tlen2 = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pTable); - void *buf = tsdbAllocBytes(pRepo, tlen1+tlen2); - ASSERT(buf != NULL); - if (pNewSchema) { - void *pBuf = tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, pTable->pSuper); - ASSERT(POINTER_DISTANCE(pBuf, buf) == tlen1); - buf = pBuf; - } - tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, pTable); - - if (tsdbCheckCommit(pRepo) < 0) return -1; - - return 0; -} - -// ------------------ INTERNAL FUNCTIONS ------------------ -static int tsdbInsertNewTableAction(STsdbRepo *pRepo, STable* pTable) { - int tlen = 0; - void *pBuf = NULL; - - tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pTable); - pBuf = tsdbAllocBytes(pRepo, tlen); - if (pBuf == NULL) { - return -1; - } - void *tBuf = tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, pBuf, pTable); - ASSERT(POINTER_DISTANCE(tBuf, pBuf) == tlen); - - return 0; -} - -STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { - STsdbMeta *pMeta = (STsdbMeta *)calloc(1, sizeof(*pMeta)); - if (pMeta == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - int code = pthread_rwlock_init(&pMeta->rwLock, NULL); - if (code != 0) { - tsdbError("vgId:%d failed to init TSDB meta r/w lock since %s", pCfg->tsdbId, strerror(code)); - terrno = TAOS_SYSTEM_ERROR(code); - goto _err; - } - - pMeta->maxTables = TSDB_INIT_NTABLES + 1; - pMeta->tables = (STable **)calloc(pMeta->maxTables, sizeof(STable *)); - if (pMeta->tables == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - pMeta->superList = tdListNew(sizeof(STable *)); - if (pMeta->superList == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - pMeta->uidMap = taosHashInit((size_t)(TSDB_INIT_NTABLES * 1.1), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - if (pMeta->uidMap == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - return pMeta; - -_err: - tsdbFreeMeta(pMeta); - return NULL; -} - -void tsdbFreeMeta(STsdbMeta *pMeta) { - if (pMeta) { - taosHashCleanup(pMeta->uidMap); - tdListFree(pMeta->superList); - tfree(pMeta->tables); - pthread_rwlock_destroy(&pMeta->rwLock); - free(pMeta); - } -} - -int tsdbOpenMeta(STsdbRepo *pRepo) { - return 0; -#if 0 - char * fname = NULL; - STsdbMeta *pMeta = pRepo->tsdbMeta; - ASSERT(pMeta != NULL); - - fname = tsdbGetMetaFileName(pRepo->rootDir); - if (fname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - // pMeta->pStore = tdOpenKVStore(fname, tsdbRestoreTable, tsdbOrgMeta, (void *)pRepo); - // if (pMeta->pStore == NULL) { - // tsdbError("vgId:%d failed to open TSDB meta while open the kv store since %s", REPO_ID(pRepo), tstrerror(terrno)); - // goto _err; - // } - - tsdbDebug("vgId:%d open TSDB meta succeed", REPO_ID(pRepo)); - tfree(fname); - return 0; - -_err: - tfree(fname); - return -1; -#endif -} - -int tsdbCloseMeta(STsdbRepo *pRepo) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - SListNode *pNode = NULL; - STable * pTable = NULL; - - if (pMeta == NULL) return 0; - // tdCloseKVStore(pMeta->pStore); - for (int i = 1; i < pMeta->maxTables; i++) { - tsdbFreeTable(pMeta->tables[i]); - } - - while ((pNode = tdListPopHead(pMeta->superList)) != NULL) { - tdListNodeGetData(pMeta->superList, pNode, (void *)(&pTable)); - tsdbFreeTable(pTable); - listNodeFree(pNode); - } - - tsdbDebug("vgId:%d TSDB meta is closed", REPO_ID(pRepo)); - return 0; -} - -STable *tsdbGetTableByUid(STsdbMeta *pMeta, uint64_t uid) { - void *ptr = taosHashGet(pMeta->uidMap, (char *)(&uid), sizeof(uid)); - - if (ptr == NULL) return NULL; - - return *(STable **)ptr; -} - -STSchema *tsdbGetTableSchemaByVersion(STable *pTable, int16_t _version) { - return tsdbGetTableSchemaImpl(pTable, true, false, _version); -} - -int tsdbWLockRepoMeta(STsdbRepo *pRepo) { - int code = pthread_rwlock_wrlock(&(pRepo->tsdbMeta->rwLock)); - if (code != 0) { - tsdbError("vgId:%d failed to write lock TSDB meta since %s", REPO_ID(pRepo), strerror(code)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - - return 0; -} - -int tsdbRLockRepoMeta(STsdbRepo *pRepo) { - int code = pthread_rwlock_rdlock(&(pRepo->tsdbMeta->rwLock)); - if (code != 0) { - tsdbError("vgId:%d failed to read lock TSDB meta since %s", REPO_ID(pRepo), strerror(code)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - - return 0; -} - -int tsdbUnlockRepoMeta(STsdbRepo *pRepo) { - int code = pthread_rwlock_unlock(&(pRepo->tsdbMeta->rwLock)); - if (code != 0) { - tsdbError("vgId:%d failed to unlock TSDB meta since %s", REPO_ID(pRepo), strerror(code)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - - return 0; -} - -void tsdbRefTable(STable *pTable) { - int32_t ref = T_REF_INC(pTable); - UNUSED(ref); - tsdbDebug("ref table %s uid %" PRIu64 " tid:%d, refCount:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), ref); -} - -void tsdbUnRefTable(STable *pTable) { - uint64_t uid = TABLE_UID(pTable); - int32_t tid = TABLE_TID(pTable); - int32_t ref = T_REF_DEC(pTable); - - tsdbDebug("unref table, uid:%" PRIu64 " tid:%d, refCount:%d", uid, tid, ref); - - if (ref == 0) { - if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { - tsdbUnRefTable(pTable->pSuper); - } - tsdbFreeTable(pTable); - } -} - -void tsdbFreeLastColumns(STable* pTable) { - if (pTable->lastCols == NULL) { - return; - } - - for (int i = 0; i < pTable->maxColNum; ++i) { - if (pTable->lastCols[i].bytes == 0) { - continue; - } - tfree(pTable->lastCols[i].pData); - pTable->lastCols[i].bytes = 0; - pTable->lastCols[i].pData = NULL; - } - tfree(pTable->lastCols); - pTable->lastCols = NULL; - pTable->maxColNum = 0; - pTable->lastColSVersion = -1; - pTable->restoreColumnNum = 0; - pTable->hasRestoreLastColumn = false; -} - -int16_t tsdbGetLastColumnsIndexByColId(STable* pTable, int16_t colId) { - if (pTable->lastCols == NULL) { - return -1; - } - // TODO: use binary search instead - for (int16_t i = 0; i < pTable->maxColNum; ++i) { - if (pTable->lastCols[i].colId == colId) { - return i; - } - } - - return -1; -} - -int tsdbInitColIdCacheWithSchema(STable* pTable, STSchema* pSchema) { - ASSERT(pTable->lastCols == NULL); - - int16_t numOfColumn = pSchema->numOfCols; - - pTable->lastCols = (SDataCol*)malloc(numOfColumn * sizeof(SDataCol)); - if (pTable->lastCols == NULL) { - return -1; - } - - for (int16_t i = 0; i < numOfColumn; ++i) { - STColumn *pCol = schemaColAt(pSchema, i); - SDataCol* pDataCol = &(pTable->lastCols[i]); - pDataCol->bytes = 0; - pDataCol->pData = NULL; - pDataCol->colId = pCol->colId; - } - - pTable->lastColSVersion = schemaVersion(pSchema); - pTable->maxColNum = numOfColumn; - pTable->restoreColumnNum = 0; - pTable->hasRestoreLastColumn = false; - return 0; -} - -STSchema* tsdbGetTableLatestSchema(STable *pTable) { - return tsdbGetTableSchemaByVersion(pTable, -1); -} - -int tsdbUpdateLastColSchema(STable *pTable, STSchema *pNewSchema) { - if (pTable->lastColSVersion == schemaVersion(pNewSchema)) { - return 0; - } - - tsdbDebug("tsdbUpdateLastColSchema:%s,%d->%d", pTable->name->data, pTable->lastColSVersion, schemaVersion(pNewSchema)); - - int16_t numOfCols = pNewSchema->numOfCols; - SDataCol *lastCols = (SDataCol*)malloc(numOfCols * sizeof(SDataCol)); - if (lastCols == NULL) { - return -1; - } - - TSDB_WLOCK_TABLE(pTable); - - for (int16_t i = 0; i < numOfCols; ++i) { - STColumn *pCol = schemaColAt(pNewSchema, i); - int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pCol->colId); - - SDataCol* pDataCol = &(lastCols[i]); - if (idx != -1) { - // move col data to new last column array - SDataCol* pOldDataCol = &(pTable->lastCols[idx]); - memcpy(pDataCol, pOldDataCol, sizeof(SDataCol)); - } else { - // init new colid data - pDataCol->colId = pCol->colId; - pDataCol->bytes = 0; - pDataCol->pData = NULL; - } - } - - SDataCol *oldLastCols = pTable->lastCols; - int16_t oldLastColNum = pTable->maxColNum; - - pTable->lastColSVersion = schemaVersion(pNewSchema); - pTable->lastCols = lastCols; - pTable->maxColNum = numOfCols; - - if (oldLastCols == NULL) { - TSDB_WUNLOCK_TABLE(pTable); - return 0; - } - - // free old schema last column datas - for (int16_t i = 0; i < oldLastColNum; ++i) { - SDataCol* pDataCol = &(oldLastCols[i]); - if (pDataCol->bytes == 0) { - continue; - } - int16_t idx = tsdbGetLastColumnsIndexByColId(pTable, pDataCol->colId); - if (idx != -1) { - continue; - } - - // free not exist column data - tfree(pDataCol->pData); - } - TSDB_WUNLOCK_TABLE(pTable); - tfree(oldLastCols); - - return 0; -} - -void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema, bool insertAct) { - ASSERT(TABLE_TYPE(pTable) != TSDB_STREAM_TABLE && TABLE_TYPE(pTable) != TSDB_SUPER_TABLE); - STsdbMeta *pMeta = pRepo->tsdbMeta; - - STable *pCTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable; - ASSERT(schemaVersion(pSchema) > schemaVersion(*(STSchema **)taosArrayGetLast(pCTable->schema))); - - TSDB_WLOCK_TABLE(pCTable); - tsdbAddSchema(pCTable, pSchema); - - if (schemaNCols(pSchema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(pSchema); - if (schemaTLen(pSchema) > pMeta->maxRowBytes) pMeta->maxRowBytes = schemaTLen(pSchema); - TSDB_WUNLOCK_TABLE(pCTable); - - if (insertAct) { - if (tsdbInsertNewTableAction(pRepo, pCTable) != 0) { - tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " tsdbInsertNewTableAction fail", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - TABLE_TID(pTable), TABLE_UID(pTable)); - } - } -} - -int tsdbRestoreTable(STsdbRepo *pRepo, void *cont, int contLen) { - STable *pTable = NULL; - - if (!taosCheckChecksumWhole((uint8_t *)cont, contLen)) { - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - tsdbDecodeTable(cont, &pTable); - - if (tsdbAddTableToMeta(pRepo, pTable, false, false) < 0) { - tsdbFreeTable(pTable); - return -1; - } - - tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is restored from file", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - TABLE_TID(pTable), TABLE_UID(pTable)); - return 0; -} - -void tsdbOrgMeta(STsdbRepo *pRepo) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - - for (int i = 1; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable != NULL && pTable->type == TSDB_CHILD_TABLE) { - tsdbAddTableIntoIndex(pMeta, pTable, true); - } - } -} - -// ------------------ LOCAL FUNCTIONS ------------------ -static char *getTagIndexKey(const void *pData) { - STable *pTable = (STable *)pData; - - STSchema *pSchema = tsdbGetTableTagSchema(pTable); - STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN); - void * res = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId); - if (res == NULL) { - // treat the column as NULL if we cannot find it - res = (char*)getNullValue(pCol->type); - } - return res; -} - -static STable *tsdbNewTable() { - STable *pTable = (STable *)calloc(1, sizeof(*pTable)); - if (pTable == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - pTable->lastKey = TSKEY_INITIAL_VAL; - - pTable->lastCols = NULL; - pTable->restoreColumnNum = 0; - pTable->maxColNum = 0; - pTable->hasRestoreLastColumn = false; - pTable->lastColSVersion = -1; - return pTable; -} - -static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable) { - STable *pTable = NULL; - size_t tsize = 0; - - pTable = tsdbNewTable(); - if (pTable == NULL) goto _err; - - if (isSuper) { - pTable->type = TSDB_SUPER_TABLE; - tsize = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN - 1); - pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1); - if (pTable->name == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - STR_WITH_SIZE_TO_VARSTR(pTable->name, pCfg->sname, (VarDataLenT)tsize); - TABLE_UID(pTable) = pCfg->superUid; - TABLE_TID(pTable) = -1; - TABLE_SUID(pTable) = -1; - pTable->pSuper = NULL; - if (tsdbAddSchema(pTable, tdDupSchema(pCfg->schema)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - pTable->tagSchema = tdDupSchema(pCfg->tagSchema); - if (pTable->tagSchema == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - pTable->tagVal = NULL; - STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, - SL_ALLOW_DUP_KEY, getTagIndexKey); - if (pTable->pIndex == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - } else { - pTable->type = pCfg->type; - tsize = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN - 1); - pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1); - if (pTable->name == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - STR_WITH_SIZE_TO_VARSTR(pTable->name, pCfg->name, (VarDataLenT)tsize); - TABLE_UID(pTable) = pCfg->tableId.uid; - TABLE_TID(pTable) = pCfg->tableId.tid; - - if (pCfg->type == TSDB_CHILD_TABLE) { - TABLE_SUID(pTable) = pCfg->superUid; - if (tsdbCheckTableTagVal(pCfg->tagValues, pSTable->tagSchema) < 0) { - goto _err; - } - pTable->tagVal = tdKVRowDup(pCfg->tagValues); - if (pTable->tagVal == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - } else { - TABLE_SUID(pTable) = -1; - if (tsdbAddSchema(pTable, tdDupSchema(pCfg->schema)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { - pTable->sql = strdup(pCfg->sql); - if (pTable->sql == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - } - } - } - - T_REF_INC(pTable); - - tsdbDebug("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), - TABLE_UID(pTable)); - - return pTable; - -_err: - tsdbFreeTable(pTable); - return NULL; -} - -static void tsdbFreeTable(STable *pTable) { - if (pTable) { - if (pTable->name != NULL) - tsdbTrace("table %s tid %d uid %" PRIu64 " is freed", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), - TABLE_UID(pTable)); - tfree(TABLE_NAME(pTable)); - if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { - tsdbFreeTableSchema(pTable); - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tdFreeSchema(pTable->tagSchema); - } - } - - kvRowFree(pTable->tagVal); - - tSkipListDestroy(pTable->pIndex); - taosTZfree(pTable->lastRow); - tfree(pTable->sql); - - tsdbFreeLastColumns(pTable); - free(pTable); - } -} - -static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, bool lock) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - - if (lock && tsdbWLockRepoMeta(pRepo) < 0) { - tsdbError("vgId:%d failed to add table %s to meta since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - tstrerror(terrno)); - return -1; - } - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - if (tdListAppend(pMeta->superList, (void *)(&pTable)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to add table %s to meta since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - tstrerror(terrno)); - goto _err; - } - } else { - if (TABLE_TID(pTable) >= pMeta->maxTables) { - if (tsdbAdjustMetaTables(pRepo, TABLE_TID(pTable)) < 0) goto _err; - } - if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index - if (tsdbAddTableIntoIndex(pMeta, pTable, true) < 0) { - tsdbDebug("vgId:%d failed to add table %s to meta while add table to index since %s", REPO_ID(pRepo), - TABLE_CHAR_NAME(pTable), tstrerror(terrno)); - goto _err; - } - } - ASSERT(TABLE_TID(pTable) < pMeta->maxTables); - pMeta->tables[TABLE_TID(pTable)] = pTable; - pMeta->nTables++; - } - - if (taosHashPut(pMeta->uidMap, (char *)(&pTable->tableId.uid), sizeof(pTable->tableId.uid), (void *)(&pTable), - sizeof(pTable)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to add table %s to meta while put into uid map since %s", REPO_ID(pRepo), - TABLE_CHAR_NAME(pTable), tstrerror(terrno)); - goto _err; - } - - if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { - STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - if (schemaNCols(pSchema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(pSchema); - if (schemaTLen(pSchema) > pMeta->maxRowBytes) pMeta->maxRowBytes = schemaTLen(pSchema); - } - - if (lock && tsdbUnlockRepoMeta(pRepo) < 0) return -1; - if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE && addIdx) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, - tsdbGetTableSchemaImpl(pTable, false, false, -1), 1); - } - - tsdbDebug("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - TABLE_TID(pTable), TABLE_UID(pTable)); - return 0; - -_err: - tsdbRemoveTableFromMeta(pRepo, pTable, false, false); - if (lock) tsdbUnlockRepoMeta(pRepo); - return -1; -} - -static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx, bool lock) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - SListIter lIter = {0}; - SListNode *pNode = NULL; - STable * tTable = NULL; - - STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - int maxCols = schemaNCols(pSchema); - int maxRowBytes = schemaTLen(pSchema); - - if (lock) tsdbWLockRepoMeta(pRepo); - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tdListInitIter(pMeta->superList, &lIter, TD_LIST_BACKWARD); - - while ((pNode = tdListNext(&lIter)) != NULL) { - tdListNodeGetData(pMeta->superList, pNode, (void *)(&tTable)); - if (pTable == tTable) { - tdListPopNode(pMeta->superList, pNode); - free(pNode); - break; - } - } - } else { - pMeta->tables[pTable->tableId.tid] = NULL; - if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE && rmFromIdx) { - tsdbRemoveTableFromIndex(pMeta, pTable); - } - - pMeta->nTables--; - } - - taosHashRemove(pMeta->uidMap, (char *)(&(TABLE_UID(pTable))), sizeof(TABLE_UID(pTable))); - - if (maxCols == pMeta->maxCols || maxRowBytes == pMeta->maxRowBytes) { - maxCols = 0; - maxRowBytes = 0; - for (int i = 0; i < pMeta->maxTables; i++) { - STable *_pTable = pMeta->tables[i]; - if (_pTable != NULL) { - pSchema = tsdbGetTableSchemaImpl(_pTable, false, false, -1); - maxCols = MAX(maxCols, schemaNCols(pSchema)); - maxRowBytes = MAX(maxRowBytes, schemaTLen(pSchema)); - } - } - } - pMeta->maxCols = maxCols; - pMeta->maxRowBytes = maxRowBytes; - - if (lock) tsdbUnlockRepoMeta(pRepo); - tsdbDebug("vgId:%d table %s uid %" PRIu64 " is removed from meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_UID(pTable)); - tsdbUnRefTable(pTable); -} - -static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper) { - ASSERT(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); - STable *pSTable = tsdbGetTableByUid(pMeta, TABLE_SUID(pTable)); - ASSERT(pSTable != NULL); - - pTable->pSuper = pSTable; - - tSkipListPut(pSTable->pIndex, (void *)pTable); - - if (refSuper) T_REF_INC(pSTable); - return 0; -} - -static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { - ASSERT(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); - - STable *pSTable = pTable->pSuper; - ASSERT(pSTable != NULL); - - char* key = getTagIndexKey(pTable); - SArray *res = tSkipListGet(pSTable->pIndex, key); - - size_t size = taosArrayGetSize(res); - ASSERT(size > 0); - - for (int32_t i = 0; i < size; ++i) { - SSkipListNode *pNode = taosArrayGetP(res, i); - - // STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode); - if ((STable *)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need - tSkipListRemoveNode(pSTable->pIndex, pNode); - } - } - - taosArrayDestroy(res); - return 0; -} - -static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid) { - if (type != TSDB_CHILD_TABLE && type != TSDB_NORMAL_TABLE && type != TSDB_STREAM_TABLE) { - terrno = TSDB_CODE_TDB_INVALID_TABLE_TYPE; - return -1; - } - - memset((void *)config, 0, sizeof(*config)); - - config->type = type; - config->superUid = TSDB_INVALID_SUPER_TABLE_ID; - config->tableId.uid = uid; - config->tableId.tid = tid; - return 0; -} - -static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup) { - if (dup) { - config->schema = tdDupSchema(pSchema); - if (config->schema == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } else { - config->schema = pSchema; - } - return 0; -} - -static int tsdbTableSetName(STableCfg *config, char *name, bool dup) { - if (dup) { - config->name = strdup(name); - if (config->name == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } else { - config->name = name; - } - - return 0; -} - -static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup) { - if (config->type != TSDB_CHILD_TABLE) { - terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; - return -1; - } - - if (dup) { - config->tagSchema = tdDupSchema(pSchema); - if (config->tagSchema == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } else { - config->tagSchema = pSchema; - } - return 0; -} - -static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup) { - if (config->type != TSDB_CHILD_TABLE) { - terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; - return -1; - } - - if (dup) { - config->sname = strdup(sname); - if (config->sname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } else { - config->sname = sname; - } - return 0; -} - -static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid) { - if (config->type != TSDB_CHILD_TABLE || uid == TSDB_INVALID_SUPER_TABLE_ID) { - terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; - return -1; - } - - config->superUid = uid; - return 0; -} - -static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup) { - if (config->type != TSDB_CHILD_TABLE) { - terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; - return -1; - } - - if (dup) { - config->tagValues = tdKVRowDup(row); - if (config->tagValues == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } else { - config->tagValues = row; - } - - return 0; -} - -static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup) { - if (config->type != TSDB_STREAM_TABLE) { - terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; - return -1; - } - - if (dup) { - config->sql = strdup(sql); - if (config->sql == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } else { - config->sql = sql; - } - - return 0; -} - -void tsdbClearTableCfg(STableCfg *config) { - if (config) { - if (config->schema) tdFreeSchema(config->schema); - if (config->tagSchema) tdFreeSchema(config->tagSchema); - if (config->tagValues) kvRowFree(config->tagValues); - tfree(config->name); - tfree(config->sname); - tfree(config->sql); - free(config); - } -} - -static int tsdbEncodeTableName(void **buf, tstr *name) { - int tlen = 0; - - tlen += taosEncodeFixedI16(buf, name->len); - if (buf != NULL) { - memcpy(*buf, name->data, name->len); - *buf = POINTER_SHIFT(*buf, name->len); - } - tlen += name->len; - - return tlen; -} - -static void *tsdbDecodeTableName(void *buf, tstr **name) { - VarDataLenT len = 0; - - buf = taosDecodeFixedI16(buf, &len); - *name = calloc(1, sizeof(tstr) + len + 1); - if (*name == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - (*name)->len = len; - memcpy((*name)->data, buf, len); - - buf = POINTER_SHIFT(buf, len); - return buf; -} - -static int tsdbEncodeTable(void **buf, STable *pTable) { - ASSERT(pTable != NULL); - int tlen = 0; - - tlen += taosEncodeFixedU8(buf, pTable->type); - tlen += tsdbEncodeTableName(buf, pTable->name); - tlen += taosEncodeFixedU64(buf, TABLE_UID(pTable)); - tlen += taosEncodeFixedI32(buf, TABLE_TID(pTable)); - - if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { - tlen += taosEncodeFixedU64(buf, TABLE_SUID(pTable)); - tlen += tdEncodeKVRow(buf, pTable->tagVal); - } else { - uint32_t arraySize = (uint32_t)taosArrayGetSize(pTable->schema); - if(arraySize > UINT8_MAX) { - tlen += taosEncodeFixedU8(buf, 0); - tlen += taosEncodeFixedU32(buf, arraySize); - } else { - tlen += taosEncodeFixedU8(buf, (uint8_t)arraySize); - } - for (uint32_t i = 0; i < arraySize; i++) { - STSchema *pSchema = taosArrayGetP(pTable->schema, i); - tlen += tdEncodeSchema(buf, pSchema); - } - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tlen += tdEncodeSchema(buf, pTable->tagSchema); - } - - if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { - tlen += taosEncodeString(buf, pTable->sql); - } - } - - return tlen; -} - -static void *tsdbDecodeTable(void *buf, STable **pRTable) { - STable *pTable = tsdbNewTable(); - if (pTable == NULL) return NULL; - - uint8_t type = 0; - - buf = taosDecodeFixedU8(buf, &type); - pTable->type = type; - buf = tsdbDecodeTableName(buf, &(pTable->name)); - buf = taosDecodeFixedU64(buf, &TABLE_UID(pTable)); - buf = taosDecodeFixedI32(buf, &TABLE_TID(pTable)); - - if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { - buf = taosDecodeFixedU64(buf, &TABLE_SUID(pTable)); - buf = tdDecodeKVRow(buf, &(pTable->tagVal)); - } else { - uint32_t nSchemas = 0; - buf = taosDecodeFixedU8(buf, (uint8_t *)&nSchemas); - if(nSchemas == 0) { - buf = taosDecodeFixedU32(buf, &nSchemas); - } - for (int i = 0; i < nSchemas; i++) { - STSchema *pSchema; - buf = tdDecodeSchema(buf, &pSchema); - tsdbAddSchema(pTable, pSchema); - } - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - buf = tdDecodeSchema(buf, &(pTable->tagSchema)); - STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, - SL_ALLOW_DUP_KEY, getTagIndexKey); - if (pTable->pIndex == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbFreeTable(pTable); - return NULL; - } - } - - if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { - buf = taosDecodeString(buf, &(pTable->sql)); - } - } - - T_REF_INC(pTable); - - *pRTable = pTable; - - return buf; -} - -static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) { - int tlen = 0; - if (act == TSDB_UPDATE_META) { - tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM); - } else { - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_SIZE(pTable->pIndex) + 1)); - } else { - tlen = sizeof(SListNode) + sizeof(SActObj); - } - } - - return tlen; -} - -static void *tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STable *pTable) { - SListNode *pNode = (SListNode *)buf; - SActObj * pAct = (SActObj *)(pNode->data); - SActCont * pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(*pAct)); - void * pBuf = (void *)pCont; - - pNode->prev = pNode->next = NULL; - pAct->act = act; - pAct->uid = TABLE_UID(pTable); - - if (act == TSDB_UPDATE_META) { - pBuf = (void *)(pCont->cont); - pCont->len = tsdbEncodeTable(&pBuf, pTable) + sizeof(TSCKSUM); - taosCalcChecksumAppend(0, (uint8_t *)pCont->cont, pCont->len); - pBuf = POINTER_SHIFT(pBuf, sizeof(TSCKSUM)); - } - - tdListAppendNode(pRepo->mem->actList, pNode); - - return pBuf; -} - -static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable) { - int tlen = tsdbGetTableEncodeSize(TSDB_DROP_META, pTable); - void *buf = tsdbAllocBytes(pRepo, tlen); - if (buf == NULL) { - return -1; - } - - void *pBuf = buf; - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - SSkipListIterator *pIter = tSkipListCreateIter(pTable->pIndex); - if (pIter == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - while (tSkipListIterNext(pIter)) { - STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); - ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE); - pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, tTable); - } - - tSkipListDestroyIter(pIter); - } - pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, pTable); - - ASSERT(POINTER_DISTANCE(pBuf, buf) == tlen); - - return 0; -} - -static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) { - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - SSkipListIterator *pIter = tSkipListCreateIter(pTable->pIndex); - if (pIter == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - tsdbWLockRepoMeta(pRepo); - - while (tSkipListIterNext(pIter)) { - STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); - tsdbRemoveTableFromMeta(pRepo, tTable, false, false); - } - - tsdbRemoveTableFromMeta(pRepo, pTable, false, false); - - tsdbUnlockRepoMeta(pRepo); - - tSkipListDestroyIter(pIter); - - } else { - if ((TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) && pTable->cqhandle) pRepo->appH.cqDropFunc(pTable->cqhandle); - tsdbRemoveTableFromMeta(pRepo, pTable, true, true); - } - - return 0; -} - -static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - ASSERT(tid >= pMeta->maxTables); - - int maxTables = tsdbGetNextMaxTables(tid); - - STable **tables = (STable **)calloc(maxTables, sizeof(STable *)); - if (tables == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - memcpy((void *)tables, (void *)pMeta->tables, sizeof(STable *) * pMeta->maxTables); - pMeta->maxTables = maxTables; - - STable **tTables = pMeta->tables; - pMeta->tables = tables; - tfree(tTables); - tsdbDebug("vgId:%d tsdb meta maxTables is adjusted as %d", REPO_ID(pRepo), maxTables); - - return 0; -} - -static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema) { - for (size_t i = 0; i < kvRowNCols(pKVRow); i++) { - SColIdx * pColIdx = kvRowColIdxAt(pKVRow, i); - STColumn *pCol = tdGetColOfID(pSchema, pColIdx->colId); - - if ((pCol == NULL) || (!IS_VAR_DATA_TYPE(pCol->type))) continue; - - void *pValue = tdGetKVRowValOfCol(pKVRow, pCol->colId); - if (varDataTLen(pValue) > pCol->bytes) { - terrno = TSDB_CODE_TDB_IVLD_TAG_VAL; - return -1; - } - } - - return 0; -} - -static int tsdbAddSchema(STable *pTable, STSchema *pSchema) { - ASSERT(TABLE_TYPE(pTable) != TSDB_CHILD_TABLE); - - if (pTable->schema == NULL) { - pTable->schema = taosArrayInit(TSDB_MAX_TABLE_SCHEMAS, sizeof(SSchema *)); - if (pTable->schema == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - } - - ASSERT(taosArrayGetSize(pTable->schema) == 0 || - schemaVersion(pSchema) > schemaVersion(*(STSchema **)taosArrayGetLast(pTable->schema))); - - if (taosArrayPush(pTable->schema, &pSchema) == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return 0; -} - -static void tsdbFreeTableSchema(STable *pTable) { - ASSERT(pTable != NULL); - - if (pTable->schema) { - for (size_t i = 0; i < taosArrayGetSize(pTable->schema); i++) { - STSchema *pSchema = taosArrayGetP(pTable->schema, i); - tdFreeSchema(pSchema); - } - - taosArrayDestroy(pTable->schema); - } -} diff --git a/src/tsdb/src/tsdbRecover.c b/src/tsdb/src/tsdbRecover.c deleted file mode 100644 index 6dea4a4e57392be988126c579648f39a8270b9bf..0000000000000000000000000000000000000000 --- a/src/tsdb/src/tsdbRecover.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/src/tsdb/tests/CMakeLists.txt b/src/tsdb/tests/CMakeLists.txt deleted file mode 100644 index a3477aef95f9e54484c106ba2afb853b48456be0..0000000000000000000000000000000000000000 --- a/src/tsdb/tests/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - -add_executable(tsdbTests ${SOURCE_LIST}) -target_link_libraries(tsdbTests gtest gtest_main pthread common tsdb tutil trpc) - -add_test(NAME unit COMMAND ${CMAKE_CURRENT_BINARY_DIR}/tsdbTests) \ No newline at end of file diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp deleted file mode 100644 index dc804856fdf1e288f2a70a2813639c1324f42851..0000000000000000000000000000000000000000 --- a/src/tsdb/tests/tsdbTests.cpp +++ /dev/null @@ -1,163 +0,0 @@ -#include -#include -#include - -#include "tsdb.h" -#include "tsdbMain.h" - -static double getCurTime() { - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec + tv.tv_usec * 1E-6; -} - -typedef struct { - STsdbRepo *pRepo; - bool isAscend; - int tid; - uint64_t uid; - int sversion; - TSKEY startTime; - TSKEY interval; - int totalRows; - int rowsPerSubmit; - STSchema * pSchema; -} SInsertInfo; - -static int insertData(SInsertInfo *pInfo) { - SSubmitMsg *pMsg = - (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + dataRowMaxBytesFromSchema(pInfo->pSchema) * pInfo->rowsPerSubmit); - if (pMsg == NULL) return -1; - TSKEY start_time = pInfo->startTime; - - // Loop to write data - double stime = getCurTime(); - - for (int k = 0; k < pInfo->totalRows/pInfo->rowsPerSubmit; k++) { - memset((void *)pMsg, 0, sizeof(SSubmitMsg)); - SSubmitBlk *pBlock = (SSubmitBlk *)pMsg->blocks; - pBlock->uid = pInfo->uid; - pBlock->tid = pInfo->tid; - pBlock->sversion = pInfo->sversion; - pBlock->dataLen = 0; - pBlock->schemaLen = 0; - pBlock->numOfRows = 0; - for (int i = 0; i < pInfo->rowsPerSubmit; i++) { - // start_time += 1000; - if (pInfo->isAscend) { - start_time += pInfo->interval; - } else { - start_time -= pInfo->interval; - } - SDataRow row = (SDataRow)(pBlock->data + pBlock->dataLen); - tdInitDataRow(row, pInfo->pSchema); - - for (int j = 0; j < schemaNCols(pInfo->pSchema); j++) { - STColumn *pTCol = schemaColAt(pInfo->pSchema, j); - if (j == 0) { // Just for timestamp - tdAppendColVal(row, (void *)(&start_time), pTCol->type, pTCol->offset); - } else { // For int - int val = 10; - tdAppendColVal(row, (void *)(&val), pTCol->type, pTCol->offset); - } - } - pBlock->dataLen += dataRowLen(row); - pBlock->numOfRows++; - } - pMsg->length = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + pBlock->dataLen; - pMsg->numOfBlocks = 1; - - pBlock->dataLen = htonl(pBlock->dataLen); - pBlock->numOfRows = htonl(pBlock->numOfRows); - pBlock->schemaLen = htonl(pBlock->schemaLen); - pBlock->uid = htobe64(pBlock->uid); - pBlock->tid = htonl(pBlock->tid); - - pBlock->sversion = htonl(pBlock->sversion); - pBlock->padding = htonl(pBlock->padding); - - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - - if (tsdbInsertData(pInfo->pRepo, pMsg, NULL) < 0) { - tfree(pMsg); - return -1; - } - } - - double etime = getCurTime(); - - printf("Spent %f seconds to write %d records\n", etime - stime, pInfo->totalRows); - tfree(pMsg); - return 0; -} - -static void tsdbSetCfg(STsdbCfg *pCfg, int32_t tsdbId, int32_t cacheBlockSize, int32_t totalBlocks, int32_t maxTables, - int32_t daysPerFile, int32_t keep, int32_t minRows, int32_t maxRows, int8_t precision, - int8_t compression) { - pCfg->tsdbId = tsdbId; - pCfg->cacheBlockSize = cacheBlockSize; - pCfg->totalBlocks = totalBlocks; - // pCfg->maxTables = maxTables; - pCfg->daysPerFile = daysPerFile; - pCfg->keep = keep; - pCfg->minRowsPerFileBlock = minRows; - pCfg->maxRowsPerFileBlock = maxRows; - pCfg->precision = precision; - pCfg->compression = compression; -} - -static void tsdbSetTableCfg(STableCfg *pCfg) { - STSchemaBuilder schemaBuilder = {0}; - - pCfg->type = TSDB_NORMAL_TABLE; - pCfg->superUid = TSDB_INVALID_SUPER_TABLE_ID; - pCfg->tableId.tid = 1; - pCfg->tableId.uid = 5849583783847394; - tdInitTSchemaBuilder(&schemaBuilder, 0); - - int colId = 0; - for (int i = 0; i < 5; i++) { - tdAddColToSchema(&schemaBuilder, (colId == 0) ? TSDB_DATA_TYPE_TIMESTAMP : TSDB_DATA_TYPE_INT, colId, 0); - colId++; - } - - pCfg->schema = tdGetSchemaFromBuilder(&schemaBuilder); - pCfg->name = strdup("t1"); - - tdDestroyTSchemaBuilder(&schemaBuilder); -} - -TEST(TsdbTest, testInsertSpeed) { - int vnode = 1; - int ret = 0; - STsdbCfg tsdbCfg; - STableCfg tableCfg; - std::string testDir = "./test"; - char * rootDir = strdup((testDir + "/vnode" + std::to_string(vnode)).c_str()); - - tsdbDebugFlag = 131; //NOTE: you must set the flag - - taosRemoveDir(rootDir); - - // Create and open repository - tsdbSetCfg(&tsdbCfg, 1, 16, 4, -1, -1, -1, -1, -1, -1, -1); - tsdbCreateRepo(rootDir, &tsdbCfg); - STsdbRepo *repo = tsdbOpenRepo(rootDir, NULL); - ASSERT_NE(repo, nullptr); - - // Create table - tsdbSetTableCfg(&tableCfg); - tsdbCreateTable(repo, &tableCfg); - - // Insert data - SInsertInfo iInfo = {repo, true, 1, 5849583783847394, 0, 1590000000000, 10, 10000000, 100, tableCfg.schema}; - - insertData(&iInfo); - - tsdbCloseRepo(repo, 1); -} - -static char *getTKey(const void *data) { - return (char *)data; -} \ No newline at end of file diff --git a/tests/script/general/user/basic1.sim b/tests/script/general/user/basic1.sim index d4a663c096f99cfcde87399bfe57b66f424e2410..33c4dc96d41c76e9ebe2661d806cb1d0ac6b304a 100644 --- a/tests/script/general/user/basic1.sim +++ b/tests/script/general/user/basic1.sim @@ -26,7 +26,7 @@ if $rows != 2 then endi print $data00 $data01 $data02 -print $data10 $data11 $data22 +print $data10 $data11 $data12 print $data20 $data11 $data22 print $data30 $data31 $data32 @@ -38,7 +38,7 @@ if $rows != 3 then endi print $data00 $data01 $data02 -print $data10 $data11 $data22 +print $data10 $data11 $data12 print $data20 $data11 $data22 print $data30 $data31 $data32 print $data40 $data41 $data42 diff --git a/tests/script/general/user/monitor.sim b/tests/script/general/user/monitor.sim deleted file mode 100644 index 90aad59932115e04263da8cc1e8b91effb09eb07..0000000000000000000000000000000000000000 --- a/tests/script/general/user/monitor.sim +++ /dev/null @@ -1,32 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 - -print ========== step1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c monitor -v 1 -system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect - -print ========== step2 -#sql drop database log -x step21 -# return -1 -#step21: -sql drop table log.dn -x step22 -# return -1 -step22: -sql drop user log -x step23 -# return -1 -step23: - -print ========== step3 - -sleep 2000 -#sql select * from log.dn -#if $rows == 0 then -# return -1 -#endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/user/pass_len.sim b/tests/script/general/user/pass_len.sim index 5eb200b51f9ac28b19ea095be6530dbada9be894..649b3efa485b07df486b87195397ae8b4d25ef18 100644 --- a/tests/script/general/user/pass_len.sim +++ b/tests/script/general/user/pass_len.sim @@ -3,8 +3,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start - -sleep 2000 sql connect $i = 0 @@ -29,7 +27,7 @@ step12: sql create user $user PASS 'taosdata' sql show users -if $rows != 4 then +if $rows != 2 then return -1 endi @@ -40,7 +38,7 @@ sql drop user $user -x step2 step2: sql create user $user PASS '1' sql show users -if $rows != 5 then +if $rows != 3 then return -1 endi @@ -52,7 +50,7 @@ step3: sql create user $user PASS 'abc0123456789' sql show users -if $rows != 6 then +if $rows != 3 then return -1 endi @@ -63,7 +61,7 @@ sql create user $user PASS 'abcd012345678901234567891234567890' -x step4 return -1 step4: sql show users -if $rows != 6 then +if $rows != 4 then return -1 endi @@ -75,7 +73,7 @@ while $i < 3 endw sql show users -if $rows != 3 then +if $rows != 1 then return -1 endi diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 4dff6393798fb0c103048e18ab23b4f34cbff048..2408faefa0418b83ab7e1fb715e81ee4fa6a4caf 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -1,417 +1,7 @@ -cd ../../../debug; cmake .. -cd ../../../debug; make #======================b1-start=============== -./test.sh -f general/field/2.sim -./test.sh -f general/field/3.sim -./test.sh -f general/field/4.sim -./test.sh -f general/field/5.sim -./test.sh -f general/field/6.sim -./test.sh -f general/field/bigint.sim -./test.sh -f general/field/binary.sim -./test.sh -f general/field/bool.sim -./test.sh -f general/field/single.sim -./test.sh -f general/field/smallint.sim -./test.sh -f general/field/tinyint.sim +./test.sh -f general/user/basic1.sim -./test.sh -f general/http/autocreate.sim -./test.sh -f general/http/chunked.sim -./test.sh -f general/http/gzip.sim -./test.sh -f general/http/restful.sim -./test.sh -f general/http/restful_insert.sim -./test.sh -f general/http/restful_limit.sim -./test.sh -f general/http/restful_full.sim -./test.sh -f general/http/prepare.sim -./test.sh -f general/http/telegraf.sim -./test.sh -f general/http/grafana_bug.sim -./test.sh -f general/http/grafana.sim - -./test.sh -f general/insert/basic.sim -./test.sh -f general/insert/insert_drop.sim -./test.sh -f general/insert/query_block1_memory.sim -./test.sh -f general/insert/query_block2_memory.sim -./test.sh -f general/insert/query_block1_file.sim -./test.sh -f general/insert/query_block2_file.sim -./test.sh -f general/insert/query_file_memory.sim -./test.sh -f general/insert/query_multi_file.sim -./test.sh -f general/insert/tcp.sim - -./test.sh -f general/parser/alter.sim -./test.sh -f general/parser/alter1.sim -./test.sh -f general/parser/alter_stable.sim -./test.sh -f general/parser/auto_create_tb.sim -./test.sh -f general/parser/auto_create_tb_drop_tb.sim -./test.sh -f general/parser/col_arithmetic_operation.sim -./test.sh -f general/parser/columnValue.sim -./test.sh -f general/parser/commit.sim -./test.sh -f general/parser/create_db.sim -./test.sh -f general/parser/create_mt.sim -./test.sh -f general/parser/create_tb.sim -./test.sh -f general/parser/dbtbnameValidate.sim -./test.sh -f general/parser/import_commit1.sim -./test.sh -f general/parser/import_commit2.sim -./test.sh -f general/parser/import_commit3.sim -./test.sh -f general/parser/insert_tb.sim -./test.sh -f general/parser/first_last.sim -./test.sh -f general/parser/lastrow.sim -./test.sh -f general/parser/nchar.sim -./test.sh -f general/parser/null_char.sim -./test.sh -f general/parser/single_row_in_tb.sim -./test.sh -f general/parser/select_from_cache_disk.sim -./test.sh -f general/parser/mixed_blocks.sim -./test.sh -f general/parser/selectResNum.sim -./test.sh -f general/parser/limit.sim -./test.sh -f general/parser/limit1.sim -./test.sh -f general/parser/limit1_tblocks100.sim -./test.sh -f general/parser/select_across_vnodes.sim -./test.sh -f general/parser/slimit1.sim -./test.sh -f general/parser/tbnameIn.sim -./test.sh -f general/parser/projection_limit_offset.sim -./test.sh -f general/parser/limit2.sim -./test.sh -f general/parser/fill.sim -./test.sh -f general/parser/fill_stb.sim -./test.sh -f general/parser/where.sim -./test.sh -f general/parser/slimit.sim -./test.sh -f general/parser/select_with_tags.sim -./test.sh -f general/parser/interp.sim -./test.sh -f general/parser/tags_dynamically_specifiy.sim -./test.sh -f general/parser/groupby.sim -./test.sh -f general/parser/set_tag_vals.sim -./test.sh -f general/parser/tags_filter.sim -./test.sh -f general/parser/slimit_alter_tags.sim -./test.sh -f general/parser/join.sim -./test.sh -f general/parser/join_multivnode.sim -./test.sh -f general/parser/binary_escapeCharacter.sim -./test.sh -f general/parser/repeatAlter.sim -./test.sh -f general/parser/union.sim -./test.sh -f general/parser/topbot.sim -./test.sh -f general/db/nosuchfile.sim -./test.sh -f general/parser/function.sim -./test.sh -f unique/cluster/vgroup100.sim - -./test.sh -f unique/http/admin.sim -./test.sh -f unique/http/opentsdb.sim - -./test.sh -f unique/import/replica2.sim -./test.sh -f unique/import/replica3.sim - -./test.sh -f general/alter/cached_schema_after_alter.sim #======================b1-end=============== -#======================b2-start=============== - - -./test.sh -f general/wal/sync.sim -./test.sh -f general/wal/kill.sim -./test.sh -f general/wal/maxtables.sim - -./test.sh -f general/user/authority.sim -./test.sh -f general/user/monitor.sim -./test.sh -f general/user/pass_alter.sim -./test.sh -f general/user/pass_len.sim -./test.sh -f general/user/user_create.sim -./test.sh -f general/user/user_len.sim - -./test.sh -f general/vector/metrics_field.sim -./test.sh -f general/vector/metrics_mix.sim -./test.sh -f general/vector/metrics_query.sim -./test.sh -f general/vector/metrics_tag.sim -./test.sh -f general/vector/metrics_time.sim -./test.sh -f general/vector/multi.sim -./test.sh -f general/vector/single.sim -./test.sh -f general/vector/table_field.sim -./test.sh -f general/vector/table_mix.sim -./test.sh -f general/vector/table_query.sim -./test.sh -f general/vector/table_time.sim - -./test.sh -f unique/account/account_create.sim -./test.sh -f unique/account/account_delete.sim -./test.sh -f unique/account/account_len.sim -./test.sh -f unique/account/authority.sim -./test.sh -f unique/account/basic.sim -./test.sh -f unique/account/paras.sim -./test.sh -f unique/account/pass_alter.sim -./test.sh -f unique/account/pass_len.sim -./test.sh -f unique/account/usage.sim -./test.sh -f unique/account/user_create.sim -./test.sh -f unique/account/user_len.sim - -./test.sh -f unique/big/maxvnodes.sim -./test.sh -f unique/big/tcp.sim - -./test.sh -f unique/cluster/alter.sim -./test.sh -f unique/cluster/cache.sim -./test.sh -f unique/http/admin.sim -./test.sh -f unique/http/opentsdb.sim - -./test.sh -f unique/import/replica2.sim -./test.sh -f unique/import/replica3.sim - -./test.sh -f general/alter/cached_schema_after_alter.sim - - -#======================b2-end=============== -#======================b3-start=============== - -./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim -#./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim -./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim -./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim -#./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim -#./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim -./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim -./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim -#./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim -# lower the priority while file corruption -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim -./test.sh -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim -./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim -./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim - -./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim -./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim -./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim -./test.sh -f unique/arbitrator/offline_replica2_dropDb_online.sim -./test.sh -f unique/arbitrator/offline_replica2_dropTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_alterTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_alterTag_online.sim -./test.sh -f unique/arbitrator/offline_replica3_createTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_dropDb_online.sim -./test.sh -f unique/arbitrator/offline_replica3_dropTable_online.sim -./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim -./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim -./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim - -./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim -./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim -./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim -./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim -./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim -./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim - -./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim -./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim -./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim -./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim - -./test.sh -f unique/stable/balance_replica1.sim -./test.sh -f unique/stable/dnode2_stop.sim -./test.sh -f unique/stable/dnode2.sim -./test.sh -f unique/stable/dnode3.sim -./test.sh -f unique/stable/replica2_dnode4.sim -./test.sh -f unique/stable/replica2_vnode3.sim -./test.sh -f unique/stable/replica3_dnode6.sim -./test.sh -f unique/stable/replica3_vnode3.sim - -#======================b3-end=============== -#======================b4-start=============== - - -./test.sh -f general/alter/count.sim -./test.sh -f general/alter/dnode.sim -./test.sh -f general/alter/import.sim -./test.sh -f general/alter/insert1.sim -./test.sh -f general/alter/insert2.sim -./test.sh -f general/alter/metrics.sim -./test.sh -f general/alter/table.sim - -./test.sh -f general/cache/new_metrics.sim -./test.sh -f general/cache/restart_metrics.sim -./test.sh -f general/cache/restart_table.sim - -./test.sh -f general/connection/connection.sim - -./test.sh -f general/column/commit.sim -./test.sh -f general/column/metrics.sim -./test.sh -f general/column/table.sim - -./test.sh -f general/compress/commitlog.sim -./test.sh -f general/compress/compress.sim -./test.sh -f general/compress/compress2.sim -./test.sh -f general/compress/uncompress.sim - -./test.sh -f general/stable/disk.sim -./test.sh -f general/stable/dnode3.sim -./test.sh -f general/stable/metrics.sim -./test.sh -f general/stable/refcount.sim -./test.sh -f general/stable/show.sim -./test.sh -f general/stable/values.sim -./test.sh -f general/stable/vnode3.sim - -./test.sh -f unique/column/replica3.sim -./test.sh -f issue/TD-2713.sim -./test.sh -f general/parser/select_distinct_tag.sim -./test.sh -f unique/mnode/mgmt30.sim -./test.sh -f issue/TD-2677.sim -./test.sh -f issue/TD-2680.sim -./test.sh -f unique/dnode/lossdata.sim - -#======================b4-end=============== -#======================b5-start=============== - -./test.sh -f unique/dnode/alternativeRole.sim -./test.sh -f unique/dnode/balance1.sim -./test.sh -f unique/dnode/balance2.sim -./test.sh -f unique/dnode/balance3.sim -./test.sh -f unique/dnode/balancex.sim -./test.sh -f unique/dnode/offline1.sim -./test.sh -f unique/dnode/offline2.sim - -./test.sh -f general/stream/metrics_del.sim -./test.sh -f general/stream/metrics_replica1_vnoden.sim -./test.sh -f general/stream/restart_stream.sim -./test.sh -f general/stream/stream_3.sim -./test.sh -f general/stream/stream_restart.sim -./test.sh -f general/stream/table_del.sim -./test.sh -f general/stream/table_replica1_vnoden.sim - -./test.sh -f general/connection/test_old_data.sim -./test.sh -f unique/dnode/datatrans_3node.sim -./test.sh -f unique/dnode/datatrans_3node_2.sim -./test.sh -f general/db/alter_tables_d2.sim -./test.sh -f general/db/alter_tables_v1.sim -./test.sh -f general/db/alter_tables_v4.sim - -#======================b5-end=============== -#======================b6-start=============== - -./test.sh -f unique/dnode/reason.sim -./test.sh -f unique/dnode/remove1.sim -./test.sh -f unique/dnode/remove2.sim -./test.sh -f unique/dnode/vnode_clean.sim - -./test.sh -f unique/db/commit.sim -./test.sh -f unique/db/delete.sim -./test.sh -f unique/db/delete_part.sim -./test.sh -f unique/db/replica_add12.sim -./test.sh -f unique/db/replica_add13.sim -./test.sh -f unique/db/replica_add23.sim -./test.sh -f unique/db/replica_reduce21.sim -./test.sh -f unique/db/replica_reduce32.sim -./test.sh -f unique/db/replica_reduce31.sim -./test.sh -f unique/db/replica_part.sim - -./test.sh -f unique/vnode/many.sim -./test.sh -f unique/vnode/replica2_basic2.sim -./test.sh -f unique/vnode/replica2_repeat.sim -./test.sh -f unique/vnode/replica3_basic.sim -./test.sh -f unique/vnode/replica3_repeat.sim -./test.sh -f unique/vnode/replica3_vgroup.sim - -./test.sh -f unique/dnode/monitor.sim -./test.sh -f unique/dnode/monitor_bug.sim -./test.sh -f unique/dnode/simple.sim -./test.sh -f unique/dnode/data1.sim -./test.sh -f unique/dnode/m2.sim -./test.sh -f unique/dnode/m3.sim -./test.sh -f unique/dnode/offline3.sim -./test.sh -f general/wal/kill.sim -./test.sh -f general/wal/maxtables.sim - -./test.sh -f general/import/basic.sim -./test.sh -f general/import/commit.sim -./test.sh -f general/import/large.sim -./test.sh -f general/import/replica1.sim -./test.sh -f unique/cluster/balance1.sim -./test.sh -f unique/cluster/balance2.sim -./test.sh -f unique/cluster/balance3.sim - -#======================b6-end=============== -#======================b7-start=============== - -./test.sh -f general/compute/avg.sim -./test.sh -f general/compute/bottom.sim -./test.sh -f general/compute/count.sim -./test.sh -f general/compute/diff.sim -./test.sh -f general/compute/diff2.sim -./test.sh -f general/compute/first.sim -./test.sh -f general/compute/interval.sim -./test.sh -f general/compute/last.sim -./test.sh -f general/compute/leastsquare.sim -./test.sh -f general/compute/max.sim -./test.sh -f general/compute/min.sim -./test.sh -f general/compute/null.sim -./test.sh -f general/compute/percentile.sim -./test.sh -f general/compute/stddev.sim -./test.sh -f general/compute/sum.sim -./test.sh -f general/compute/top.sim - -./test.sh -f general/db/alter_option.sim -./test.sh -f general/db/alter_vgroups.sim -./test.sh -f general/db/basic.sim -./test.sh -f general/db/basic1.sim -./test.sh -f general/db/basic2.sim -./test.sh -f general/db/basic3.sim -./test.sh -f general/db/basic4.sim -./test.sh -f general/db/basic5.sim -./test.sh -f general/db/delete_reuse1.sim -./test.sh -f general/db/delete_reuse2.sim -./test.sh -f general/db/delete_reusevnode.sim -./test.sh -f general/db/delete_reusevnode2.sim -./test.sh -f general/db/delete_writing1.sim -./test.sh -f general/db/delete_writing2.sim -./test.sh -f general/db/delete.sim -./test.sh -f general/db/len.sim -./test.sh -f general/db/repeat.sim -./test.sh -f general/db/tables.sim -./test.sh -f general/db/vnodes.sim -./test.sh -f general/db/topic1.sim -./test.sh -f general/db/topic2.sim -./test.sh -f general/table/autocreate.sim -./test.sh -f general/table/basic1.sim -./test.sh -f general/table/basic2.sim -./test.sh -f general/table/basic3.sim -./test.sh -f general/table/bigint.sim -./test.sh -f general/table/binary.sim -./test.sh -f general/table/bool.sim -./test.sh -f general/table/column_name.sim -./test.sh -f general/table/column_num.sim -./test.sh -f general/table/column_value.sim -./test.sh -f general/table/column2.sim -./test.sh -f general/table/date.sim -./test.sh -f general/table/db.table.sim -./test.sh -f general/table/delete_reuse1.sim -./test.sh -f general/table/delete_reuse2.sim -./test.sh -f general/table/delete_writing.sim -./test.sh -f general/table/describe.sim -./test.sh -f general/table/double.sim -./test.sh -f general/table/float.sim -./test.sh -f general/table/int.sim -./test.sh -f general/table/limit.sim -./test.sh -f general/table/smallint.sim -./test.sh -f general/table/table_len.sim -./test.sh -f general/table/table.sim -./test.sh -f general/table/tinyint.sim -./test.sh -f general/table/vgroup.sim -./test.sh -f general/table/createmulti.sim - -./test.sh -f unique/mnode/mgmt20.sim -./test.sh -f unique/mnode/mgmt21.sim -./test.sh -f unique/mnode/mgmt22.sim -./test.sh -f unique/mnode/mgmt23.sim -./test.sh -f unique/mnode/mgmt24.sim -./test.sh -f unique/mnode/mgmt25.sim -./test.sh -f unique/mnode/mgmt26.sim -./test.sh -f unique/mnode/mgmt33.sim -./test.sh -f unique/mnode/mgmt34.sim -./test.sh -f unique/mnode/mgmtr2.sim - -./test.sh -f unique/arbitrator/insert_duplicationTs.sim -./test.sh -f general/parser/join_manyblocks.sim -./test.sh -f general/parser/stableOp.sim -./test.sh -f general/parser/timestamp.sim -./test.sh -f general/parser/sliding.sim -./test.sh -f general/parser/having.sim -./test.sh -f general/parser/having_child.sim -./test.sh -f general/parser/between_and.sim -./test.sh -f general/parser/last_cache.sim -./test.sh -f unique/big/balance.sim - -#======================b7-end=============== diff --git a/tests/test-all.sh b/tests/test-all.sh index eea623b27e482d67e0d3e94a27c7f4376449d556..29aecd2f0880744fdc1a2f026d3dbbf7f32d9ff8 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -41,7 +41,7 @@ function dohavecore(){ else cd ../../ if [[ $1 == 1 ]];then - tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug/build/bin/taosd debug/build/bin/tsim debug/build/lib/libtaos*so* + #tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` fi fi @@ -67,8 +67,8 @@ function runSimCaseOneByOne { else echo -n $case ./test.sh -f $case > /dev/null 2>&1 && \ - ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ - ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ( grep -q 'script.*success.*m$' ../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ echo -e "${RED} failed${NC}" | tee -a out.log fi out_log=`tail -1 out.log ` @@ -99,11 +99,12 @@ function runSimCaseOneByOnefq { ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) else + pwd echo -n $case ./test.sh -f $case > ../../sim/case.log 2>&1 && \ ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ - ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) + ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && pwd && cat ../../sim/case.log ) fi out_log=`tail -1 out.log `